Jason WittyandClaude Opus 5.5
0c519cd9a3
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1690 lines
57 KiB
JSON
1690 lines
57 KiB
JSON
{
|
|
"format": "strata-standard-benchmark",
|
|
"schema_version": 1,
|
|
"protocol": {
|
|
"id": "strata-qwen38-thinking-v1-draft.1",
|
|
"git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"profile": "qwen3.8-flash-next-thinking",
|
|
"corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407",
|
|
"model_card_revision": null
|
|
},
|
|
"run": {
|
|
"id": "20261027T035636Z-16ea71d0",
|
|
"started_at": "2026-10-27T03:56:36Z",
|
|
"status": "nonconforming",
|
|
"required_duration_s": 95.043,
|
|
"optional_duration_s": null,
|
|
"optional_concurrency": false,
|
|
"command": "<user> bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes power cap 275W rep 2, case closed --quantization IQ3_S"
|
|
},
|
|
"hardware": {
|
|
"gpus": [
|
|
{
|
|
"device_index": 0,
|
|
"name": "NVIDIA GeForce RTX 3090",
|
|
"memory_bytes": 25769803776,
|
|
"memory_kind": "dedicated",
|
|
"power_limit_w": 275.0
|
|
}
|
|
],
|
|
"cpu": "Intel(R) Core(TM) i9-14900KS",
|
|
"ram_bytes": 118089846784,
|
|
"storage": null,
|
|
"background_workloads": null
|
|
},
|
|
"software": {
|
|
"os": {
|
|
"name": "NixOS 26.05 (Yarara)",
|
|
"version": "6.18.55"
|
|
},
|
|
"drivers": [
|
|
{
|
|
"name": "gpu-driver",
|
|
"version": "595.71.05"
|
|
}
|
|
],
|
|
"runtime": [
|
|
{
|
|
"name": "cuda",
|
|
"version": "13.2"
|
|
}
|
|
],
|
|
"strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"engine_version": "0.1.51",
|
|
"build_kind": "source",
|
|
"runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"build_options": null
|
|
},
|
|
"model": {
|
|
"id": "qwen3.8-flash-next-iq3_s",
|
|
"revision": null,
|
|
"source_revision": null,
|
|
"quantization": "IQ3_S",
|
|
"files": null,
|
|
"tokenizer_revision": null,
|
|
"custom_pack": null
|
|
},
|
|
"configuration": {
|
|
"track": "custom",
|
|
"track_notes": "power cap 275W rep 2, case closed",
|
|
"context_tokens": 131072,
|
|
"server_capacity": 1,
|
|
"kv_cache": null,
|
|
"expert_cache": null,
|
|
"offloading": null,
|
|
"prefill": null,
|
|
"mtp": null,
|
|
"calibration": null,
|
|
"experimental_options": null,
|
|
"template": {
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
"sampling": {
|
|
"temperature": 1.0,
|
|
"top_p": 0.95,
|
|
"top_k": 20,
|
|
"min_p": 0.0,
|
|
"presence_penalty": 0.0,
|
|
"repetition_penalty": 1.0,
|
|
"reasoning_effort": "xhigh"
|
|
},
|
|
"chat_template_kwargs": {
|
|
"enable_thinking": true,
|
|
"preserve_thinking": true
|
|
}
|
|
},
|
|
"request_counts": {
|
|
"warmup": {
|
|
"planned": 1,
|
|
"attempted": 1,
|
|
"completed": 0,
|
|
"truncated": 1,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"performance": {
|
|
"planned": 9,
|
|
"attempted": 9,
|
|
"completed": 3,
|
|
"truncated": 6,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"sanity": {
|
|
"planned": 12,
|
|
"attempted": 12,
|
|
"completed": 12,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"concurrency": {
|
|
"planned": 0,
|
|
"attempted": 0,
|
|
"completed": 0,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
}
|
|
},
|
|
"requests": [
|
|
{
|
|
"id": "warmup",
|
|
"phase": "warmup",
|
|
"case_id": "warmup",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 128,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 75,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 75,
|
|
"output_tokens": 128,
|
|
"decode_tokens": 128
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4041,
|
|
"decode_s": 1.1877,
|
|
"ttft_s": 0.431352,
|
|
"first_answer_s": 1.0375,
|
|
"total_s": 1.595383
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 185.597624,
|
|
"decode_tps": 107.771323
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r1",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 474,
|
|
"decode_tokens": 474
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3666,
|
|
"decode_s": 3.7942,
|
|
"ttft_s": 0.383818,
|
|
"first_answer_s": 0.880742,
|
|
"total_s": 4.165678
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 210.038189,
|
|
"decode_tps": 124.927521
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.609,
|
|
"decode_s": 5.2227,
|
|
"ttft_s": 2.629003,
|
|
"first_answer_s": null,
|
|
"total_s": 7.839609
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1568.800307,
|
|
"decode_tps": 98.033584
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.5282,
|
|
"decode_s": 4.98,
|
|
"ttft_s": 5.558195,
|
|
"first_answer_s": null,
|
|
"total_s": 10.526218
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2964.255997,
|
|
"decode_tps": 102.811245
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r2",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 368,
|
|
"decode_tokens": 368
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4564,
|
|
"decode_s": 3.2219,
|
|
"ttft_s": 0.472927,
|
|
"first_answer_s": 1.173951,
|
|
"total_s": 3.682968
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 168.711656,
|
|
"decode_tps": 114.218318
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3479,
|
|
"decode_s": 4.5724,
|
|
"ttft_s": 2.369827,
|
|
"first_answer_s": 6.049044,
|
|
"total_s": 6.929865
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1743.259934,
|
|
"decode_tps": 111.976205
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.4276,
|
|
"decode_s": 4.612,
|
|
"ttft_s": 5.450919,
|
|
"first_answer_s": null,
|
|
"total_s": 10.05188
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 3019.198172,
|
|
"decode_tps": 111.014744
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r3",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 493,
|
|
"decode_tokens": 493
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4537,
|
|
"decode_s": 4.3846,
|
|
"ttft_s": 0.470928,
|
|
"first_answer_s": 3.134974,
|
|
"total_s": 4.843024
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 169.715671,
|
|
"decode_tps": 112.438991
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.2959,
|
|
"decode_s": 4.5861,
|
|
"ttft_s": 2.314749,
|
|
"first_answer_s": null,
|
|
"total_s": 6.888165
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1782.743151,
|
|
"decode_tps": 111.6417
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.4342,
|
|
"decode_s": 3.9326,
|
|
"ttft_s": 5.45946,
|
|
"first_answer_s": 7.632408,
|
|
"total_s": 9.37984
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 3015.531265,
|
|
"decode_tps": 130.193765
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "sanity-I1",
|
|
"phase": "sanity",
|
|
"case_id": "I1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 82,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 82,
|
|
"output_tokens": 76,
|
|
"decode_tokens": 76
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4832,
|
|
"decode_s": 0.6419,
|
|
"ttft_s": 0.499436,
|
|
"first_answer_s": 1.007442,
|
|
"total_s": 1.129499
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 169.701987,
|
|
"decode_tps": 118.398504
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I2",
|
|
"phase": "sanity",
|
|
"case_id": "I2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 90,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 90,
|
|
"output_tokens": 60,
|
|
"decode_tokens": 60
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3541,
|
|
"decode_s": 0.451,
|
|
"ttft_s": 0.370771,
|
|
"first_answer_s": 0.704771,
|
|
"total_s": 0.810818
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 254.16549,
|
|
"decode_tps": 133.037694
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I3",
|
|
"phase": "sanity",
|
|
"case_id": "I3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 85,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 85,
|
|
"output_tokens": 89,
|
|
"decode_tokens": 89
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.361,
|
|
"decode_s": 0.6975,
|
|
"ttft_s": 0.378858,
|
|
"first_answer_s": 0.86987,
|
|
"total_s": 1.066093
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 235.457064,
|
|
"decode_tps": 127.598566
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I4",
|
|
"phase": "sanity",
|
|
"case_id": "I4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 71,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 71,
|
|
"output_tokens": 42,
|
|
"decode_tokens": 42
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3611,
|
|
"decode_s": 0.342,
|
|
"ttft_s": 0.375516,
|
|
"first_answer_s": 0.666532,
|
|
"total_s": 0.706724
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 196.621435,
|
|
"decode_tps": 122.807018
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R1",
|
|
"phase": "sanity",
|
|
"case_id": "R1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 63,
|
|
"decode_tokens": 63
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3279,
|
|
"decode_s": 0.514,
|
|
"ttft_s": 2.346661,
|
|
"first_answer_s": 2.786657,
|
|
"total_s": 2.848697
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1756.518751,
|
|
"decode_tps": 122.568093
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R2",
|
|
"phase": "sanity",
|
|
"case_id": "R2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 64,
|
|
"decode_tokens": 64
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.2861,
|
|
"decode_s": 0.5087,
|
|
"ttft_s": 2.30392,
|
|
"first_answer_s": 2.742917,
|
|
"total_s": 2.800213
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1788.635668,
|
|
"decode_tps": 125.810891
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R3",
|
|
"phase": "sanity",
|
|
"case_id": "R3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16382,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16382,
|
|
"output_tokens": 67,
|
|
"decode_tokens": 67
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.3776,
|
|
"decode_s": 0.4965,
|
|
"ttft_s": 5.409518,
|
|
"first_answer_s": 5.837574,
|
|
"total_s": 5.894858
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 3046.340375,
|
|
"decode_tps": 134.944612
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R4",
|
|
"phase": "sanity",
|
|
"case_id": "R4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16390,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16390,
|
|
"output_tokens": 222,
|
|
"decode_tokens": 222
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.5407,
|
|
"decode_s": 1.8556,
|
|
"ttft_s": 5.563441,
|
|
"first_answer_s": 7.176501,
|
|
"total_s": 7.406744
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2958.109986,
|
|
"decode_tps": 119.637853
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T1",
|
|
"phase": "sanity",
|
|
"case_id": "T1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 325,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 325,
|
|
"output_tokens": 61,
|
|
"decode_tokens": 61
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.2924,
|
|
"decode_s": 0.4638,
|
|
"ttft_s": 1.310527,
|
|
"first_answer_s": 1.617554,
|
|
"total_s": 1.763561
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 251.470133,
|
|
"decode_tps": 131.522208
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T2",
|
|
"phase": "sanity",
|
|
"case_id": "T2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 335,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 335,
|
|
"output_tokens": 88,
|
|
"decode_tokens": 88
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9012,
|
|
"decode_s": 0.6932,
|
|
"ttft_s": 0.917,
|
|
"first_answer_s": 1.380054,
|
|
"total_s": 1.599908
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 371.726587,
|
|
"decode_tps": 126.94749
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T3",
|
|
"phase": "sanity",
|
|
"case_id": "T3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 352,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 352,
|
|
"output_tokens": 90,
|
|
"decode_tokens": 90
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9017,
|
|
"decode_s": 0.598,
|
|
"ttft_s": 0.918879,
|
|
"first_answer_s": 1.222893,
|
|
"total_s": 1.504977
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 390.373738,
|
|
"decode_tps": 150.501672
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T4",
|
|
"phase": "sanity",
|
|
"case_id": "T4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 339,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 339,
|
|
"output_tokens": 100,
|
|
"decode_tokens": 100
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.8672,
|
|
"decode_s": 0.7255,
|
|
"ttft_s": 0.884768,
|
|
"first_answer_s": 1.373774,
|
|
"total_s": 1.599804
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 390.913284,
|
|
"decode_tps": 137.835975
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
}
|
|
],
|
|
"summaries": [
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 169.715671,
|
|
"min": 168.711656,
|
|
"max": 210.038189,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 114.218318,
|
|
"min": 112.438991,
|
|
"max": 124.927521,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 0.470928,
|
|
"min": 0.383818,
|
|
"max": 0.472927,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 1.173951,
|
|
"min": 0.880742,
|
|
"max": 3.134974,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 4.165678,
|
|
"min": 3.682968,
|
|
"max": 4.843024,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 1743.259934,
|
|
"min": 1568.800307,
|
|
"max": 1782.743151,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 111.6417,
|
|
"min": 98.033584,
|
|
"max": 111.976205,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 2.369827,
|
|
"min": 2.314749,
|
|
"max": 2.629003,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 6.049044,
|
|
"min": 6.049044,
|
|
"max": 6.049044,
|
|
"n": 1,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r2"
|
|
],
|
|
"excluded": [
|
|
{
|
|
"request_id": "performance-prompt-4k-r1",
|
|
"reason": "first_answer_s is not available for this request"
|
|
},
|
|
{
|
|
"request_id": "performance-prompt-4k-r3",
|
|
"reason": "first_answer_s is not available for this request"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 6.929865,
|
|
"min": 6.888165,
|
|
"max": 7.839609,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 3015.531265,
|
|
"min": 2964.255997,
|
|
"max": 3019.198172,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 111.014744,
|
|
"min": 102.811245,
|
|
"max": 130.193765,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 5.45946,
|
|
"min": 5.450919,
|
|
"max": 5.558195,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 7.632408,
|
|
"min": 7.632408,
|
|
"max": 7.632408,
|
|
"n": 1,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": [
|
|
{
|
|
"request_id": "performance-prompt-16k-r1",
|
|
"reason": "first_answer_s is not available for this request"
|
|
},
|
|
{
|
|
"request_id": "performance-prompt-16k-r2",
|
|
"reason": "first_answer_s is not available for this request"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 10.05188,
|
|
"min": 9.37984,
|
|
"max": 10.526218,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
}
|
|
],
|
|
"concurrency_rounds": [],
|
|
"quality": {
|
|
"passed": 12,
|
|
"total": 12,
|
|
"categories": {
|
|
"instruction": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"retrieval": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"tool_call": {
|
|
"passed": 4,
|
|
"total": 4
|
|
}
|
|
}
|
|
},
|
|
"artifacts": [
|
|
{
|
|
"kind": "outputs",
|
|
"path": "outputs.jsonl",
|
|
"sha256": "d22002c44aadd2bdf78e19be1933c5d3f27bb206179fd68ad81b78ff04e138ad"
|
|
},
|
|
{
|
|
"kind": "config",
|
|
"path": "config.json",
|
|
"sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-manifest.sha256",
|
|
"sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-build.json",
|
|
"sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2"
|
|
}
|
|
],
|
|
"missing": {
|
|
"/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)",
|
|
"/run/optional_duration_s": "the optional concurrency stage was not selected",
|
|
"/hardware/storage": "the drive holding the model files could not be identified",
|
|
"/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background",
|
|
"/software/build_options": "the server does not report build options; pass --build-options for a source build",
|
|
"/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",
|
|
"/model/source_revision": "not given: pass --source-revision",
|
|
"/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)",
|
|
"/model/tokenizer_revision": "not given: pass --tokenizer-revision",
|
|
"/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)",
|
|
"/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/requests/0/round": "not applicable outside the concurrency stage",
|
|
"/requests/0/error": "none: the request ended normally",
|
|
"/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/0/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/1/round": "not applicable outside the concurrency stage",
|
|
"/requests/1/error": "none: the request ended normally",
|
|
"/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/1/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/2/round": "not applicable outside the concurrency stage",
|
|
"/requests/2/error": "none: the request ended normally",
|
|
"/requests/2/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/2/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/3/round": "not applicable outside the concurrency stage",
|
|
"/requests/3/error": "none: the request ended normally",
|
|
"/requests/3/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/3/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/4/round": "not applicable outside the concurrency stage",
|
|
"/requests/4/error": "none: the request ended normally",
|
|
"/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/4/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/5/round": "not applicable outside the concurrency stage",
|
|
"/requests/5/error": "none: the request ended normally",
|
|
"/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/5/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/6/round": "not applicable outside the concurrency stage",
|
|
"/requests/6/error": "none: the request ended normally",
|
|
"/requests/6/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/6/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/7/round": "not applicable outside the concurrency stage",
|
|
"/requests/7/error": "none: the request ended normally",
|
|
"/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/7/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/8/round": "not applicable outside the concurrency stage",
|
|
"/requests/8/error": "none: the request ended normally",
|
|
"/requests/8/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/8/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/9/round": "not applicable outside the concurrency stage",
|
|
"/requests/9/error": "none: the request ended normally",
|
|
"/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/9/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/10/round": "not applicable outside the concurrency stage",
|
|
"/requests/10/error": "none: the request ended normally",
|
|
"/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/11/round": "not applicable outside the concurrency stage",
|
|
"/requests/11/error": "none: the request ended normally",
|
|
"/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/12/round": "not applicable outside the concurrency stage",
|
|
"/requests/12/error": "none: the request ended normally",
|
|
"/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/13/round": "not applicable outside the concurrency stage",
|
|
"/requests/13/error": "none: the request ended normally",
|
|
"/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/14/round": "not applicable outside the concurrency stage",
|
|
"/requests/14/error": "none: the request ended normally",
|
|
"/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/15/round": "not applicable outside the concurrency stage",
|
|
"/requests/15/error": "none: the request ended normally",
|
|
"/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/16/round": "not applicable outside the concurrency stage",
|
|
"/requests/16/error": "none: the request ended normally",
|
|
"/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/17/round": "not applicable outside the concurrency stage",
|
|
"/requests/17/error": "none: the request ended normally",
|
|
"/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/18/round": "not applicable outside the concurrency stage",
|
|
"/requests/18/error": "none: the request ended normally",
|
|
"/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/19/round": "not applicable outside the concurrency stage",
|
|
"/requests/19/error": "none: the request ended normally",
|
|
"/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/20/round": "not applicable outside the concurrency stage",
|
|
"/requests/20/error": "none: the request ended normally",
|
|
"/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/21/round": "not applicable outside the concurrency stage",
|
|
"/requests/21/error": "none: the request ended normally",
|
|
"/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists"
|
|
},
|
|
"extensions": {
|
|
"suite": "standard-v1",
|
|
"runner": "strata_cli.bench_standard",
|
|
"status_reasons": [
|
|
"no model file could be hashed"
|
|
],
|
|
"started_engine": false,
|
|
"server_requests_before_run": 0,
|
|
"other_engine_options": {},
|
|
"request_checks": {
|
|
"warmup": {
|
|
"max_shared_prefix_tokens": 47,
|
|
"frozen_input_tokens": 75,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r1": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r1": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r1": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r2": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r2": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r2": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r3": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r3": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r3": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 82,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 90,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 85,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 71,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16382,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16390,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T1": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 325,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T2": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 335,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T3": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 352,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T4": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 339,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
}
|
|
},
|
|
"optional_stage_skipped": false,
|
|
"api_key_sent": true
|
|
}
|
|
}
|