Jason WittyandClaude Opus 5.5
698005c602
Standard benchmark, two fresh-restart runs: prompt -51% at 4K, -37% at 16K, decode -40% vs IQ3_S. 12/12 sanity checks. Kept active for husky-bench. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1677 lines
56 KiB
JSON
1677 lines
56 KiB
JSON
{
|
|
"format": "strata-standard-benchmark",
|
|
"schema_version": 1,
|
|
"protocol": {
|
|
"id": "strata-qwen38-thinking-v1-draft.1",
|
|
"git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"profile": "qwen3.8-flash-next-thinking",
|
|
"corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407",
|
|
"model_card_revision": null
|
|
},
|
|
"run": {
|
|
"id": "20261027T192209Z-58540dcc",
|
|
"started_at": "2026-10-27T19:22:09Z",
|
|
"status": "nonconforming",
|
|
"required_duration_s": 159.025,
|
|
"optional_duration_s": null,
|
|
"optional_concurrency": false,
|
|
"command": "<user> bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes UD-Q4_K_XL rep 1, 275 W, case closed --quantization UD-Q4_K_XL"
|
|
},
|
|
"hardware": {
|
|
"gpus": [
|
|
{
|
|
"device_index": 0,
|
|
"name": "NVIDIA GeForce RTX 3090",
|
|
"memory_bytes": 25769803776,
|
|
"memory_kind": "dedicated",
|
|
"power_limit_w": 275.0
|
|
}
|
|
],
|
|
"cpu": "Intel(R) Core(TM) i9-14900KS",
|
|
"ram_bytes": 118089846784,
|
|
"storage": null,
|
|
"background_workloads": null
|
|
},
|
|
"software": {
|
|
"os": {
|
|
"name": "NixOS 26.05 (Yarara)",
|
|
"version": "6.18.55"
|
|
},
|
|
"drivers": [
|
|
{
|
|
"name": "gpu-driver",
|
|
"version": "595.71.05"
|
|
}
|
|
],
|
|
"runtime": [
|
|
{
|
|
"name": "cuda",
|
|
"version": "13.2"
|
|
}
|
|
],
|
|
"strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"engine_version": "0.1.51",
|
|
"build_kind": "source",
|
|
"runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"build_options": null
|
|
},
|
|
"model": {
|
|
"id": "qwen3.8-flash-next-unsloth-ud-q4_k_xl",
|
|
"revision": null,
|
|
"source_revision": null,
|
|
"quantization": "UD-Q4_K_XL",
|
|
"files": null,
|
|
"tokenizer_revision": null,
|
|
"custom_pack": null
|
|
},
|
|
"configuration": {
|
|
"track": "custom",
|
|
"track_notes": "UD-Q4_K_XL rep 1, 275 W, case closed",
|
|
"context_tokens": 131072,
|
|
"server_capacity": 1,
|
|
"kv_cache": null,
|
|
"expert_cache": null,
|
|
"offloading": null,
|
|
"prefill": null,
|
|
"mtp": null,
|
|
"calibration": null,
|
|
"experimental_options": null,
|
|
"template": {
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
"sampling": {
|
|
"temperature": 1.0,
|
|
"top_p": 0.95,
|
|
"top_k": 20,
|
|
"min_p": 0.0,
|
|
"presence_penalty": 0.0,
|
|
"repetition_penalty": 1.0,
|
|
"reasoning_effort": "xhigh"
|
|
},
|
|
"chat_template_kwargs": {
|
|
"enable_thinking": true,
|
|
"preserve_thinking": true
|
|
}
|
|
},
|
|
"request_counts": {
|
|
"warmup": {
|
|
"planned": 1,
|
|
"attempted": 1,
|
|
"completed": 0,
|
|
"truncated": 1,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"performance": {
|
|
"planned": 9,
|
|
"attempted": 9,
|
|
"completed": 3,
|
|
"truncated": 6,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"sanity": {
|
|
"planned": 12,
|
|
"attempted": 12,
|
|
"completed": 12,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"concurrency": {
|
|
"planned": 0,
|
|
"attempted": 0,
|
|
"completed": 0,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
}
|
|
},
|
|
"requests": [
|
|
{
|
|
"id": "warmup",
|
|
"phase": "warmup",
|
|
"case_id": "warmup",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 128,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 75,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 75,
|
|
"output_tokens": 128,
|
|
"decode_tokens": 128
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.0416,
|
|
"decode_s": 2.3312,
|
|
"ttft_s": 1.079833,
|
|
"first_answer_s": 2.33383,
|
|
"total_s": 3.375916
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 72.004608,
|
|
"decode_tps": 54.907344
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r1",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 329,
|
|
"decode_tokens": 329
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9203,
|
|
"decode_s": 4.9423,
|
|
"ttft_s": 0.941702,
|
|
"first_answer_s": 2.4247,
|
|
"total_s": 5.868223
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 83.668369,
|
|
"decode_tps": 66.568197
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.0424,
|
|
"decode_s": 7.6155,
|
|
"ttft_s": 5.070346,
|
|
"first_answer_s": 9.82132,
|
|
"total_s": 12.667393
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 811.716643,
|
|
"decode_tps": 67.231305
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 8.5451,
|
|
"decode_s": 7.5699,
|
|
"ttft_s": 8.573693,
|
|
"first_answer_s": 15.644759,
|
|
"total_s": 16.126982
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1917.707224,
|
|
"decode_tps": 67.636296
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r2",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 333,
|
|
"decode_tokens": 333
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.1392,
|
|
"decode_s": 4.9765,
|
|
"ttft_s": 1.159629,
|
|
"first_answer_s": 2.92636,
|
|
"total_s": 6.119234
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 67.591292,
|
|
"decode_tps": 66.914498
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 4.839,
|
|
"decode_s": 6.9679,
|
|
"ttft_s": 4.862148,
|
|
"first_answer_s": 8.270155,
|
|
"total_s": 11.813198
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 845.835917,
|
|
"decode_tps": 73.479815
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 8.5377,
|
|
"decode_s": 7.5529,
|
|
"ttft_s": 8.572743,
|
|
"first_answer_s": 15.105133,
|
|
"total_s": 16.106588
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1919.369385,
|
|
"decode_tps": 67.788532
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r3",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 284,
|
|
"decode_tokens": 284
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.1175,
|
|
"decode_s": 4.0529,
|
|
"ttft_s": 1.141948,
|
|
"first_answer_s": 2.331866,
|
|
"total_s": 5.174936
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 68.903803,
|
|
"decode_tps": 70.073281
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 4.7839,
|
|
"decode_s": 7.3467,
|
|
"ttft_s": 4.807855,
|
|
"first_answer_s": null,
|
|
"total_s": 12.137832
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 855.578085,
|
|
"decode_tps": 69.691154
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 8.5605,
|
|
"decode_s": 6.6441,
|
|
"ttft_s": 8.60382,
|
|
"first_answer_s": 12.546901,
|
|
"total_s": 15.224179
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1914.257345,
|
|
"decode_tps": 77.060851
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "sanity-I1",
|
|
"phase": "sanity",
|
|
"case_id": "I1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 82,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 82,
|
|
"output_tokens": 73,
|
|
"decode_tokens": 73
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.1293,
|
|
"decode_s": 1.3344,
|
|
"ttft_s": 1.150423,
|
|
"first_answer_s": 2.040546,
|
|
"total_s": 2.468717
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 72.611352,
|
|
"decode_tps": 54.706235
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I2",
|
|
"phase": "sanity",
|
|
"case_id": "I2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 90,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 90,
|
|
"output_tokens": 69,
|
|
"decode_tokens": 69
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.0433,
|
|
"decode_s": 1.1381,
|
|
"ttft_s": 1.0605,
|
|
"first_answer_s": 1.971506,
|
|
"total_s": 2.186554
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 86.264737,
|
|
"decode_tps": 60.627361
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I3",
|
|
"phase": "sanity",
|
|
"case_id": "I3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 85,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 85,
|
|
"output_tokens": 85,
|
|
"decode_tokens": 85
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.0732,
|
|
"decode_s": 1.1339,
|
|
"ttft_s": 1.091779,
|
|
"first_answer_s": 1.89378,
|
|
"total_s": 2.212826
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 79.202385,
|
|
"decode_tps": 74.962519
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I4",
|
|
"phase": "sanity",
|
|
"case_id": "I4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 71,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 71,
|
|
"output_tokens": 43,
|
|
"decode_tokens": 43
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9394,
|
|
"decode_s": 0.4397,
|
|
"ttft_s": 0.95794,
|
|
"first_answer_s": 1.384948,
|
|
"total_s": 1.384983
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 75.580158,
|
|
"decode_tps": 97.79395
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R1",
|
|
"phase": "sanity",
|
|
"case_id": "R1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 59,
|
|
"decode_tokens": 59
|
|
},
|
|
"timings": {
|
|
"prompt_s": 4.8037,
|
|
"decode_s": 0.7969,
|
|
"ttft_s": 4.823941,
|
|
"first_answer_s": 5.477063,
|
|
"total_s": 5.607205
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 851.218852,
|
|
"decode_tps": 74.036893
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R2",
|
|
"phase": "sanity",
|
|
"case_id": "R2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 59,
|
|
"decode_tokens": 59
|
|
},
|
|
"timings": {
|
|
"prompt_s": 4.7632,
|
|
"decode_s": 0.5983,
|
|
"ttft_s": 4.786522,
|
|
"first_answer_s": 5.250526,
|
|
"total_s": 5.37257
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 858.4565,
|
|
"decode_tps": 98.612736
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R3",
|
|
"phase": "sanity",
|
|
"case_id": "R3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16382,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16382,
|
|
"output_tokens": 65,
|
|
"decode_tokens": 65
|
|
},
|
|
"timings": {
|
|
"prompt_s": 8.458,
|
|
"decode_s": 0.7629,
|
|
"ttft_s": 8.489877,
|
|
"first_answer_s": 9.135875,
|
|
"total_s": 9.238931
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1936.864507,
|
|
"decode_tps": 85.201206
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R4",
|
|
"phase": "sanity",
|
|
"case_id": "R4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16390,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16390,
|
|
"output_tokens": 202,
|
|
"decode_tokens": 202
|
|
},
|
|
"timings": {
|
|
"prompt_s": 8.6627,
|
|
"decode_s": 2.7004,
|
|
"ttft_s": 8.693934,
|
|
"first_answer_s": 10.868011,
|
|
"total_s": 11.382227
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1892.019809,
|
|
"decode_tps": 74.803733
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T1",
|
|
"phase": "sanity",
|
|
"case_id": "T1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 325,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 325,
|
|
"output_tokens": 58,
|
|
"decode_tokens": 58
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.8827,
|
|
"decode_s": 0.8213,
|
|
"ttft_s": 2.902472,
|
|
"first_answer_s": 3.4055,
|
|
"total_s": 3.710527
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 112.741527,
|
|
"decode_tps": 70.619749
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T2",
|
|
"phase": "sanity",
|
|
"case_id": "T2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 335,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 335,
|
|
"output_tokens": 85,
|
|
"decode_tokens": 85
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.5571,
|
|
"decode_s": 1.3755,
|
|
"ttft_s": 2.575024,
|
|
"first_answer_s": 3.336137,
|
|
"total_s": 3.938317
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 131.007782,
|
|
"decode_tps": 61.795711
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T3",
|
|
"phase": "sanity",
|
|
"case_id": "T3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 352,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 352,
|
|
"output_tokens": 90,
|
|
"decode_tokens": 90
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.5202,
|
|
"decode_s": 1.1394,
|
|
"ttft_s": 2.539428,
|
|
"first_answer_s": 3.010445,
|
|
"total_s": 3.665482
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 139.671455,
|
|
"decode_tps": 78.988942
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T4",
|
|
"phase": "sanity",
|
|
"case_id": "T4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 339,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 339,
|
|
"output_tokens": 61,
|
|
"decode_tokens": 61
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.4675,
|
|
"decode_s": 0.7593,
|
|
"ttft_s": 2.485926,
|
|
"first_answer_s": 2.811964,
|
|
"total_s": 3.232355
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 137.386018,
|
|
"decode_tps": 80.337153
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
}
|
|
],
|
|
"summaries": [
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 68.903803,
|
|
"min": 67.591292,
|
|
"max": 83.668369,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 66.914498,
|
|
"min": 66.568197,
|
|
"max": 70.073281,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 1.141948,
|
|
"min": 0.941702,
|
|
"max": 1.159629,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 2.4247,
|
|
"min": 2.331866,
|
|
"max": 2.92636,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 5.868223,
|
|
"min": 5.174936,
|
|
"max": 6.119234,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 845.835917,
|
|
"min": 811.716643,
|
|
"max": 855.578085,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 69.691154,
|
|
"min": 67.231305,
|
|
"max": 73.479815,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 4.862148,
|
|
"min": 4.807855,
|
|
"max": 5.070346,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 9.045738,
|
|
"min": 8.270155,
|
|
"max": 9.82132,
|
|
"n": 2,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2"
|
|
],
|
|
"excluded": [
|
|
{
|
|
"request_id": "performance-prompt-4k-r3",
|
|
"reason": "first_answer_s is not available for this request"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 12.137832,
|
|
"min": 11.813198,
|
|
"max": 12.667393,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 1917.707224,
|
|
"min": 1914.257345,
|
|
"max": 1919.369385,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 67.788532,
|
|
"min": 67.636296,
|
|
"max": 77.060851,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 8.573693,
|
|
"min": 8.572743,
|
|
"max": 8.60382,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 15.105133,
|
|
"min": 12.546901,
|
|
"max": 15.644759,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 16.106588,
|
|
"min": 15.224179,
|
|
"max": 16.126982,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
}
|
|
],
|
|
"concurrency_rounds": [],
|
|
"quality": {
|
|
"passed": 12,
|
|
"total": 12,
|
|
"categories": {
|
|
"instruction": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"retrieval": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"tool_call": {
|
|
"passed": 4,
|
|
"total": 4
|
|
}
|
|
}
|
|
},
|
|
"artifacts": [
|
|
{
|
|
"kind": "outputs",
|
|
"path": "outputs.jsonl",
|
|
"sha256": "999f3ba5c895aab8927c18c33fb2a28398163768671560f6d07131eb75d13aaf"
|
|
},
|
|
{
|
|
"kind": "config",
|
|
"path": "config.json",
|
|
"sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-manifest.sha256",
|
|
"sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-build.json",
|
|
"sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2"
|
|
}
|
|
],
|
|
"missing": {
|
|
"/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)",
|
|
"/run/optional_duration_s": "the optional concurrency stage was not selected",
|
|
"/hardware/storage": "the drive holding the model files could not be identified",
|
|
"/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background",
|
|
"/software/build_options": "the server does not report build options; pass --build-options for a source build",
|
|
"/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",
|
|
"/model/source_revision": "not given: pass --source-revision",
|
|
"/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)",
|
|
"/model/tokenizer_revision": "not given: pass --tokenizer-revision",
|
|
"/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)",
|
|
"/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/requests/0/round": "not applicable outside the concurrency stage",
|
|
"/requests/0/error": "none: the request ended normally",
|
|
"/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/0/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/1/round": "not applicable outside the concurrency stage",
|
|
"/requests/1/error": "none: the request ended normally",
|
|
"/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/1/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/2/round": "not applicable outside the concurrency stage",
|
|
"/requests/2/error": "none: the request ended normally",
|
|
"/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/2/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/3/round": "not applicable outside the concurrency stage",
|
|
"/requests/3/error": "none: the request ended normally",
|
|
"/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/3/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/4/round": "not applicable outside the concurrency stage",
|
|
"/requests/4/error": "none: the request ended normally",
|
|
"/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/4/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/5/round": "not applicable outside the concurrency stage",
|
|
"/requests/5/error": "none: the request ended normally",
|
|
"/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/5/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/6/round": "not applicable outside the concurrency stage",
|
|
"/requests/6/error": "none: the request ended normally",
|
|
"/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/6/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/7/round": "not applicable outside the concurrency stage",
|
|
"/requests/7/error": "none: the request ended normally",
|
|
"/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/7/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/8/round": "not applicable outside the concurrency stage",
|
|
"/requests/8/error": "none: the request ended normally",
|
|
"/requests/8/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/8/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/9/round": "not applicable outside the concurrency stage",
|
|
"/requests/9/error": "none: the request ended normally",
|
|
"/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/9/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/10/round": "not applicable outside the concurrency stage",
|
|
"/requests/10/error": "none: the request ended normally",
|
|
"/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/11/round": "not applicable outside the concurrency stage",
|
|
"/requests/11/error": "none: the request ended normally",
|
|
"/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/12/round": "not applicable outside the concurrency stage",
|
|
"/requests/12/error": "none: the request ended normally",
|
|
"/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/13/round": "not applicable outside the concurrency stage",
|
|
"/requests/13/error": "none: the request ended normally",
|
|
"/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/14/round": "not applicable outside the concurrency stage",
|
|
"/requests/14/error": "none: the request ended normally",
|
|
"/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/15/round": "not applicable outside the concurrency stage",
|
|
"/requests/15/error": "none: the request ended normally",
|
|
"/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/16/round": "not applicable outside the concurrency stage",
|
|
"/requests/16/error": "none: the request ended normally",
|
|
"/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/17/round": "not applicable outside the concurrency stage",
|
|
"/requests/17/error": "none: the request ended normally",
|
|
"/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/18/round": "not applicable outside the concurrency stage",
|
|
"/requests/18/error": "none: the request ended normally",
|
|
"/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/19/round": "not applicable outside the concurrency stage",
|
|
"/requests/19/error": "none: the request ended normally",
|
|
"/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/20/round": "not applicable outside the concurrency stage",
|
|
"/requests/20/error": "none: the request ended normally",
|
|
"/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/21/round": "not applicable outside the concurrency stage",
|
|
"/requests/21/error": "none: the request ended normally",
|
|
"/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists"
|
|
},
|
|
"extensions": {
|
|
"suite": "standard-v1",
|
|
"runner": "strata_cli.bench_standard",
|
|
"status_reasons": [
|
|
"no model file could be hashed"
|
|
],
|
|
"started_engine": false,
|
|
"server_requests_before_run": 0,
|
|
"other_engine_options": {},
|
|
"request_checks": {
|
|
"warmup": {
|
|
"max_shared_prefix_tokens": 47,
|
|
"frozen_input_tokens": 75,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r1": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r1": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r1": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r2": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r2": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r2": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r3": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r3": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r3": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 82,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 90,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 85,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 71,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16382,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16390,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T1": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 325,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T2": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 335,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T3": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 352,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T4": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 339,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
}
|
|
},
|
|
"optional_stage_skipped": false,
|
|
"api_key_sent": true
|
|
}
|
|
}
|