Jason WittyandClaude Opus 5.5
d525f315f4
standard-v1 on the idle server: ~2,900 prompt tok/s at 16K, decode ~110-131. --adapt-async 1 neutral, --pipeline-windows 2 slower decode, --prefill auto:32768 22% slower prompt at 16K. Setup defaults kept and the original config restored. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1677 lines
56 KiB
JSON
1677 lines
56 KiB
JSON
{
|
|
"format": "strata-standard-benchmark",
|
|
"schema_version": 1,
|
|
"protocol": {
|
|
"id": "strata-qwen38-thinking-v1-draft.1",
|
|
"git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"profile": "qwen3.8-flash-next-thinking",
|
|
"corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407",
|
|
"model_card_revision": null
|
|
},
|
|
"run": {
|
|
"id": "20261027T000709Z-53bbc5d8",
|
|
"started_at": "2026-10-27T00:07:09Z",
|
|
"status": "nonconforming",
|
|
"required_duration_s": 102.071,
|
|
"optional_duration_s": null,
|
|
"optional_concurrency": false,
|
|
"command": "<user> bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes arm combined: [\"--adapt-async\",\"1\",\"--pipeline-windows\",\"2\",\"--prefill\",\"auto:32768\"] --quantization IQ3_S"
|
|
},
|
|
"hardware": {
|
|
"gpus": [
|
|
{
|
|
"device_index": 0,
|
|
"name": "NVIDIA GeForce RTX 3090",
|
|
"memory_bytes": 25769803776,
|
|
"memory_kind": "dedicated",
|
|
"power_limit_w": 250.0
|
|
}
|
|
],
|
|
"cpu": "Intel(R) Core(TM) i9-14900KS",
|
|
"ram_bytes": 118089846784,
|
|
"storage": null,
|
|
"background_workloads": null
|
|
},
|
|
"software": {
|
|
"os": {
|
|
"name": "NixOS 26.05 (Yarara)",
|
|
"version": "6.18.55"
|
|
},
|
|
"drivers": [
|
|
{
|
|
"name": "gpu-driver",
|
|
"version": "595.71.05"
|
|
}
|
|
],
|
|
"runtime": [
|
|
{
|
|
"name": "cuda",
|
|
"version": "13.2"
|
|
}
|
|
],
|
|
"strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"engine_version": "0.1.51",
|
|
"build_kind": "source",
|
|
"runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"build_options": null
|
|
},
|
|
"model": {
|
|
"id": "qwen3.8-flash-next-iq3_s",
|
|
"revision": null,
|
|
"source_revision": null,
|
|
"quantization": "IQ3_S",
|
|
"files": null,
|
|
"tokenizer_revision": null,
|
|
"custom_pack": null
|
|
},
|
|
"configuration": {
|
|
"track": "custom",
|
|
"track_notes": "arm combined: [\"--adapt-async\",\"1\",\"--pipeline-windows\",\"2\",\"--prefill\",\"auto:32768\"]",
|
|
"context_tokens": 131072,
|
|
"server_capacity": 1,
|
|
"kv_cache": null,
|
|
"expert_cache": null,
|
|
"offloading": null,
|
|
"prefill": null,
|
|
"mtp": null,
|
|
"calibration": null,
|
|
"experimental_options": null,
|
|
"template": {
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
"sampling": {
|
|
"temperature": 1.0,
|
|
"top_p": 0.95,
|
|
"top_k": 20,
|
|
"min_p": 0.0,
|
|
"presence_penalty": 0.0,
|
|
"repetition_penalty": 1.0,
|
|
"reasoning_effort": "xhigh"
|
|
},
|
|
"chat_template_kwargs": {
|
|
"enable_thinking": true,
|
|
"preserve_thinking": true
|
|
}
|
|
},
|
|
"request_counts": {
|
|
"warmup": {
|
|
"planned": 1,
|
|
"attempted": 1,
|
|
"completed": 0,
|
|
"truncated": 1,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"performance": {
|
|
"planned": 9,
|
|
"attempted": 9,
|
|
"completed": 4,
|
|
"truncated": 5,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"sanity": {
|
|
"planned": 12,
|
|
"attempted": 12,
|
|
"completed": 12,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"concurrency": {
|
|
"planned": 0,
|
|
"attempted": 0,
|
|
"completed": 0,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
}
|
|
},
|
|
"requests": [
|
|
{
|
|
"id": "warmup",
|
|
"phase": "warmup",
|
|
"case_id": "warmup",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 128,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 75,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 75,
|
|
"output_tokens": 128,
|
|
"decode_tokens": 128
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.5133,
|
|
"decode_s": 1.4776,
|
|
"ttft_s": 0.535657,
|
|
"first_answer_s": 1.405646,
|
|
"total_s": 1.994651
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 146.113384,
|
|
"decode_tps": 86.626963
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r1",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 380,
|
|
"decode_tokens": 380
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3764,
|
|
"decode_s": 3.5496,
|
|
"ttft_s": 0.392835,
|
|
"first_answer_s": 1.266814,
|
|
"total_s": 3.931351
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 204.569607,
|
|
"decode_tps": 107.054316
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.6557,
|
|
"decode_s": 5.3853,
|
|
"ttft_s": 2.674357,
|
|
"first_answer_s": 6.681325,
|
|
"total_s": 8.048543
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1541.213239,
|
|
"decode_tps": 95.073626
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.5738,
|
|
"decode_s": 5.0325,
|
|
"ttft_s": 7.603484,
|
|
"first_answer_s": null,
|
|
"total_s": 12.623808
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2163.643085,
|
|
"decode_tps": 101.738698
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r2",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 432,
|
|
"decode_tokens": 432
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4727,
|
|
"decode_s": 3.5681,
|
|
"ttft_s": 0.490477,
|
|
"first_answer_s": 1.42846,
|
|
"total_s": 4.046846
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 162.894013,
|
|
"decode_tps": 121.07284
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 425,
|
|
"decode_tokens": 425
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.4085,
|
|
"decode_s": 3.4927,
|
|
"ttft_s": 2.426906,
|
|
"first_answer_s": 4.197035,
|
|
"total_s": 5.907928
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1699.397966,
|
|
"decode_tps": 121.682366
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.3074,
|
|
"decode_s": 4.3822,
|
|
"ttft_s": 7.336762,
|
|
"first_answer_s": 10.695498,
|
|
"total_s": 11.707019
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2242.52128,
|
|
"decode_tps": 116.836292
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r3",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 322,
|
|
"decode_tokens": 322
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4656,
|
|
"decode_s": 2.6442,
|
|
"ttft_s": 0.479584,
|
|
"first_answer_s": 1.57057,
|
|
"total_s": 3.1124
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 165.378007,
|
|
"decode_tps": 121.775962
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3518,
|
|
"decode_s": 4.0682,
|
|
"ttft_s": 2.372037,
|
|
"first_answer_s": 4.115053,
|
|
"total_s": 6.429321
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1740.369079,
|
|
"decode_tps": 125.854186
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.2527,
|
|
"decode_s": 4.5163,
|
|
"ttft_s": 7.282458,
|
|
"first_answer_s": 10.813452,
|
|
"total_s": 11.786075
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2259.434418,
|
|
"decode_tps": 113.367137
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "sanity-I1",
|
|
"phase": "sanity",
|
|
"case_id": "I1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 82,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 82,
|
|
"output_tokens": 73,
|
|
"decode_tokens": 73
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4998,
|
|
"decode_s": 0.5871,
|
|
"ttft_s": 0.515335,
|
|
"first_answer_s": 0.987318,
|
|
"total_s": 1.091586
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 164.065626,
|
|
"decode_tps": 124.339976
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I2",
|
|
"phase": "sanity",
|
|
"case_id": "I2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 90,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 90,
|
|
"output_tokens": 60,
|
|
"decode_tokens": 60
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3694,
|
|
"decode_s": 0.4736,
|
|
"ttft_s": 0.383467,
|
|
"first_answer_s": 0.727489,
|
|
"total_s": 0.847506
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 243.638332,
|
|
"decode_tps": 126.689189
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I3",
|
|
"phase": "sanity",
|
|
"case_id": "I3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 85,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 85,
|
|
"output_tokens": 94,
|
|
"decode_tokens": 94
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3725,
|
|
"decode_s": 0.6368,
|
|
"ttft_s": 0.388917,
|
|
"first_answer_s": 0.926927,
|
|
"total_s": 1.014058
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 228.187919,
|
|
"decode_tps": 147.613065
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I4",
|
|
"phase": "sanity",
|
|
"case_id": "I4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 71,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 71,
|
|
"output_tokens": 42,
|
|
"decode_tokens": 42
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.387,
|
|
"decode_s": 0.3572,
|
|
"ttft_s": 0.40372,
|
|
"first_answer_s": 0.733739,
|
|
"total_s": 0.749765
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 183.462532,
|
|
"decode_tps": 117.581187
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R1",
|
|
"phase": "sanity",
|
|
"case_id": "R1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 65,
|
|
"decode_tokens": 65
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.4038,
|
|
"decode_s": 0.4852,
|
|
"ttft_s": 2.422928,
|
|
"first_answer_s": 2.855936,
|
|
"total_s": 2.896958
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1701.05666,
|
|
"decode_tps": 133.965375
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R2",
|
|
"phase": "sanity",
|
|
"case_id": "R2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 59,
|
|
"decode_tokens": 59
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.4084,
|
|
"decode_s": 0.4608,
|
|
"ttft_s": 2.427928,
|
|
"first_answer_s": 2.823916,
|
|
"total_s": 2.876941
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1697.807673,
|
|
"decode_tps": 128.038194
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R3",
|
|
"phase": "sanity",
|
|
"case_id": "R3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16382,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16382,
|
|
"output_tokens": 80,
|
|
"decode_tokens": 80
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.2229,
|
|
"decode_s": 0.6702,
|
|
"ttft_s": 7.246777,
|
|
"first_answer_s": 7.849771,
|
|
"total_s": 7.905787
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2268.064074,
|
|
"decode_tps": 119.367353
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R4",
|
|
"phase": "sanity",
|
|
"case_id": "R4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16390,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16390,
|
|
"output_tokens": 162,
|
|
"decode_tokens": 162
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.3936,
|
|
"decode_s": 1.2835,
|
|
"ttft_s": 7.417866,
|
|
"first_answer_s": 8.491859,
|
|
"total_s": 8.68888
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2216.782082,
|
|
"decode_tps": 126.217374
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T1",
|
|
"phase": "sanity",
|
|
"case_id": "T1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 325,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 325,
|
|
"output_tokens": 64,
|
|
"decode_tokens": 64
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.3004,
|
|
"decode_s": 0.4614,
|
|
"ttft_s": 1.318928,
|
|
"first_answer_s": 1.652993,
|
|
"total_s": 1.768956
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 249.923101,
|
|
"decode_tps": 138.708279
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T2",
|
|
"phase": "sanity",
|
|
"case_id": "T2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 335,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 335,
|
|
"output_tokens": 83,
|
|
"decode_tokens": 83
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9959,
|
|
"decode_s": 0.6065,
|
|
"ttft_s": 1.012924,
|
|
"first_answer_s": 1.447942,
|
|
"total_s": 1.60895
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 336.379155,
|
|
"decode_tps": 136.850783
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T3",
|
|
"phase": "sanity",
|
|
"case_id": "T3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 352,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 352,
|
|
"output_tokens": 90,
|
|
"decode_tokens": 90
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.947,
|
|
"decode_s": 0.5262,
|
|
"ttft_s": 0.963924,
|
|
"first_answer_s": 1.252936,
|
|
"total_s": 1.479945
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 371.700106,
|
|
"decode_tps": 171.037628
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T4",
|
|
"phase": "sanity",
|
|
"case_id": "T4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 339,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 339,
|
|
"output_tokens": 88,
|
|
"decode_tokens": 88
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9265,
|
|
"decode_s": 0.6079,
|
|
"ttft_s": 0.943922,
|
|
"first_answer_s": 1.372938,
|
|
"total_s": 1.540962
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 365.893146,
|
|
"decode_tps": 144.760651
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
}
|
|
],
|
|
"summaries": [
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 165.378007,
|
|
"min": 162.894013,
|
|
"max": 204.569607,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 121.07284,
|
|
"min": 107.054316,
|
|
"max": 121.775962,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 0.479584,
|
|
"min": 0.392835,
|
|
"max": 0.490477,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 1.42846,
|
|
"min": 1.266814,
|
|
"max": 1.57057,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 3.931351,
|
|
"min": 3.1124,
|
|
"max": 4.046846,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 1699.397966,
|
|
"min": 1541.213239,
|
|
"max": 1740.369079,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 121.682366,
|
|
"min": 95.073626,
|
|
"max": 125.854186,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 2.426906,
|
|
"min": 2.372037,
|
|
"max": 2.674357,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 4.197035,
|
|
"min": 4.115053,
|
|
"max": 6.681325,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 6.429321,
|
|
"min": 5.907928,
|
|
"max": 8.048543,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 2242.52128,
|
|
"min": 2163.643085,
|
|
"max": 2259.434418,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 113.367137,
|
|
"min": 101.738698,
|
|
"max": 116.836292,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 7.336762,
|
|
"min": 7.282458,
|
|
"max": 7.603484,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 10.754475,
|
|
"min": 10.695498,
|
|
"max": 10.813452,
|
|
"n": 2,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": [
|
|
{
|
|
"request_id": "performance-prompt-16k-r1",
|
|
"reason": "first_answer_s is not available for this request"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 11.786075,
|
|
"min": 11.707019,
|
|
"max": 12.623808,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
}
|
|
],
|
|
"concurrency_rounds": [],
|
|
"quality": {
|
|
"passed": 12,
|
|
"total": 12,
|
|
"categories": {
|
|
"instruction": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"retrieval": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"tool_call": {
|
|
"passed": 4,
|
|
"total": 4
|
|
}
|
|
}
|
|
},
|
|
"artifacts": [
|
|
{
|
|
"kind": "outputs",
|
|
"path": "outputs.jsonl",
|
|
"sha256": "8a35b6f6e17dc4d7adc9f7ce7a05cca12985ab1962fa156e12d1316cd72dcfae"
|
|
},
|
|
{
|
|
"kind": "config",
|
|
"path": "config.json",
|
|
"sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-manifest.sha256",
|
|
"sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-build.json",
|
|
"sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2"
|
|
}
|
|
],
|
|
"missing": {
|
|
"/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)",
|
|
"/run/optional_duration_s": "the optional concurrency stage was not selected",
|
|
"/hardware/storage": "the drive holding the model files could not be identified",
|
|
"/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background",
|
|
"/software/build_options": "the server does not report build options; pass --build-options for a source build",
|
|
"/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",
|
|
"/model/source_revision": "not given: pass --source-revision",
|
|
"/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)",
|
|
"/model/tokenizer_revision": "not given: pass --tokenizer-revision",
|
|
"/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)",
|
|
"/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/requests/0/round": "not applicable outside the concurrency stage",
|
|
"/requests/0/error": "none: the request ended normally",
|
|
"/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/0/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/1/round": "not applicable outside the concurrency stage",
|
|
"/requests/1/error": "none: the request ended normally",
|
|
"/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/1/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/2/round": "not applicable outside the concurrency stage",
|
|
"/requests/2/error": "none: the request ended normally",
|
|
"/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/2/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/3/round": "not applicable outside the concurrency stage",
|
|
"/requests/3/error": "none: the request ended normally",
|
|
"/requests/3/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/3/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/4/round": "not applicable outside the concurrency stage",
|
|
"/requests/4/error": "none: the request ended normally",
|
|
"/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/4/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/5/round": "not applicable outside the concurrency stage",
|
|
"/requests/5/error": "none: the request ended normally",
|
|
"/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/5/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/6/round": "not applicable outside the concurrency stage",
|
|
"/requests/6/error": "none: the request ended normally",
|
|
"/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/6/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/7/round": "not applicable outside the concurrency stage",
|
|
"/requests/7/error": "none: the request ended normally",
|
|
"/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/7/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/8/round": "not applicable outside the concurrency stage",
|
|
"/requests/8/error": "none: the request ended normally",
|
|
"/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/8/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/9/round": "not applicable outside the concurrency stage",
|
|
"/requests/9/error": "none: the request ended normally",
|
|
"/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/9/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/10/round": "not applicable outside the concurrency stage",
|
|
"/requests/10/error": "none: the request ended normally",
|
|
"/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/11/round": "not applicable outside the concurrency stage",
|
|
"/requests/11/error": "none: the request ended normally",
|
|
"/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/12/round": "not applicable outside the concurrency stage",
|
|
"/requests/12/error": "none: the request ended normally",
|
|
"/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/13/round": "not applicable outside the concurrency stage",
|
|
"/requests/13/error": "none: the request ended normally",
|
|
"/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/14/round": "not applicable outside the concurrency stage",
|
|
"/requests/14/error": "none: the request ended normally",
|
|
"/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/15/round": "not applicable outside the concurrency stage",
|
|
"/requests/15/error": "none: the request ended normally",
|
|
"/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/16/round": "not applicable outside the concurrency stage",
|
|
"/requests/16/error": "none: the request ended normally",
|
|
"/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/17/round": "not applicable outside the concurrency stage",
|
|
"/requests/17/error": "none: the request ended normally",
|
|
"/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/18/round": "not applicable outside the concurrency stage",
|
|
"/requests/18/error": "none: the request ended normally",
|
|
"/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/19/round": "not applicable outside the concurrency stage",
|
|
"/requests/19/error": "none: the request ended normally",
|
|
"/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/20/round": "not applicable outside the concurrency stage",
|
|
"/requests/20/error": "none: the request ended normally",
|
|
"/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/21/round": "not applicable outside the concurrency stage",
|
|
"/requests/21/error": "none: the request ended normally",
|
|
"/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists"
|
|
},
|
|
"extensions": {
|
|
"suite": "standard-v1",
|
|
"runner": "strata_cli.bench_standard",
|
|
"status_reasons": [
|
|
"no model file could be hashed"
|
|
],
|
|
"started_engine": false,
|
|
"server_requests_before_run": 0,
|
|
"other_engine_options": {},
|
|
"request_checks": {
|
|
"warmup": {
|
|
"max_shared_prefix_tokens": 47,
|
|
"frozen_input_tokens": 75,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r1": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r1": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r1": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r2": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r2": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r2": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r3": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r3": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r3": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 82,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 90,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 85,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 71,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16382,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16390,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T1": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 325,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T2": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 335,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T3": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 352,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T4": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 339,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
}
|
|
},
|
|
"optional_stage_skipped": false,
|
|
"api_key_sent": true
|
|
}
|
|
}
|