{ "format": "strata-standard-benchmark", "schema_version": 1, "protocol": { "id": "strata-qwen38-thinking-v1-draft.1", "git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "profile": "qwen3.8-flash-next-thinking", "corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407", "model_card_revision": null }, "run": { "id": "20261027T000709Z-53bbc5d8", "started_at": "2026-10-27T00:07:09Z", "status": "nonconforming", "required_duration_s": 102.071, "optional_duration_s": null, "optional_concurrency": false, "command": " bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes arm combined: [\"--adapt-async\",\"1\",\"--pipeline-windows\",\"2\",\"--prefill\",\"auto:32768\"] --quantization IQ3_S" }, "hardware": { "gpus": [ { "device_index": 0, "name": "NVIDIA GeForce RTX 3090", "memory_bytes": 25769803776, "memory_kind": "dedicated", "power_limit_w": 250.0 } ], "cpu": "Intel(R) Core(TM) i9-14900KS", "ram_bytes": 118089846784, "storage": null, "background_workloads": null }, "software": { "os": { "name": "NixOS 26.05 (Yarara)", "version": "6.18.55" }, "drivers": [ { "name": "gpu-driver", "version": "595.71.05" } ], "runtime": [ { "name": "cuda", "version": "13.2" } ], "strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "engine_version": "0.1.51", "build_kind": "source", "runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "build_options": null }, "model": { "id": "qwen3.8-flash-next-iq3_s", "revision": null, "source_revision": null, "quantization": "IQ3_S", "files": null, "tokenizer_revision": null, "custom_pack": null }, "configuration": { "track": "custom", "track_notes": "arm combined: [\"--adapt-async\",\"1\",\"--pipeline-windows\",\"2\",\"--prefill\",\"auto:32768\"]", "context_tokens": 131072, "server_capacity": 1, "kv_cache": null, "expert_cache": null, "offloading": null, "prefill": null, "mtp": null, "calibration": null, "experimental_options": null, "template": { "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, "sampling": { "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0, "reasoning_effort": "xhigh" }, "chat_template_kwargs": { "enable_thinking": true, "preserve_thinking": true } }, "request_counts": { "warmup": { "planned": 1, "attempted": 1, "completed": 0, "truncated": 1, "failed": 0, "skipped": 0 }, "performance": { "planned": 9, "attempted": 9, "completed": 4, "truncated": 5, "failed": 0, "skipped": 0 }, "sanity": { "planned": 12, "attempted": 12, "completed": 12, "truncated": 0, "failed": 0, "skipped": 0 }, "concurrency": { "planned": 0, "attempted": 0, "completed": 0, "truncated": 0, "failed": 0, "skipped": 0 } }, "requests": [ { "id": "warmup", "phase": "warmup", "case_id": "warmup", "repetition": 1, "seed": 1, "max_output_tokens": 128, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 75, "cached_tokens": 0, "read_tokens": 75, "output_tokens": 128, "decode_tokens": 128 }, "timings": { "prompt_s": 0.5133, "decode_s": 1.4776, "ttft_s": 0.535657, "first_answer_s": 1.405646, "total_s": 1.994651 }, "metrics": { "prompt_tps": 146.113384, "decode_tps": 86.626963 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r1", "phase": "performance", "case_id": "short", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 380, "decode_tokens": 380 }, "timings": { "prompt_s": 0.3764, "decode_s": 3.5496, "ttft_s": 0.392835, "first_answer_s": 1.266814, "total_s": 3.931351 }, "metrics": { "prompt_tps": 204.569607, "decode_tps": 107.054316 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r1", "phase": "performance", "case_id": "prompt-4k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.6557, "decode_s": 5.3853, "ttft_s": 2.674357, "first_answer_s": 6.681325, "total_s": 8.048543 }, "metrics": { "prompt_tps": 1541.213239, "decode_tps": 95.073626 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r1", "phase": "performance", "case_id": "prompt-16k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 7.5738, "decode_s": 5.0325, "ttft_s": 7.603484, "first_answer_s": null, "total_s": 12.623808 }, "metrics": { "prompt_tps": 2163.643085, "decode_tps": 101.738698 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r2", "phase": "performance", "case_id": "short", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 432, "decode_tokens": 432 }, "timings": { "prompt_s": 0.4727, "decode_s": 3.5681, "ttft_s": 0.490477, "first_answer_s": 1.42846, "total_s": 4.046846 }, "metrics": { "prompt_tps": 162.894013, "decode_tps": 121.07284 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r2", "phase": "performance", "case_id": "prompt-4k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 425, "decode_tokens": 425 }, "timings": { "prompt_s": 2.4085, "decode_s": 3.4927, "ttft_s": 2.426906, "first_answer_s": 4.197035, "total_s": 5.907928 }, "metrics": { "prompt_tps": 1699.397966, "decode_tps": 121.682366 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r2", "phase": "performance", "case_id": "prompt-16k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 7.3074, "decode_s": 4.3822, "ttft_s": 7.336762, "first_answer_s": 10.695498, "total_s": 11.707019 }, "metrics": { "prompt_tps": 2242.52128, "decode_tps": 116.836292 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r3", "phase": "performance", "case_id": "short", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 322, "decode_tokens": 322 }, "timings": { "prompt_s": 0.4656, "decode_s": 2.6442, "ttft_s": 0.479584, "first_answer_s": 1.57057, "total_s": 3.1124 }, "metrics": { "prompt_tps": 165.378007, "decode_tps": 121.775962 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r3", "phase": "performance", "case_id": "prompt-4k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.3518, "decode_s": 4.0682, "ttft_s": 2.372037, "first_answer_s": 4.115053, "total_s": 6.429321 }, "metrics": { "prompt_tps": 1740.369079, "decode_tps": 125.854186 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r3", "phase": "performance", "case_id": "prompt-16k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 7.2527, "decode_s": 4.5163, "ttft_s": 7.282458, "first_answer_s": 10.813452, "total_s": 11.786075 }, "metrics": { "prompt_tps": 2259.434418, "decode_tps": 113.367137 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "sanity-I1", "phase": "sanity", "case_id": "I1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 82, "cached_tokens": 0, "read_tokens": 82, "output_tokens": 73, "decode_tokens": 73 }, "timings": { "prompt_s": 0.4998, "decode_s": 0.5871, "ttft_s": 0.515335, "first_answer_s": 0.987318, "total_s": 1.091586 }, "metrics": { "prompt_tps": 164.065626, "decode_tps": 124.339976 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I2", "phase": "sanity", "case_id": "I2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 90, "cached_tokens": 0, "read_tokens": 90, "output_tokens": 60, "decode_tokens": 60 }, "timings": { "prompt_s": 0.3694, "decode_s": 0.4736, "ttft_s": 0.383467, "first_answer_s": 0.727489, "total_s": 0.847506 }, "metrics": { "prompt_tps": 243.638332, "decode_tps": 126.689189 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I3", "phase": "sanity", "case_id": "I3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 85, "cached_tokens": 0, "read_tokens": 85, "output_tokens": 94, "decode_tokens": 94 }, "timings": { "prompt_s": 0.3725, "decode_s": 0.6368, "ttft_s": 0.388917, "first_answer_s": 0.926927, "total_s": 1.014058 }, "metrics": { "prompt_tps": 228.187919, "decode_tps": 147.613065 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I4", "phase": "sanity", "case_id": "I4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 71, "cached_tokens": 0, "read_tokens": 71, "output_tokens": 42, "decode_tokens": 42 }, "timings": { "prompt_s": 0.387, "decode_s": 0.3572, "ttft_s": 0.40372, "first_answer_s": 0.733739, "total_s": 0.749765 }, "metrics": { "prompt_tps": 183.462532, "decode_tps": 117.581187 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R1", "phase": "sanity", "case_id": "R1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 65, "decode_tokens": 65 }, "timings": { "prompt_s": 2.4038, "decode_s": 0.4852, "ttft_s": 2.422928, "first_answer_s": 2.855936, "total_s": 2.896958 }, "metrics": { "prompt_tps": 1701.05666, "decode_tps": 133.965375 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R2", "phase": "sanity", "case_id": "R2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 59, "decode_tokens": 59 }, "timings": { "prompt_s": 2.4084, "decode_s": 0.4608, "ttft_s": 2.427928, "first_answer_s": 2.823916, "total_s": 2.876941 }, "metrics": { "prompt_tps": 1697.807673, "decode_tps": 128.038194 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R3", "phase": "sanity", "case_id": "R3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16382, "cached_tokens": 0, "read_tokens": 16382, "output_tokens": 80, "decode_tokens": 80 }, "timings": { "prompt_s": 7.2229, "decode_s": 0.6702, "ttft_s": 7.246777, "first_answer_s": 7.849771, "total_s": 7.905787 }, "metrics": { "prompt_tps": 2268.064074, "decode_tps": 119.367353 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R4", "phase": "sanity", "case_id": "R4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16390, "cached_tokens": 0, "read_tokens": 16390, "output_tokens": 162, "decode_tokens": 162 }, "timings": { "prompt_s": 7.3936, "decode_s": 1.2835, "ttft_s": 7.417866, "first_answer_s": 8.491859, "total_s": 8.68888 }, "metrics": { "prompt_tps": 2216.782082, "decode_tps": 126.217374 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-T1", "phase": "sanity", "case_id": "T1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 325, "cached_tokens": 0, "read_tokens": 325, "output_tokens": 64, "decode_tokens": 64 }, "timings": { "prompt_s": 1.3004, "decode_s": 0.4614, "ttft_s": 1.318928, "first_answer_s": 1.652993, "total_s": 1.768956 }, "metrics": { "prompt_tps": 249.923101, "decode_tps": 138.708279 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T2", "phase": "sanity", "case_id": "T2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 335, "cached_tokens": 0, "read_tokens": 335, "output_tokens": 83, "decode_tokens": 83 }, "timings": { "prompt_s": 0.9959, "decode_s": 0.6065, "ttft_s": 1.012924, "first_answer_s": 1.447942, "total_s": 1.60895 }, "metrics": { "prompt_tps": 336.379155, "decode_tps": 136.850783 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T3", "phase": "sanity", "case_id": "T3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 352, "cached_tokens": 0, "read_tokens": 352, "output_tokens": 90, "decode_tokens": 90 }, "timings": { "prompt_s": 0.947, "decode_s": 0.5262, "ttft_s": 0.963924, "first_answer_s": 1.252936, "total_s": 1.479945 }, "metrics": { "prompt_tps": 371.700106, "decode_tps": 171.037628 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T4", "phase": "sanity", "case_id": "T4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 339, "cached_tokens": 0, "read_tokens": 339, "output_tokens": 88, "decode_tokens": 88 }, "timings": { "prompt_s": 0.9265, "decode_s": 0.6079, "ttft_s": 0.943922, "first_answer_s": 1.372938, "total_s": 1.540962 }, "metrics": { "prompt_tps": 365.893146, "decode_tps": 144.760651 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } } ], "summaries": [ { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 165.378007, "min": 162.894013, "max": 204.569607, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 121.07284, "min": 107.054316, "max": 121.775962, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 0.479584, "min": 0.392835, "max": 0.490477, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 1.42846, "min": 1.266814, "max": 1.57057, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 3.931351, "min": 3.1124, "max": 4.046846, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 1699.397966, "min": 1541.213239, "max": 1740.369079, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 121.682366, "min": 95.073626, "max": 125.854186, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 2.426906, "min": 2.372037, "max": 2.674357, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 4.197035, "min": 4.115053, "max": 6.681325, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 6.429321, "min": 5.907928, "max": 8.048543, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 2242.52128, "min": 2163.643085, "max": 2259.434418, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 113.367137, "min": 101.738698, "max": 116.836292, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 7.336762, "min": 7.282458, "max": 7.603484, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 10.754475, "min": 10.695498, "max": 10.813452, "n": 2, "included_request_ids": [ "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [ { "request_id": "performance-prompt-16k-r1", "reason": "first_answer_s is not available for this request" } ] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 11.786075, "min": 11.707019, "max": 12.623808, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] } ], "concurrency_rounds": [], "quality": { "passed": 12, "total": 12, "categories": { "instruction": { "passed": 4, "total": 4 }, "retrieval": { "passed": 4, "total": 4 }, "tool_call": { "passed": 4, "total": 4 } } }, "artifacts": [ { "kind": "outputs", "path": "outputs.jsonl", "sha256": "8a35b6f6e17dc4d7adc9f7ce7a05cca12985ab1962fa156e12d1316cd72dcfae" }, { "kind": "config", "path": "config.json", "sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410" }, { "kind": "provenance", "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, { "kind": "provenance", "path": "provenance/corpus-manifest.sha256", "sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407" }, { "kind": "provenance", "path": "provenance/corpus-build.json", "sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2" } ], "missing": { "/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)", "/run/optional_duration_s": "the optional concurrency stage was not selected", "/hardware/storage": "the drive holding the model files could not be identified", "/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background", "/software/build_options": "the server does not report build options; pass --build-options for a source build", "/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision", "/model/source_revision": "not given: pass --source-revision", "/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)", "/model/tokenizer_revision": "not given: pass --tokenizer-revision", "/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)", "/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config", "/requests/0/round": "not applicable outside the concurrency stage", "/requests/0/error": "none: the request ended normally", "/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/0/evaluation": "not applicable outside the sanity stage", "/requests/1/round": "not applicable outside the concurrency stage", "/requests/1/error": "none: the request ended normally", "/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/1/evaluation": "not applicable outside the sanity stage", "/requests/2/round": "not applicable outside the concurrency stage", "/requests/2/error": "none: the request ended normally", "/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/2/evaluation": "not applicable outside the sanity stage", "/requests/3/round": "not applicable outside the concurrency stage", "/requests/3/error": "none: the request ended normally", "/requests/3/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)", "/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/3/evaluation": "not applicable outside the sanity stage", "/requests/4/round": "not applicable outside the concurrency stage", "/requests/4/error": "none: the request ended normally", "/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/4/evaluation": "not applicable outside the sanity stage", "/requests/5/round": "not applicable outside the concurrency stage", "/requests/5/error": "none: the request ended normally", "/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/5/evaluation": "not applicable outside the sanity stage", "/requests/6/round": "not applicable outside the concurrency stage", "/requests/6/error": "none: the request ended normally", "/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/6/evaluation": "not applicable outside the sanity stage", "/requests/7/round": "not applicable outside the concurrency stage", "/requests/7/error": "none: the request ended normally", "/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/7/evaluation": "not applicable outside the sanity stage", "/requests/8/round": "not applicable outside the concurrency stage", "/requests/8/error": "none: the request ended normally", "/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/8/evaluation": "not applicable outside the sanity stage", "/requests/9/round": "not applicable outside the concurrency stage", "/requests/9/error": "none: the request ended normally", "/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/9/evaluation": "not applicable outside the sanity stage", "/requests/10/round": "not applicable outside the concurrency stage", "/requests/10/error": "none: the request ended normally", "/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/11/round": "not applicable outside the concurrency stage", "/requests/11/error": "none: the request ended normally", "/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/12/round": "not applicable outside the concurrency stage", "/requests/12/error": "none: the request ended normally", "/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/13/round": "not applicable outside the concurrency stage", "/requests/13/error": "none: the request ended normally", "/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/14/round": "not applicable outside the concurrency stage", "/requests/14/error": "none: the request ended normally", "/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/15/round": "not applicable outside the concurrency stage", "/requests/15/error": "none: the request ended normally", "/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/16/round": "not applicable outside the concurrency stage", "/requests/16/error": "none: the request ended normally", "/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/17/round": "not applicable outside the concurrency stage", "/requests/17/error": "none: the request ended normally", "/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/18/round": "not applicable outside the concurrency stage", "/requests/18/error": "none: the request ended normally", "/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/19/round": "not applicable outside the concurrency stage", "/requests/19/error": "none: the request ended normally", "/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/20/round": "not applicable outside the concurrency stage", "/requests/20/error": "none: the request ended normally", "/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/21/round": "not applicable outside the concurrency stage", "/requests/21/error": "none: the request ended normally", "/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists" }, "extensions": { "suite": "standard-v1", "runner": "strata_cli.bench_standard", "status_reasons": [ "no model file could be hashed" ], "started_engine": false, "server_requests_before_run": 0, "other_engine_options": {}, "request_checks": { "warmup": { "max_shared_prefix_tokens": 47, "frozen_input_tokens": 75, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r1": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r1": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r1": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r2": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r2": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r2": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r3": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r3": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r3": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 82, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 90, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 85, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 71, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16382, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16390, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T1": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 325, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T2": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 335, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T3": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 352, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T4": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 339, "unexpected_payload_reuse": false, "length_outside_tolerance": false } }, "optional_stage_skipped": false, "api_key_sent": true } }