{ "format": "strata-standard-benchmark", "schema_version": 1, "protocol": { "id": "strata-qwen38-thinking-v1-draft.1", "git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "profile": "qwen3.8-flash-next-thinking", "corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407", "model_card_revision": null }, "run": { "id": "20261027T034528Z-6a05efdc", "started_at": "2026-10-27T03:45:28Z", "status": "nonconforming", "required_duration_s": 96.064, "optional_duration_s": null, "optional_concurrency": false, "command": " bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes power cap 250W rep 1, case closed --quantization IQ3_S" }, "hardware": { "gpus": [ { "device_index": 0, "name": "NVIDIA GeForce RTX 3090", "memory_bytes": 25769803776, "memory_kind": "dedicated", "power_limit_w": 250.0 } ], "cpu": "Intel(R) Core(TM) i9-14900KS", "ram_bytes": 118089846784, "storage": null, "background_workloads": null }, "software": { "os": { "name": "NixOS 26.05 (Yarara)", "version": "6.18.55" }, "drivers": [ { "name": "gpu-driver", "version": "595.71.05" } ], "runtime": [ { "name": "cuda", "version": "13.2" } ], "strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "engine_version": "0.1.51", "build_kind": "source", "runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "build_options": null }, "model": { "id": "qwen3.8-flash-next-iq3_s", "revision": null, "source_revision": null, "quantization": "IQ3_S", "files": null, "tokenizer_revision": null, "custom_pack": null }, "configuration": { "track": "custom", "track_notes": "power cap 250W rep 1, case closed", "context_tokens": 131072, "server_capacity": 1, "kv_cache": null, "expert_cache": null, "offloading": null, "prefill": null, "mtp": null, "calibration": null, "experimental_options": null, "template": { "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, "sampling": { "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0, "reasoning_effort": "xhigh" }, "chat_template_kwargs": { "enable_thinking": true, "preserve_thinking": true } }, "request_counts": { "warmup": { "planned": 1, "attempted": 1, "completed": 0, "truncated": 1, "failed": 0, "skipped": 0 }, "performance": { "planned": 9, "attempted": 9, "completed": 3, "truncated": 6, "failed": 0, "skipped": 0 }, "sanity": { "planned": 12, "attempted": 12, "completed": 12, "truncated": 0, "failed": 0, "skipped": 0 }, "concurrency": { "planned": 0, "attempted": 0, "completed": 0, "truncated": 0, "failed": 0, "skipped": 0 } }, "requests": [ { "id": "warmup", "phase": "warmup", "case_id": "warmup", "repetition": 1, "seed": 1, "max_output_tokens": 128, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 75, "cached_tokens": 0, "read_tokens": 75, "output_tokens": 128, "decode_tokens": 128 }, "timings": { "prompt_s": 0.3975, "decode_s": 1.2086, "ttft_s": 0.424657, "first_answer_s": 1.060999, "total_s": 1.609254 }, "metrics": { "prompt_tps": 188.679245, "decode_tps": 105.907662 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r1", "phase": "performance", "case_id": "short", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 474, "decode_tokens": 474 }, "timings": { "prompt_s": 0.3682, "decode_s": 3.8277, "ttft_s": 0.384271, "first_answer_s": 0.882254, "total_s": 4.20006 }, "metrics": { "prompt_tps": 209.125475, "decode_tps": 123.834156 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r1", "phase": "performance", "case_id": "prompt-4k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.6257, "decode_s": 4.6419, "ttft_s": 2.645078, "first_answer_s": 6.090085, "total_s": 7.275433 }, "metrics": { "prompt_tps": 1558.822409, "decode_tps": 110.299662 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r1", "phase": "performance", "case_id": "prompt-16k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 5.7201, "decode_s": 5.1285, "ttft_s": 5.74338, "first_answer_s": null, "total_s": 10.859748 }, "metrics": { "prompt_tps": 2864.810056, "decode_tps": 99.83426 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r2", "phase": "performance", "case_id": "short", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 428, "decode_tokens": 428 }, "timings": { "prompt_s": 0.4643, "decode_s": 3.4818, "ttft_s": 0.485776, "first_answer_s": 1.016923, "total_s": 3.956819 }, "metrics": { "prompt_tps": 165.841051, "decode_tps": 122.924924 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r2", "phase": "performance", "case_id": "prompt-4k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.3814, "decode_s": 4.0911, "ttft_s": 2.399576, "first_answer_s": 5.137586, "total_s": 6.478621 }, "metrics": { "prompt_tps": 1718.736877, "decode_tps": 125.149715 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r2", "phase": "performance", "case_id": "prompt-16k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 5.6569, "decode_s": 4.731, "ttft_s": 5.688752, "first_answer_s": null, "total_s": 10.407609 }, "metrics": { "prompt_tps": 2896.816277, "decode_tps": 108.222363 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r3", "phase": "performance", "case_id": "short", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 317, "decode_tokens": 317 }, "timings": { "prompt_s": 0.4459, "decode_s": 2.5984, "ttft_s": 0.463904, "first_answer_s": 1.365892, "total_s": 3.050159 }, "metrics": { "prompt_tps": 172.684458, "decode_tps": 121.998153 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r3", "phase": "performance", "case_id": "prompt-4k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.3272, "decode_s": 4.533, "ttft_s": 2.345556, "first_answer_s": 6.357582, "total_s": 6.866221 }, "metrics": { "prompt_tps": 1758.765899, "decode_tps": 112.949482 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r3", "phase": "performance", "case_id": "prompt-16k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 5.6322, "decode_s": 4.6611, "ttft_s": 5.654213, "first_answer_s": 9.99711, "total_s": 10.304463 }, "metrics": { "prompt_tps": 2909.520259, "decode_tps": 109.845315 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "sanity-I1", "phase": "sanity", "case_id": "I1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 82, "cached_tokens": 0, "read_tokens": 82, "output_tokens": 74, "decode_tokens": 74 }, "timings": { "prompt_s": 0.4807, "decode_s": 0.5896, "ttft_s": 0.497558, "first_answer_s": 0.987623, "total_s": 1.075092 }, "metrics": { "prompt_tps": 170.584564, "decode_tps": 125.50882 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I2", "phase": "sanity", "case_id": "I2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 90, "cached_tokens": 0, "read_tokens": 90, "output_tokens": 60, "decode_tokens": 60 }, "timings": { "prompt_s": 0.3628, "decode_s": 0.453, "ttft_s": 0.379262, "first_answer_s": 0.712261, "total_s": 0.820376 }, "metrics": { "prompt_tps": 248.070562, "decode_tps": 132.450331 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I3", "phase": "sanity", "case_id": "I3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 85, "cached_tokens": 0, "read_tokens": 85, "output_tokens": 94, "decode_tokens": 94 }, "timings": { "prompt_s": 0.3577, "decode_s": 0.6801, "ttft_s": 0.374766, "first_answer_s": 0.939785, "total_s": 1.043563 }, "metrics": { "prompt_tps": 237.629298, "decode_tps": 138.214968 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I4", "phase": "sanity", "case_id": "I4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 71, "cached_tokens": 0, "read_tokens": 71, "output_tokens": 43, "decode_tokens": 43 }, "timings": { "prompt_s": 0.3718, "decode_s": 0.3075, "ttft_s": 0.38716, "first_answer_s": 0.684152, "total_s": 0.684363 }, "metrics": { "prompt_tps": 190.962883, "decode_tps": 139.837398 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R1", "phase": "sanity", "case_id": "R1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 86, "decode_tokens": 86 }, "timings": { "prompt_s": 2.3603, "decode_s": 0.7084, "ttft_s": 2.384613, "first_answer_s": 3.01378, "total_s": 3.081229 }, "metrics": { "prompt_tps": 1732.406897, "decode_tps": 121.400339 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R2", "phase": "sanity", "case_id": "R2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 59, "decode_tokens": 59 }, "timings": { "prompt_s": 2.3216, "decode_s": 0.4739, "ttft_s": 2.341247, "first_answer_s": 2.733301, "total_s": 2.803634 }, "metrics": { "prompt_tps": 1761.28532, "decode_tps": 124.498839 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R3", "phase": "sanity", "case_id": "R3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16382, "cached_tokens": 0, "read_tokens": 16382, "output_tokens": 67, "decode_tokens": 67 }, "timings": { "prompt_s": 5.5574, "decode_s": 0.4849, "ttft_s": 5.57943, "first_answer_s": 5.988426, "total_s": 6.053483 }, "metrics": { "prompt_tps": 2947.781337, "decode_tps": 138.172819 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R4", "phase": "sanity", "case_id": "R4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16390, "cached_tokens": 0, "read_tokens": 16390, "output_tokens": 389, "decode_tokens": 389 }, "timings": { "prompt_s": 5.7488, "decode_s": 3.0998, "ttft_s": 5.772861, "first_answer_s": 8.657922, "total_s": 8.861268 }, "metrics": { "prompt_tps": 2851.02978, "decode_tps": 125.491967 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-T1", "phase": "sanity", "case_id": "T1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 325, "cached_tokens": 0, "read_tokens": 325, "output_tokens": 61, "decode_tokens": 61 }, "timings": { "prompt_s": 1.3087, "decode_s": 0.4521, "ttft_s": 1.327412, "first_answer_s": 1.617626, "total_s": 1.76764 }, "metrics": { "prompt_tps": 248.338045, "decode_tps": 134.925901 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T2", "phase": "sanity", "case_id": "T2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 335, "cached_tokens": 0, "read_tokens": 335, "output_tokens": 113, "decode_tokens": 113 }, "timings": { "prompt_s": 0.9003, "decode_s": 0.9808, "ttft_s": 0.915581, "first_answer_s": 1.623599, "total_s": 1.885056 }, "metrics": { "prompt_tps": 372.098189, "decode_tps": 115.212072 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T3", "phase": "sanity", "case_id": "T3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 352, "cached_tokens": 0, "read_tokens": 352, "output_tokens": 86, "decode_tokens": 86 }, "timings": { "prompt_s": 0.9045, "decode_s": 0.5767, "ttft_s": 0.921407, "first_answer_s": 1.203417, "total_s": 1.486321 }, "metrics": { "prompt_tps": 389.165285, "decode_tps": 149.124328 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T4", "phase": "sanity", "case_id": "T4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 339, "cached_tokens": 0, "read_tokens": 339, "output_tokens": 84, "decode_tokens": 84 }, "timings": { "prompt_s": 0.8834, "decode_s": 0.5997, "ttft_s": 0.899896, "first_answer_s": 1.284013, "total_s": 1.488235 }, "metrics": { "prompt_tps": 383.744623, "decode_tps": 140.070035 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } } ], "summaries": [ { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 172.684458, "min": 165.841051, "max": 209.125475, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 122.924924, "min": 121.998153, "max": 123.834156, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 0.463904, "min": 0.384271, "max": 0.485776, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 1.016923, "min": 0.882254, "max": 1.365892, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 3.956819, "min": 3.050159, "max": 4.20006, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 1718.736877, "min": 1558.822409, "max": 1758.765899, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 112.949482, "min": 110.299662, "max": 125.149715, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 2.399576, "min": 2.345556, "max": 2.645078, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 6.090085, "min": 5.137586, "max": 6.357582, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 6.866221, "min": 6.478621, "max": 7.275433, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 2896.816277, "min": 2864.810056, "max": 2909.520259, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 108.222363, "min": 99.83426, "max": 109.845315, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 5.688752, "min": 5.654213, "max": 5.74338, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 9.99711, "min": 9.99711, "max": 9.99711, "n": 1, "included_request_ids": [ "performance-prompt-16k-r3" ], "excluded": [ { "request_id": "performance-prompt-16k-r1", "reason": "first_answer_s is not available for this request" }, { "request_id": "performance-prompt-16k-r2", "reason": "first_answer_s is not available for this request" } ] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 10.407609, "min": 10.304463, "max": 10.859748, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] } ], "concurrency_rounds": [], "quality": { "passed": 12, "total": 12, "categories": { "instruction": { "passed": 4, "total": 4 }, "retrieval": { "passed": 4, "total": 4 }, "tool_call": { "passed": 4, "total": 4 } } }, "artifacts": [ { "kind": "outputs", "path": "outputs.jsonl", "sha256": "49fb7ad0c32a991b5e129e266e17bc69a95bc07ca8c27673a29d54b8174a2b66" }, { "kind": "config", "path": "config.json", "sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410" }, { "kind": "provenance", "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, { "kind": "provenance", "path": "provenance/corpus-manifest.sha256", "sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407" }, { "kind": "provenance", "path": "provenance/corpus-build.json", "sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2" } ], "missing": { "/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)", "/run/optional_duration_s": "the optional concurrency stage was not selected", "/hardware/storage": "the drive holding the model files could not be identified", "/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background", "/software/build_options": "the server does not report build options; pass --build-options for a source build", "/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision", "/model/source_revision": "not given: pass --source-revision", "/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)", "/model/tokenizer_revision": "not given: pass --tokenizer-revision", "/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)", "/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config", "/requests/0/round": "not applicable outside the concurrency stage", "/requests/0/error": "none: the request ended normally", "/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/0/evaluation": "not applicable outside the sanity stage", "/requests/1/round": "not applicable outside the concurrency stage", "/requests/1/error": "none: the request ended normally", "/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/1/evaluation": "not applicable outside the sanity stage", "/requests/2/round": "not applicable outside the concurrency stage", "/requests/2/error": "none: the request ended normally", "/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/2/evaluation": "not applicable outside the sanity stage", "/requests/3/round": "not applicable outside the concurrency stage", "/requests/3/error": "none: the request ended normally", "/requests/3/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)", "/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/3/evaluation": "not applicable outside the sanity stage", "/requests/4/round": "not applicable outside the concurrency stage", "/requests/4/error": "none: the request ended normally", "/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/4/evaluation": "not applicable outside the sanity stage", "/requests/5/round": "not applicable outside the concurrency stage", "/requests/5/error": "none: the request ended normally", "/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/5/evaluation": "not applicable outside the sanity stage", "/requests/6/round": "not applicable outside the concurrency stage", "/requests/6/error": "none: the request ended normally", "/requests/6/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)", "/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/6/evaluation": "not applicable outside the sanity stage", "/requests/7/round": "not applicable outside the concurrency stage", "/requests/7/error": "none: the request ended normally", "/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/7/evaluation": "not applicable outside the sanity stage", "/requests/8/round": "not applicable outside the concurrency stage", "/requests/8/error": "none: the request ended normally", "/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/8/evaluation": "not applicable outside the sanity stage", "/requests/9/round": "not applicable outside the concurrency stage", "/requests/9/error": "none: the request ended normally", "/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/9/evaluation": "not applicable outside the sanity stage", "/requests/10/round": "not applicable outside the concurrency stage", "/requests/10/error": "none: the request ended normally", "/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/11/round": "not applicable outside the concurrency stage", "/requests/11/error": "none: the request ended normally", "/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/12/round": "not applicable outside the concurrency stage", "/requests/12/error": "none: the request ended normally", "/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/13/round": "not applicable outside the concurrency stage", "/requests/13/error": "none: the request ended normally", "/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/14/round": "not applicable outside the concurrency stage", "/requests/14/error": "none: the request ended normally", "/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/15/round": "not applicable outside the concurrency stage", "/requests/15/error": "none: the request ended normally", "/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/16/round": "not applicable outside the concurrency stage", "/requests/16/error": "none: the request ended normally", "/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/17/round": "not applicable outside the concurrency stage", "/requests/17/error": "none: the request ended normally", "/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/18/round": "not applicable outside the concurrency stage", "/requests/18/error": "none: the request ended normally", "/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/19/round": "not applicable outside the concurrency stage", "/requests/19/error": "none: the request ended normally", "/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/20/round": "not applicable outside the concurrency stage", "/requests/20/error": "none: the request ended normally", "/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/21/round": "not applicable outside the concurrency stage", "/requests/21/error": "none: the request ended normally", "/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists" }, "extensions": { "suite": "standard-v1", "runner": "strata_cli.bench_standard", "status_reasons": [ "no model file could be hashed" ], "started_engine": false, "server_requests_before_run": 0, "other_engine_options": {}, "request_checks": { "warmup": { "max_shared_prefix_tokens": 47, "frozen_input_tokens": 75, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r1": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r1": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r1": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r2": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r2": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r2": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r3": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r3": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r3": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 82, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 90, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 85, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 71, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16382, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16390, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T1": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 325, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T2": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 335, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T3": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 352, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T4": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 339, "unexpected_payload_reuse": false, "length_outside_tolerance": false } }, "optional_stage_skipped": false, "api_key_sent": true } }