{ "format": "strata-standard-benchmark", "schema_version": 1, "protocol": { "id": "strata-qwen38-thinking-v1-draft.1", "git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "profile": "qwen3.8-flash-next-thinking", "corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407", "model_card_revision": null }, "run": { "id": "20261026T234702Z-0f07e24d", "started_at": "2026-10-26T23:47:02Z", "status": "complete", "required_duration_s": 91.197, "optional_duration_s": 47.974, "optional_concurrency": true, "command": " bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --optional-concurrency --track default --track-notes lovey 2x3090, 250 W cap, gpu-unlock, setup defaults + --max-context 131072 --quantization IQ3_S" }, "hardware": { "gpus": [ { "device_index": 0, "name": "NVIDIA GeForce RTX 3090", "memory_bytes": 25769803776, "memory_kind": "dedicated", "power_limit_w": 250.0 } ], "cpu": "Intel(R) Core(TM) i9-14900KS", "ram_bytes": 118089846784, "storage": null, "background_workloads": null }, "software": { "os": { "name": "NixOS 26.05 (Yarara)", "version": "6.18.55" }, "drivers": [ { "name": "gpu-driver", "version": "595.71.05" } ], "runtime": [ { "name": "cuda", "version": "13.2" } ], "strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "engine_version": "0.1.51", "build_kind": "source", "runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "build_options": null }, "model": { "id": "qwen3.8-flash-next-iq3_s", "revision": null, "source_revision": null, "quantization": "IQ3_S", "files": [ { "path": "Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf", "sha256": "4c1eb2ceb4915e1192f4f386021897bde56a97f40a0bb78bb86465e0f7d2aca3" }, { "path": "Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf", "sha256": "316b46f3a2dbd68c900f43136ab9449f9dcc3725dfd8c794847c204bc161e113" } ], "tokenizer_revision": null, "custom_pack": null }, "configuration": { "track": "default", "track_notes": "lovey 2x3090, 250 W cap, gpu-unlock, setup defaults + --max-context 131072", "context_tokens": 131072, "server_capacity": 1, "kv_cache": null, "expert_cache": null, "offloading": null, "prefill": null, "mtp": null, "calibration": null, "experimental_options": null, "template": { "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, "sampling": { "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0, "reasoning_effort": "xhigh" }, "chat_template_kwargs": { "enable_thinking": true, "preserve_thinking": true } }, "request_counts": { "warmup": { "planned": 1, "attempted": 1, "completed": 0, "truncated": 1, "failed": 0, "skipped": 0 }, "performance": { "planned": 9, "attempted": 9, "completed": 2, "truncated": 7, "failed": 0, "skipped": 0 }, "sanity": { "planned": 12, "attempted": 12, "completed": 12, "truncated": 0, "failed": 0, "skipped": 0 }, "concurrency": { "planned": 12, "attempted": 12, "completed": 3, "truncated": 9, "failed": 0, "skipped": 0 } }, "requests": [ { "id": "warmup", "phase": "warmup", "case_id": "warmup", "repetition": 1, "seed": 1, "max_output_tokens": 128, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 75, "cached_tokens": 0, "read_tokens": 75, "output_tokens": 128, "decode_tokens": 128 }, "timings": { "prompt_s": 0.5378, "decode_s": 1.208, "ttft_s": 0.561074, "first_answer_s": 1.308051, "total_s": 1.750101 }, "metrics": { "prompt_tps": 139.457047, "decode_tps": 105.960265 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r1", "phase": "performance", "case_id": "short", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.3929, "decode_s": 3.7771, "ttft_s": 0.409799, "first_answer_s": 0.89164, "total_s": 4.174695 }, "metrics": { "prompt_tps": 195.978621, "decode_tps": 135.553732 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r1", "phase": "performance", "case_id": "prompt-4k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.307, "decode_s": 4.5458, "ttft_s": 2.326971, "first_answer_s": null, "total_s": 6.861012 }, "metrics": { "prompt_tps": 1774.165583, "decode_tps": 112.63144 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r1", "phase": "performance", "case_id": "prompt-16k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 5.5643, "decode_s": 4.1108, "ttft_s": 5.591829, "first_answer_s": 8.142807, "total_s": 9.690426 }, "metrics": { "prompt_tps": 2945.024531, "decode_tps": 124.549966 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r2", "phase": "performance", "case_id": "short", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 338, "decode_tokens": 338 }, "timings": { "prompt_s": 0.3396, "decode_s": 2.7229, "ttft_s": 0.353313, "first_answer_s": 1.171217, "total_s": 3.065101 }, "metrics": { "prompt_tps": 226.737338, "decode_tps": 124.132359 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r2", "phase": "performance", "case_id": "prompt-4k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.302, "decode_s": 3.9869, "ttft_s": 2.320082, "first_answer_s": 4.956982, "total_s": 6.296036 }, "metrics": { "prompt_tps": 1778.019114, "decode_tps": 128.420577 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r2", "phase": "performance", "case_id": "prompt-16k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 5.5777, "decode_s": 3.8954, "ttft_s": 5.608843, "first_answer_s": 7.790823, "total_s": 9.493855 }, "metrics": { "prompt_tps": 2937.949334, "decode_tps": 131.43708 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r3", "phase": "performance", "case_id": "short", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 510, "decode_tokens": 510 }, "timings": { "prompt_s": 0.3254, "decode_s": 3.9007, "ttft_s": 0.341909, "first_answer_s": 2.546901, "total_s": 4.23085 }, "metrics": { "prompt_tps": 236.631838, "decode_tps": 130.745764 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r3", "phase": "performance", "case_id": "prompt-4k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.3045, "decode_s": 4.1569, "ttft_s": 2.323905, "first_answer_s": 5.64188, "total_s": 6.469928 }, "metrics": { "prompt_tps": 1776.090258, "decode_tps": 123.168707 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r3", "phase": "performance", "case_id": "prompt-16k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 5.5843, "decode_s": 4.452, "ttft_s": 5.613231, "first_answer_s": null, "total_s": 10.055156 }, "metrics": { "prompt_tps": 2934.477016, "decode_tps": 115.004492 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "sanity-I1", "phase": "sanity", "case_id": "I1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 82, "cached_tokens": 0, "read_tokens": 82, "output_tokens": 68, "decode_tokens": 68 }, "timings": { "prompt_s": 0.3304, "decode_s": 0.5127, "ttft_s": 0.346933, "first_answer_s": 0.741939, "total_s": 0.84901 }, "metrics": { "prompt_tps": 248.184019, "decode_tps": 132.631168 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I2", "phase": "sanity", "case_id": "I2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 90, "cached_tokens": 0, "read_tokens": 90, "output_tokens": 60, "decode_tokens": 60 }, "timings": { "prompt_s": 0.3295, "decode_s": 0.3899, "ttft_s": 0.345914, "first_answer_s": 0.634915, "total_s": 0.725301 }, "metrics": { "prompt_tps": 273.141123, "decode_tps": 153.885612 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I3", "phase": "sanity", "case_id": "I3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 85, "cached_tokens": 0, "read_tokens": 85, "output_tokens": 77, "decode_tokens": 77 }, "timings": { "prompt_s": 0.3485, "decode_s": 0.4826, "ttft_s": 0.365442, "first_answer_s": 0.756434, "total_s": 0.835819 }, "metrics": { "prompt_tps": 243.902439, "decode_tps": 159.552424 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I4", "phase": "sanity", "case_id": "I4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 71, "cached_tokens": 0, "read_tokens": 71, "output_tokens": 44, "decode_tokens": 44 }, "timings": { "prompt_s": 0.2982, "decode_s": 0.309, "ttft_s": 0.315554, "first_answer_s": 0.614546, "total_s": 0.614778 }, "metrics": { "prompt_tps": 238.095238, "decode_tps": 142.394822 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R1", "phase": "sanity", "case_id": "R1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 72, "decode_tokens": 72 }, "timings": { "prompt_s": 2.3058, "decode_s": 0.5465, "ttft_s": 2.324606, "first_answer_s": 2.804535, "total_s": 2.859877 }, "metrics": { "prompt_tps": 1773.35415, "decode_tps": 131.747484 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R2", "phase": "sanity", "case_id": "R2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 59, "decode_tokens": 59 }, "timings": { "prompt_s": 2.3156, "decode_s": 0.4081, "ttft_s": 2.335515, "first_answer_s": 2.678503, "total_s": 2.732544 }, "metrics": { "prompt_tps": 1765.849024, "decode_tps": 144.572409 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R3", "phase": "sanity", "case_id": "R3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16382, "cached_tokens": 0, "read_tokens": 16382, "output_tokens": 63, "decode_tokens": 63 }, "timings": { "prompt_s": 5.5868, "decode_s": 0.4552, "ttft_s": 5.615875, "first_answer_s": 6.008867, "total_s": 6.059926 }, "metrics": { "prompt_tps": 2932.26892, "decode_tps": 138.400703 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R4", "phase": "sanity", "case_id": "R4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16390, "cached_tokens": 0, "read_tokens": 16390, "output_tokens": 382, "decode_tokens": 382 }, "timings": { "prompt_s": 5.6744, "decode_s": 2.9063, "ttft_s": 5.703869, "first_answer_s": 8.408836, "total_s": 8.598088 }, "metrics": { "prompt_tps": 2888.41111, "decode_tps": 131.438599 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-T1", "phase": "sanity", "case_id": "T1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 325, "cached_tokens": 0, "read_tokens": 325, "output_tokens": 77, "decode_tokens": 77 }, "timings": { "prompt_s": 0.8607, "decode_s": 0.4764, "ttft_s": 0.876618, "first_answer_s": 1.20064, "total_s": 1.342668 }, "metrics": { "prompt_tps": 377.599628, "decode_tps": 161.628883 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T2", "phase": "sanity", "case_id": "T2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 335, "cached_tokens": 0, "read_tokens": 335, "output_tokens": 91, "decode_tokens": 91 }, "timings": { "prompt_s": 0.8719, "decode_s": 0.691, "ttft_s": 0.888903, "first_answer_s": 1.368934, "total_s": 1.569968 }, "metrics": { "prompt_tps": 384.218374, "decode_tps": 131.693198 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T3", "phase": "sanity", "case_id": "T3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 352, "cached_tokens": 0, "read_tokens": 352, "output_tokens": 87, "decode_tokens": 87 }, "timings": { "prompt_s": 0.8466, "decode_s": 0.609, "ttft_s": 0.863912, "first_answer_s": 1.190931, "total_s": 1.461985 }, "metrics": { "prompt_tps": 415.78077, "decode_tps": 142.857143 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T4", "phase": "sanity", "case_id": "T4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 339, "cached_tokens": 0, "read_tokens": 339, "output_tokens": 90, "decode_tokens": 90 }, "timings": { "prompt_s": 0.849, "decode_s": 0.5909, "ttft_s": 0.866735, "first_answer_s": 1.25176, "total_s": 1.446792 }, "metrics": { "prompt_tps": 399.293286, "decode_tps": 152.310036 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "concurrency-r1-c1", "phase": "concurrency", "case_id": "short-burst", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 4, "round": 1, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 395, "decode_tokens": 395 }, "timings": { "prompt_s": 0.2999, "decode_s": 2.8552, "ttft_s": 8.740826, "first_answer_s": 9.478704, "total_s": 11.584859 }, "metrics": { "prompt_tps": 260.086696, "decode_tps": 138.344074 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r1-c2", "phase": "concurrency", "case_id": "short-burst", "repetition": 1, "seed": 2, "max_output_tokens": 512, "concurrency": 4, "round": 1, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 458, "decode_tokens": 458 }, "timings": { "prompt_s": 0.3293, "decode_s": 3.372, "ttft_s": 0.345774, "first_answer_s": 0.889766, "total_s": 3.706825 }, "metrics": { "prompt_tps": 236.86608, "decode_tps": 135.824437 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r1-c3", "phase": "concurrency", "case_id": "short-burst", "repetition": 1, "seed": 3, "max_output_tokens": 512, "concurrency": 4, "round": 1, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.3092, "decode_s": 4.4084, "ttft_s": 4.030741, "first_answer_s": 7.659896, "total_s": 8.42778 }, "metrics": { "prompt_tps": 252.263907, "decode_tps": 116.141911 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r1-c4", "phase": "concurrency", "case_id": "short-burst", "repetition": 1, "seed": 4, "max_output_tokens": 512, "concurrency": 4, "round": 1, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.3034, "decode_s": 4.3636, "ttft_s": 11.901639, "first_answer_s": 15.534626, "total_s": 16.253254 }, "metrics": { "prompt_tps": 257.086355, "decode_tps": 117.334311 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r2-c1", "phase": "concurrency", "case_id": "short-burst", "repetition": 2, "seed": 5, "max_output_tokens": 512, "concurrency": 4, "round": 2, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.302, "decode_s": 3.4163, "ttft_s": 11.905763, "first_answer_s": 12.623437, "total_s": 15.310805 }, "metrics": { "prompt_tps": 258.278146, "decode_tps": 149.869742 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r2-c2", "phase": "concurrency", "case_id": "short-burst", "repetition": 2, "seed": 6, "max_output_tokens": 512, "concurrency": 4, "round": 2, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 401, "decode_tokens": 401 }, "timings": { "prompt_s": 0.2983, "decode_s": 2.785, "ttft_s": 0.314868, "first_answer_s": 0.917163, "total_s": 3.088908 }, "metrics": { "prompt_tps": 261.48173, "decode_tps": 143.985637 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r2-c3", "phase": "concurrency", "case_id": "short-burst", "repetition": 2, "seed": 7, "max_output_tokens": 512, "concurrency": 4, "round": 2, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.2985, "decode_s": 4.1389, "ttft_s": 7.461829, "first_answer_s": 10.869258, "total_s": 11.589246 }, "metrics": { "prompt_tps": 261.306533, "decode_tps": 123.704366 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r2-c4", "phase": "concurrency", "case_id": "short-burst", "repetition": 2, "seed": 8, "max_output_tokens": 512, "concurrency": 4, "round": 2, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.2994, "decode_s": 3.7582, "ttft_s": 3.402808, "first_answer_s": 4.669764, "total_s": 7.14983 }, "metrics": { "prompt_tps": 260.521042, "decode_tps": 136.235432 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r3-c1", "phase": "concurrency", "case_id": "short-burst", "repetition": 3, "seed": 9, "max_output_tokens": 512, "concurrency": 4, "round": 3, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.302, "decode_s": 3.7219, "ttft_s": 4.174711, "first_answer_s": 5.235479, "total_s": 7.885758 }, "metrics": { "prompt_tps": 258.278146, "decode_tps": 137.564147 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r3-c2", "phase": "concurrency", "case_id": "short-burst", "repetition": 3, "seed": 10, "max_output_tokens": 512, "concurrency": 4, "round": 3, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.2994, "decode_s": 4.0292, "ttft_s": 12.211817, "first_answer_s": 14.241092, "total_s": 16.230166 }, "metrics": { "prompt_tps": 260.521042, "decode_tps": 127.072372 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r3-c3", "phase": "concurrency", "case_id": "short-burst", "repetition": 3, "seed": 11, "max_output_tokens": 512, "concurrency": 4, "round": 3, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.303, "decode_s": 3.7087, "ttft_s": 8.201769, "first_answer_s": 9.483954, "total_s": 11.899125 }, "metrics": { "prompt_tps": 257.425743, "decode_tps": 138.053765 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "concurrency-r3-c4", "phase": "concurrency", "case_id": "short-burst", "repetition": 3, "seed": 12, "max_output_tokens": 512, "concurrency": 4, "round": 3, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 78, "cached_tokens": 0, "read_tokens": 78, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 0.2973, "decode_s": 3.5563, "ttft_s": 0.312613, "first_answer_s": 1.198234, "total_s": 3.85865 }, "metrics": { "prompt_tps": 262.361251, "decode_tps": 143.969856 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null } ], "summaries": [ { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 226.737338, "min": 195.978621, "max": 236.631838, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 130.745764, "min": 124.132359, "max": 135.553732, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 0.353313, "min": 0.341909, "max": 0.409799, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 1.171217, "min": 0.89164, "max": 2.546901, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 4.174695, "min": 3.065101, "max": 4.23085, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 1776.090258, "min": 1774.165583, "max": 1778.019114, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 123.168707, "min": 112.63144, "max": 128.420577, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 2.323905, "min": 2.320082, "max": 2.326971, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 5.299431, "min": 4.956982, "max": 5.64188, "n": 2, "included_request_ids": [ "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [ { "request_id": "performance-prompt-4k-r1", "reason": "first_answer_s is not available for this request" } ] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 6.469928, "min": 6.296036, "max": 6.861012, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 2937.949334, "min": 2934.477016, "max": 2945.024531, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 124.549966, "min": 115.004492, "max": 131.43708, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 5.608843, "min": 5.591829, "max": 5.613231, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 7.966815, "min": 7.790823, "max": 8.142807, "n": 2, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2" ], "excluded": [ { "request_id": "performance-prompt-16k-r3", "reason": "first_answer_s is not available for this request" } ] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 9.690426, "min": 9.493855, "max": 10.055156, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "concurrency", "case_id": "short-burst", "concurrency": 4, "metric": "prompt_tps", "unit": "tokens/s", "median": 259.182421, "min": 236.86608, "max": 262.361251, "n": 12, "included_request_ids": [ "concurrency-r1-c1", "concurrency-r1-c2", "concurrency-r1-c3", "concurrency-r1-c4", "concurrency-r2-c1", "concurrency-r2-c2", "concurrency-r2-c3", "concurrency-r2-c4", "concurrency-r3-c1", "concurrency-r3-c2", "concurrency-r3-c3", "concurrency-r3-c4" ], "excluded": [] }, { "phase": "concurrency", "case_id": "short-burst", "concurrency": 4, "metric": "decode_tps", "unit": "tokens/s", "median": 136.899789, "min": 116.141911, "max": 149.869742, "n": 12, "included_request_ids": [ "concurrency-r1-c1", "concurrency-r1-c2", "concurrency-r1-c3", "concurrency-r1-c4", "concurrency-r2-c1", "concurrency-r2-c2", "concurrency-r2-c3", "concurrency-r2-c4", "concurrency-r3-c1", "concurrency-r3-c2", "concurrency-r3-c3", "concurrency-r3-c4" ], "excluded": [] }, { "phase": "concurrency", "case_id": "short-burst", "concurrency": 4, "metric": "ttft_s", "unit": "s", "median": 5.81827, "min": 0.312613, "max": 12.211817, "n": 12, "included_request_ids": [ "concurrency-r1-c1", "concurrency-r1-c2", "concurrency-r1-c3", "concurrency-r1-c4", "concurrency-r2-c1", "concurrency-r2-c2", "concurrency-r2-c3", "concurrency-r2-c4", "concurrency-r3-c1", "concurrency-r3-c2", "concurrency-r3-c3", "concurrency-r3-c4" ], "excluded": [] }, { "phase": "concurrency", "case_id": "short-burst", "concurrency": 4, "metric": "first_answer_s", "unit": "s", "median": 8.5693, "min": 0.889766, "max": 15.534626, "n": 12, "included_request_ids": [ "concurrency-r1-c1", "concurrency-r1-c2", "concurrency-r1-c3", "concurrency-r1-c4", "concurrency-r2-c1", "concurrency-r2-c2", "concurrency-r2-c3", "concurrency-r2-c4", "concurrency-r3-c1", "concurrency-r3-c2", "concurrency-r3-c3", "concurrency-r3-c4" ], "excluded": [] }, { "phase": "concurrency", "case_id": "short-burst", "concurrency": 4, "metric": "total_s", "unit": "s", "median": 10.00632, "min": 3.088908, "max": 16.253254, "n": 12, "included_request_ids": [ "concurrency-r1-c1", "concurrency-r1-c2", "concurrency-r1-c3", "concurrency-r1-c4", "concurrency-r2-c1", "concurrency-r2-c2", "concurrency-r2-c3", "concurrency-r2-c4", "concurrency-r3-c1", "concurrency-r3-c2", "concurrency-r3-c3", "concurrency-r3-c4" ], "excluded": [] } ], "concurrency_rounds": [ { "round": 1, "request_ids": [ "concurrency-r1-c1", "concurrency-r1-c2", "concurrency-r1-c3", "concurrency-r1-c4" ], "wall_s": 16.25344, "output_tokens": 1877, "aggregate_tps": 115.483245, "failed": 0 }, { "round": 2, "request_ids": [ "concurrency-r2-c1", "concurrency-r2-c2", "concurrency-r2-c3", "concurrency-r2-c4" ], "wall_s": 15.310917, "output_tokens": 1937, "aggregate_tps": 126.511038, "failed": 0 }, { "round": 3, "request_ids": [ "concurrency-r3-c1", "concurrency-r3-c2", "concurrency-r3-c3", "concurrency-r3-c4" ], "wall_s": 16.230166, "output_tokens": 2048, "aggregate_tps": 126.184785, "failed": 0 } ], "quality": { "passed": 12, "total": 12, "categories": { "instruction": { "passed": 4, "total": 4 }, "retrieval": { "passed": 4, "total": 4 }, "tool_call": { "passed": 4, "total": 4 } } }, "artifacts": [ { "kind": "outputs", "path": "outputs.jsonl", "sha256": "d6aae509aa76e077f381700178931e540314b3f2b15916a83cf4735d6f463a4b" }, { "kind": "config", "path": "config.json", "sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410" }, { "kind": "provenance", "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, { "kind": "provenance", "path": "provenance/corpus-manifest.sha256", "sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407" }, { "kind": "provenance", "path": "provenance/corpus-build.json", "sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2" } ], "missing": { "/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)", "/hardware/storage": "the drive holding the model files could not be identified", "/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background", "/software/build_options": "the server does not report build options; pass --build-options for a source build", "/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision", "/model/source_revision": "not given: pass --source-revision", "/model/tokenizer_revision": "not given: pass --tokenizer-revision", "/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)", "/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config", "/requests/0/round": "not applicable outside the concurrency stage", "/requests/0/error": "none: the request ended normally", "/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/0/evaluation": "not applicable outside the sanity stage", "/requests/1/round": "not applicable outside the concurrency stage", "/requests/1/error": "none: the request ended normally", "/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/1/evaluation": "not applicable outside the sanity stage", "/requests/2/round": "not applicable outside the concurrency stage", "/requests/2/error": "none: the request ended normally", "/requests/2/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)", "/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/2/evaluation": "not applicable outside the sanity stage", "/requests/3/round": "not applicable outside the concurrency stage", "/requests/3/error": "none: the request ended normally", "/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/3/evaluation": "not applicable outside the sanity stage", "/requests/4/round": "not applicable outside the concurrency stage", "/requests/4/error": "none: the request ended normally", "/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/4/evaluation": "not applicable outside the sanity stage", "/requests/5/round": "not applicable outside the concurrency stage", "/requests/5/error": "none: the request ended normally", "/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/5/evaluation": "not applicable outside the sanity stage", "/requests/6/round": "not applicable outside the concurrency stage", "/requests/6/error": "none: the request ended normally", "/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/6/evaluation": "not applicable outside the sanity stage", "/requests/7/round": "not applicable outside the concurrency stage", "/requests/7/error": "none: the request ended normally", "/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/7/evaluation": "not applicable outside the sanity stage", "/requests/8/round": "not applicable outside the concurrency stage", "/requests/8/error": "none: the request ended normally", "/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/8/evaluation": "not applicable outside the sanity stage", "/requests/9/round": "not applicable outside the concurrency stage", "/requests/9/error": "none: the request ended normally", "/requests/9/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)", "/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/9/evaluation": "not applicable outside the sanity stage", "/requests/10/round": "not applicable outside the concurrency stage", "/requests/10/error": "none: the request ended normally", "/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/11/round": "not applicable outside the concurrency stage", "/requests/11/error": "none: the request ended normally", "/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/12/round": "not applicable outside the concurrency stage", "/requests/12/error": "none: the request ended normally", "/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/13/round": "not applicable outside the concurrency stage", "/requests/13/error": "none: the request ended normally", "/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/14/round": "not applicable outside the concurrency stage", "/requests/14/error": "none: the request ended normally", "/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/15/round": "not applicable outside the concurrency stage", "/requests/15/error": "none: the request ended normally", "/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/16/round": "not applicable outside the concurrency stage", "/requests/16/error": "none: the request ended normally", "/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/17/round": "not applicable outside the concurrency stage", "/requests/17/error": "none: the request ended normally", "/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/18/round": "not applicable outside the concurrency stage", "/requests/18/error": "none: the request ended normally", "/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/19/round": "not applicable outside the concurrency stage", "/requests/19/error": "none: the request ended normally", "/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/20/round": "not applicable outside the concurrency stage", "/requests/20/error": "none: the request ended normally", "/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/21/round": "not applicable outside the concurrency stage", "/requests/21/error": "none: the request ended normally", "/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/22/error": "none: the request ended normally", "/requests/22/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/22/evaluation": "not applicable outside the sanity stage", "/requests/23/error": "none: the request ended normally", "/requests/23/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/23/evaluation": "not applicable outside the sanity stage", "/requests/24/error": "none: the request ended normally", "/requests/24/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/24/evaluation": "not applicable outside the sanity stage", "/requests/25/error": "none: the request ended normally", "/requests/25/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/25/evaluation": "not applicable outside the sanity stage", "/requests/26/error": "none: the request ended normally", "/requests/26/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/26/evaluation": "not applicable outside the sanity stage", "/requests/27/error": "none: the request ended normally", "/requests/27/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/27/evaluation": "not applicable outside the sanity stage", "/requests/28/error": "none: the request ended normally", "/requests/28/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/28/evaluation": "not applicable outside the sanity stage", "/requests/29/error": "none: the request ended normally", "/requests/29/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/29/evaluation": "not applicable outside the sanity stage", "/requests/30/error": "none: the request ended normally", "/requests/30/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/30/evaluation": "not applicable outside the sanity stage", "/requests/31/error": "none: the request ended normally", "/requests/31/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/31/evaluation": "not applicable outside the sanity stage", "/requests/32/error": "none: the request ended normally", "/requests/32/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/32/evaluation": "not applicable outside the sanity stage", "/requests/33/error": "none: the request ended normally", "/requests/33/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/33/evaluation": "not applicable outside the sanity stage" }, "extensions": { "suite": "standard-v1", "runner": "strata_cli.bench_standard", "status_reasons": [], "started_engine": false, "server_requests_before_run": 1122, "other_engine_options": {}, "request_checks": { "warmup": { "max_shared_prefix_tokens": 47, "frozen_input_tokens": 75, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r1": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r1": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r1": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r2": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r2": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r2": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r3": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r3": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r3": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 82, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 90, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 85, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 71, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16382, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16390, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T1": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 325, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T2": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 335, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T3": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 352, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T4": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 339, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r1-c1": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r1-c2": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r1-c3": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r1-c4": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r2-c1": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r2-c2": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r2-c3": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r2-c4": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r3-c1": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r3-c2": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r3-c3": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "concurrency-r3-c4": { "max_shared_prefix_tokens": 51, "frozen_input_tokens": 78, "unexpected_payload_reuse": false, "length_outside_tolerance": false } }, "optional_stage_skipped": false, "api_key_sent": true } }