{ "format": "strata-standard-benchmark", "schema_version": 1, "protocol": { "id": "strata-qwen38-thinking-v1-draft.1", "git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "profile": "qwen3.8-flash-next-thinking", "corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407", "model_card_revision": null }, "run": { "id": "20261027T000315Z-92972582", "started_at": "2026-10-27T00:03:15Z", "status": "nonconforming", "required_duration_s": 102.34, "optional_duration_s": null, "optional_concurrency": false, "command": " bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes arm prefill-32k: [\"--prefill\",\"auto:32768\"] --quantization IQ3_S" }, "hardware": { "gpus": [ { "device_index": 0, "name": "NVIDIA GeForce RTX 3090", "memory_bytes": 25769803776, "memory_kind": "dedicated", "power_limit_w": 250.0 } ], "cpu": "Intel(R) Core(TM) i9-14900KS", "ram_bytes": 118089846784, "storage": null, "background_workloads": null }, "software": { "os": { "name": "NixOS 26.05 (Yarara)", "version": "6.18.55" }, "drivers": [ { "name": "gpu-driver", "version": "595.71.05" } ], "runtime": [ { "name": "cuda", "version": "13.2" } ], "strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "engine_version": "0.1.51", "build_kind": "source", "runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35", "build_options": null }, "model": { "id": "qwen3.8-flash-next-iq3_s", "revision": null, "source_revision": null, "quantization": "IQ3_S", "files": null, "tokenizer_revision": null, "custom_pack": null }, "configuration": { "track": "custom", "track_notes": "arm prefill-32k: [\"--prefill\",\"auto:32768\"]", "context_tokens": 131072, "server_capacity": 1, "kv_cache": null, "expert_cache": null, "offloading": null, "prefill": null, "mtp": null, "calibration": null, "experimental_options": null, "template": { "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, "sampling": { "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0, "reasoning_effort": "xhigh" }, "chat_template_kwargs": { "enable_thinking": true, "preserve_thinking": true } }, "request_counts": { "warmup": { "planned": 1, "attempted": 1, "completed": 0, "truncated": 1, "failed": 0, "skipped": 0 }, "performance": { "planned": 9, "attempted": 9, "completed": 3, "truncated": 6, "failed": 0, "skipped": 0 }, "sanity": { "planned": 12, "attempted": 12, "completed": 12, "truncated": 0, "failed": 0, "skipped": 0 }, "concurrency": { "planned": 0, "attempted": 0, "completed": 0, "truncated": 0, "failed": 0, "skipped": 0 } }, "requests": [ { "id": "warmup", "phase": "warmup", "case_id": "warmup", "repetition": 1, "seed": 1, "max_output_tokens": 128, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 75, "cached_tokens": 0, "read_tokens": 75, "output_tokens": 128, "decode_tokens": 128 }, "timings": { "prompt_s": 0.4055, "decode_s": 1.1719, "ttft_s": 0.433842, "first_answer_s": 1.042827, "total_s": 1.581193 }, "metrics": { "prompt_tps": 184.956843, "decode_tps": 109.224337 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r1", "phase": "performance", "case_id": "short", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 447, "decode_tokens": 447 }, "timings": { "prompt_s": 0.3761, "decode_s": 3.6417, "ttft_s": 0.394492, "first_answer_s": 0.892493, "total_s": 4.024395 }, "metrics": { "prompt_tps": 204.732784, "decode_tps": 122.744872 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r1", "phase": "performance", "case_id": "prompt-4k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.6489, "decode_s": 4.588, "ttft_s": 2.666856, "first_answer_s": 6.134936, "total_s": 7.243141 }, "metrics": { "prompt_tps": 1545.169693, "decode_tps": 111.595466 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r1", "phase": "performance", "case_id": "prompt-16k", "repetition": 1, "seed": 1, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 7.6395, "decode_s": 4.329, "ttft_s": 7.669504, "first_answer_s": 9.770124, "total_s": 11.988469 }, "metrics": { "prompt_tps": 2145.03567, "decode_tps": 118.272118 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r2", "phase": "performance", "case_id": "short", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 449, "decode_tokens": 449 }, "timings": { "prompt_s": 0.4712, "decode_s": 3.7583, "ttft_s": 0.488926, "first_answer_s": 1.709907, "total_s": 4.236185 }, "metrics": { "prompt_tps": 163.412564, "decode_tps": 119.468909 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r2", "phase": "performance", "case_id": "prompt-4k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.4042, "decode_s": 4.0489, "ttft_s": 2.422599, "first_answer_s": 4.841742, "total_s": 6.460611 }, "metrics": { "prompt_tps": 1702.437401, "decode_tps": 126.454099 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r2", "phase": "performance", "case_id": "prompt-16k", "repetition": 2, "seed": 2, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 7.2906, "decode_s": 4.8339, "ttft_s": 7.315847, "first_answer_s": 11.621347, "total_s": 12.137885 }, "metrics": { "prompt_tps": 2247.688805, "decode_tps": 105.918616 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-short-r3", "phase": "performance", "case_id": "short", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 77, "cached_tokens": 0, "read_tokens": 77, "output_tokens": 260, "decode_tokens": 260 }, "timings": { "prompt_s": 0.4674, "decode_s": 2.0275, "ttft_s": 0.481843, "first_answer_s": 1.113049, "total_s": 2.497374 }, "metrics": { "prompt_tps": 164.741121, "decode_tps": 128.236745 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-4k-r3", "phase": "performance", "case_id": "prompt-4k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 4093, "cached_tokens": 0, "read_tokens": 4093, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 2.3474, "decode_s": 4.9717, "ttft_s": 2.363708, "first_answer_s": null, "total_s": 7.325637 }, "metrics": { "prompt_tps": 1743.631252, "decode_tps": 102.982883 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "performance-prompt-16k-r3", "phase": "performance", "case_id": "prompt-16k", "repetition": 3, "seed": 3, "max_output_tokens": 512, "concurrency": 1, "round": null, "status": "truncated", "finish_reason": "length", "error": null, "tokens": { "input_tokens": 16387, "cached_tokens": 0, "read_tokens": 16387, "output_tokens": 512, "decode_tokens": 512 }, "timings": { "prompt_s": 7.3176, "decode_s": 5.1227, "ttft_s": 7.347428, "first_answer_s": null, "total_s": 12.457752 }, "metrics": { "prompt_tps": 2239.39543, "decode_tps": 99.947293 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": null }, { "id": "sanity-I1", "phase": "sanity", "case_id": "I1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 82, "cached_tokens": 0, "read_tokens": 82, "output_tokens": 74, "decode_tokens": 74 }, "timings": { "prompt_s": 0.4924, "decode_s": 0.5651, "ttft_s": 0.507508, "first_answer_s": 0.976536, "total_s": 1.061626 }, "metrics": { "prompt_tps": 166.531275, "decode_tps": 130.950274 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I2", "phase": "sanity", "case_id": "I2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 90, "cached_tokens": 0, "read_tokens": 90, "output_tokens": 60, "decode_tokens": 60 }, "timings": { "prompt_s": 0.366, "decode_s": 0.4516, "ttft_s": 0.38275, "first_answer_s": 0.714752, "total_s": 0.823786 }, "metrics": { "prompt_tps": 245.901639, "decode_tps": 132.860939 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I3", "phase": "sanity", "case_id": "I3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 85, "cached_tokens": 0, "read_tokens": 85, "output_tokens": 90, "decode_tokens": 90 }, "timings": { "prompt_s": 0.368, "decode_s": 0.6786, "ttft_s": 0.3848, "first_answer_s": 0.858799, "total_s": 1.052106 }, "metrics": { "prompt_tps": 230.978261, "decode_tps": 132.625995 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-I4", "phase": "sanity", "case_id": "I4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 71, "cached_tokens": 0, "read_tokens": 71, "output_tokens": 44, "decode_tokens": 44 }, "timings": { "prompt_s": 0.39, "decode_s": 0.3196, "ttft_s": 0.40651, "first_answer_s": 0.714583, "total_s": 0.714734 }, "metrics": { "prompt_tps": 182.051282, "decode_tps": 137.67209 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "instruction", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R1", "phase": "sanity", "case_id": "R1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 79, "decode_tokens": 79 }, "timings": { "prompt_s": 2.3997, "decode_s": 0.6404, "ttft_s": 2.418636, "first_answer_s": 3.020719, "total_s": 3.047915 }, "metrics": { "prompt_tps": 1703.962995, "decode_tps": 123.3604 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R2", "phase": "sanity", "case_id": "R2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 4089, "cached_tokens": 0, "read_tokens": 4089, "output_tokens": 59, "decode_tokens": 59 }, "timings": { "prompt_s": 2.4058, "decode_s": 0.4686, "ttft_s": 2.423593, "first_answer_s": 2.822058, "total_s": 2.880385 }, "metrics": { "prompt_tps": 1699.642531, "decode_tps": 125.906957 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R3", "phase": "sanity", "case_id": "R3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16382, "cached_tokens": 0, "read_tokens": 16382, "output_tokens": 68, "decode_tokens": 68 }, "timings": { "prompt_s": 7.3172, "decode_s": 0.5111, "ttft_s": 7.346867, "first_answer_s": 7.788916, "total_s": 7.846141 }, "metrics": { "prompt_tps": 2238.834527, "decode_tps": 133.046371 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the exact expected text" } }, { "id": "sanity-R4", "phase": "sanity", "case_id": "R4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "stop", "error": null, "tokens": { "input_tokens": 16390, "cached_tokens": 0, "read_tokens": 16390, "output_tokens": 126, "decode_tokens": 126 }, "timings": { "prompt_s": 7.4424, "decode_s": 0.9913, "ttft_s": 7.472494, "first_answer_s": 8.260505, "total_s": 8.452904 }, "metrics": { "prompt_tps": 2202.246587, "decode_tps": 127.105821 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "retrieval", "passed": true, "reason": "the expected JSON value" } }, { "id": "sanity-T1", "phase": "sanity", "case_id": "T1", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 325, "cached_tokens": 0, "read_tokens": 325, "output_tokens": 64, "decode_tokens": 64 }, "timings": { "prompt_s": 1.2811, "decode_s": 0.499, "ttft_s": 1.29742, "first_answer_s": 1.636457, "total_s": 1.784789 }, "metrics": { "prompt_tps": 253.688237, "decode_tps": 128.256513 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T2", "phase": "sanity", "case_id": "T2", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 335, "cached_tokens": 0, "read_tokens": 335, "output_tokens": 92, "decode_tokens": 92 }, "timings": { "prompt_s": 0.9724, "decode_s": 0.7489, "ttft_s": 0.989003, "first_answer_s": 1.466015, "total_s": 1.727261 }, "metrics": { "prompt_tps": 344.508433, "decode_tps": 122.846842 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T3", "phase": "sanity", "case_id": "T3", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 352, "cached_tokens": 0, "read_tokens": 352, "output_tokens": 87, "decode_tokens": 87 }, "timings": { "prompt_s": 0.9077, "decode_s": 0.612, "ttft_s": 0.92355, "first_answer_s": 1.23857, "total_s": 1.523853 }, "metrics": { "prompt_tps": 387.793324, "decode_tps": 142.156863 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } }, { "id": "sanity-T4", "phase": "sanity", "case_id": "T4", "repetition": 1, "seed": 1, "max_output_tokens": 2048, "concurrency": 1, "round": null, "status": "completed", "finish_reason": "tool_calls", "error": null, "tokens": { "input_tokens": 339, "cached_tokens": 0, "read_tokens": 339, "output_tokens": 77, "decode_tokens": 77 }, "timings": { "prompt_s": 0.9077, "decode_s": 0.551, "ttft_s": 0.924561, "first_answer_s": 1.237569, "total_s": 1.464608 }, "metrics": { "prompt_tps": 373.471411, "decode_tps": 139.745917 }, "sources": { "input_tokens": "usage.prompt_tokens of the final stream chunk", "cached_tokens": "usage.prompt_tokens_details.cached_tokens", "read_tokens": "timings.prompt_n", "output_tokens": "usage.completion_tokens of the final stream chunk", "decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)", "engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)", "client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl" }, "first_delta_kind": "reasoning", "token_accounting_note": null, "evaluation": { "category": "tool_call", "passed": true, "reason": "the expected tool call" } } ], "summaries": [ { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 164.741121, "min": 163.412564, "max": 204.732784, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 122.744872, "min": 119.468909, "max": 128.236745, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 0.481843, "min": 0.394492, "max": 0.488926, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 1.113049, "min": 0.892493, "max": 1.709907, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "short", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 4.024395, "min": 2.497374, "max": 4.236185, "n": 3, "included_request_ids": [ "performance-short-r1", "performance-short-r2", "performance-short-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 1702.437401, "min": 1545.169693, "max": 1743.631252, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 111.595466, "min": 102.982883, "max": 126.454099, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 2.422599, "min": 2.363708, "max": 2.666856, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 5.488339, "min": 4.841742, "max": 6.134936, "n": 2, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2" ], "excluded": [ { "request_id": "performance-prompt-4k-r3", "reason": "first_answer_s is not available for this request" } ] }, { "phase": "performance", "case_id": "prompt-4k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 7.243141, "min": 6.460611, "max": 7.325637, "n": 3, "included_request_ids": [ "performance-prompt-4k-r1", "performance-prompt-4k-r2", "performance-prompt-4k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "prompt_tps", "unit": "tokens/s", "median": 2239.39543, "min": 2145.03567, "max": 2247.688805, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "decode_tps", "unit": "tokens/s", "median": 105.918616, "min": 99.947293, "max": 118.272118, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "ttft_s", "unit": "s", "median": 7.347428, "min": 7.315847, "max": 7.669504, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "first_answer_s", "unit": "s", "median": 10.695735, "min": 9.770124, "max": 11.621347, "n": 2, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2" ], "excluded": [ { "request_id": "performance-prompt-16k-r3", "reason": "first_answer_s is not available for this request" } ] }, { "phase": "performance", "case_id": "prompt-16k", "concurrency": 1, "metric": "total_s", "unit": "s", "median": 12.137885, "min": 11.988469, "max": 12.457752, "n": 3, "included_request_ids": [ "performance-prompt-16k-r1", "performance-prompt-16k-r2", "performance-prompt-16k-r3" ], "excluded": [] } ], "concurrency_rounds": [], "quality": { "passed": 12, "total": 12, "categories": { "instruction": { "passed": 4, "total": 4 }, "retrieval": { "passed": 4, "total": 4 }, "tool_call": { "passed": 4, "total": 4 } } }, "artifacts": [ { "kind": "outputs", "path": "outputs.jsonl", "sha256": "478f818041a3f774094ce94acb120db85063a6a6d536e0eecfd4e6619d2e31c7" }, { "kind": "config", "path": "config.json", "sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410" }, { "kind": "provenance", "path": "provenance/chat_template.jinja", "sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce" }, { "kind": "provenance", "path": "provenance/corpus-manifest.sha256", "sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407" }, { "kind": "provenance", "path": "provenance/corpus-build.json", "sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2" } ], "missing": { "/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)", "/run/optional_duration_s": "the optional concurrency stage was not selected", "/hardware/storage": "the drive holding the model files could not be identified", "/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background", "/software/build_options": "the server does not report build options; pass --build-options for a source build", "/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision", "/model/source_revision": "not given: pass --source-revision", "/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)", "/model/tokenizer_revision": "not given: pass --tokenizer-revision", "/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)", "/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config", "/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config", "/requests/0/round": "not applicable outside the concurrency stage", "/requests/0/error": "none: the request ended normally", "/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/0/evaluation": "not applicable outside the sanity stage", "/requests/1/round": "not applicable outside the concurrency stage", "/requests/1/error": "none: the request ended normally", "/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/1/evaluation": "not applicable outside the sanity stage", "/requests/2/round": "not applicable outside the concurrency stage", "/requests/2/error": "none: the request ended normally", "/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/2/evaluation": "not applicable outside the sanity stage", "/requests/3/round": "not applicable outside the concurrency stage", "/requests/3/error": "none: the request ended normally", "/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/3/evaluation": "not applicable outside the sanity stage", "/requests/4/round": "not applicable outside the concurrency stage", "/requests/4/error": "none: the request ended normally", "/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/4/evaluation": "not applicable outside the sanity stage", "/requests/5/round": "not applicable outside the concurrency stage", "/requests/5/error": "none: the request ended normally", "/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/5/evaluation": "not applicable outside the sanity stage", "/requests/6/round": "not applicable outside the concurrency stage", "/requests/6/error": "none: the request ended normally", "/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/6/evaluation": "not applicable outside the sanity stage", "/requests/7/round": "not applicable outside the concurrency stage", "/requests/7/error": "none: the request ended normally", "/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/7/evaluation": "not applicable outside the sanity stage", "/requests/8/round": "not applicable outside the concurrency stage", "/requests/8/error": "none: the request ended normally", "/requests/8/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)", "/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/8/evaluation": "not applicable outside the sanity stage", "/requests/9/round": "not applicable outside the concurrency stage", "/requests/9/error": "none: the request ended normally", "/requests/9/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)", "/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/9/evaluation": "not applicable outside the sanity stage", "/requests/10/round": "not applicable outside the concurrency stage", "/requests/10/error": "none: the request ended normally", "/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/11/round": "not applicable outside the concurrency stage", "/requests/11/error": "none: the request ended normally", "/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/12/round": "not applicable outside the concurrency stage", "/requests/12/error": "none: the request ended normally", "/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/13/round": "not applicable outside the concurrency stage", "/requests/13/error": "none: the request ended normally", "/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/14/round": "not applicable outside the concurrency stage", "/requests/14/error": "none: the request ended normally", "/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/15/round": "not applicable outside the concurrency stage", "/requests/15/error": "none: the request ended normally", "/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/16/round": "not applicable outside the concurrency stage", "/requests/16/error": "none: the request ended normally", "/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/17/round": "not applicable outside the concurrency stage", "/requests/17/error": "none: the request ended normally", "/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/18/round": "not applicable outside the concurrency stage", "/requests/18/error": "none: the request ended normally", "/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/19/round": "not applicable outside the concurrency stage", "/requests/19/error": "none: the request ended normally", "/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/20/round": "not applicable outside the concurrency stage", "/requests/20/error": "none: the request ended normally", "/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists", "/requests/21/round": "not applicable outside the concurrency stage", "/requests/21/error": "none: the request ended normally", "/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists" }, "extensions": { "suite": "standard-v1", "runner": "strata_cli.bench_standard", "status_reasons": [ "no model file could be hashed" ], "started_engine": false, "server_requests_before_run": 0, "other_engine_options": {}, "request_checks": { "warmup": { "max_shared_prefix_tokens": 47, "frozen_input_tokens": 75, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r1": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r1": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r1": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r2": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r2": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r2": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-short-r3": { "max_shared_prefix_tokens": 50, "frozen_input_tokens": 77, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-4k-r3": { "max_shared_prefix_tokens": 54, "frozen_input_tokens": 4093, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "performance-prompt-16k-r3": { "max_shared_prefix_tokens": 55, "frozen_input_tokens": 16387, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 82, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 90, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 85, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-I4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 71, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R1": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R2": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 4089, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R3": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16382, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-R4": { "max_shared_prefix_tokens": 49, "frozen_input_tokens": 16390, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T1": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 325, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T2": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 335, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T3": { "max_shared_prefix_tokens": 61, "frozen_input_tokens": 352, "unexpected_payload_reuse": false, "length_outside_tolerance": false }, "sanity-T4": { "max_shared_prefix_tokens": 62, "frozen_input_tokens": 339, "unexpected_payload_reuse": false, "length_outside_tolerance": false } }, "optional_stage_skipped": false, "api_key_sent": true } }