Jason WittyandClaude Opus 5.5
d525f315f4
standard-v1 on the idle server: ~2,900 prompt tok/s at 16K, decode ~110-131. --adapt-async 1 neutral, --pipeline-windows 2 slower decode, --prefill auto:32768 22% slower prompt at 16K. Setup defaults kept and the original config restored. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1681 lines
56 KiB
JSON
1681 lines
56 KiB
JSON
{
|
|
"format": "strata-standard-benchmark",
|
|
"schema_version": 1,
|
|
"protocol": {
|
|
"id": "strata-qwen38-thinking-v1-draft.1",
|
|
"git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"profile": "qwen3.8-flash-next-thinking",
|
|
"corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407",
|
|
"model_card_revision": null
|
|
},
|
|
"run": {
|
|
"id": "20261026T235154Z-87dfe6da",
|
|
"started_at": "2026-10-26T23:51:54Z",
|
|
"status": "nonconforming",
|
|
"required_duration_s": 94.201,
|
|
"optional_duration_s": null,
|
|
"optional_concurrency": false,
|
|
"command": "<user> bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes arm restart-baseline: [] --quantization IQ3_S"
|
|
},
|
|
"hardware": {
|
|
"gpus": [
|
|
{
|
|
"device_index": 0,
|
|
"name": "NVIDIA GeForce RTX 3090",
|
|
"memory_bytes": 25769803776,
|
|
"memory_kind": "dedicated",
|
|
"power_limit_w": 250.0
|
|
}
|
|
],
|
|
"cpu": "Intel(R) Core(TM) i9-14900KS",
|
|
"ram_bytes": 118089846784,
|
|
"storage": null,
|
|
"background_workloads": null
|
|
},
|
|
"software": {
|
|
"os": {
|
|
"name": "NixOS 26.05 (Yarara)",
|
|
"version": "6.18.55"
|
|
},
|
|
"drivers": [
|
|
{
|
|
"name": "gpu-driver",
|
|
"version": "595.71.05"
|
|
}
|
|
],
|
|
"runtime": [
|
|
{
|
|
"name": "cuda",
|
|
"version": "13.2"
|
|
}
|
|
],
|
|
"strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"engine_version": "0.1.51",
|
|
"build_kind": "source",
|
|
"runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"build_options": null
|
|
},
|
|
"model": {
|
|
"id": "qwen3.8-flash-next-iq3_s",
|
|
"revision": null,
|
|
"source_revision": null,
|
|
"quantization": "IQ3_S",
|
|
"files": null,
|
|
"tokenizer_revision": null,
|
|
"custom_pack": null
|
|
},
|
|
"configuration": {
|
|
"track": "custom",
|
|
"track_notes": "arm restart-baseline: []",
|
|
"context_tokens": 131072,
|
|
"server_capacity": 1,
|
|
"kv_cache": null,
|
|
"expert_cache": null,
|
|
"offloading": null,
|
|
"prefill": null,
|
|
"mtp": null,
|
|
"calibration": null,
|
|
"experimental_options": null,
|
|
"template": {
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
"sampling": {
|
|
"temperature": 1.0,
|
|
"top_p": 0.95,
|
|
"top_k": 20,
|
|
"min_p": 0.0,
|
|
"presence_penalty": 0.0,
|
|
"repetition_penalty": 1.0,
|
|
"reasoning_effort": "xhigh"
|
|
},
|
|
"chat_template_kwargs": {
|
|
"enable_thinking": true,
|
|
"preserve_thinking": true
|
|
}
|
|
},
|
|
"request_counts": {
|
|
"warmup": {
|
|
"planned": 1,
|
|
"attempted": 1,
|
|
"completed": 0,
|
|
"truncated": 1,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"performance": {
|
|
"planned": 9,
|
|
"attempted": 9,
|
|
"completed": 3,
|
|
"truncated": 6,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"sanity": {
|
|
"planned": 12,
|
|
"attempted": 12,
|
|
"completed": 12,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"concurrency": {
|
|
"planned": 0,
|
|
"attempted": 0,
|
|
"completed": 0,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
}
|
|
},
|
|
"requests": [
|
|
{
|
|
"id": "warmup",
|
|
"phase": "warmup",
|
|
"case_id": "warmup",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 128,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 75,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 75,
|
|
"output_tokens": 128,
|
|
"decode_tokens": 128
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4038,
|
|
"decode_s": 1.1406,
|
|
"ttft_s": 0.430244,
|
|
"first_answer_s": 1.045328,
|
|
"total_s": 1.547782
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 185.735513,
|
|
"decode_tps": 112.221638
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r1",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 293,
|
|
"decode_tokens": 293
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3694,
|
|
"decode_s": 2.7519,
|
|
"ttft_s": 0.385169,
|
|
"first_answer_s": 1.12217,
|
|
"total_s": 3.125534
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 208.446129,
|
|
"decode_tps": 106.471892
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.6568,
|
|
"decode_s": 5.4607,
|
|
"ttft_s": 2.674452,
|
|
"first_answer_s": null,
|
|
"total_s": 8.122705
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1540.575128,
|
|
"decode_tps": 93.760873
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.7738,
|
|
"decode_s": 4.4808,
|
|
"ttft_s": 5.794524,
|
|
"first_answer_s": 8.833015,
|
|
"total_s": 10.262911
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2838.165506,
|
|
"decode_tps": 114.26531
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r2",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 463,
|
|
"decode_tokens": 463
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4554,
|
|
"decode_s": 3.488,
|
|
"ttft_s": 0.470504,
|
|
"first_answer_s": 0.993512,
|
|
"total_s": 3.946793
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 169.082126,
|
|
"decode_tps": 132.740826
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3905,
|
|
"decode_s": 4.6494,
|
|
"ttft_s": 2.407603,
|
|
"first_answer_s": null,
|
|
"total_s": 7.046894
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1712.194102,
|
|
"decode_tps": 110.121736
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.6881,
|
|
"decode_s": 4.1073,
|
|
"ttft_s": 5.710381,
|
|
"first_answer_s": 7.454714,
|
|
"total_s": 9.805629
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2880.926847,
|
|
"decode_tps": 124.6561
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r3",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 429,
|
|
"decode_tokens": 429
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4544,
|
|
"decode_s": 3.5034,
|
|
"ttft_s": 0.470604,
|
|
"first_answer_s": 1.816585,
|
|
"total_s": 3.962074
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 169.454225,
|
|
"decode_tps": 122.452475
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3388,
|
|
"decode_s": 3.9382,
|
|
"ttft_s": 2.357393,
|
|
"first_answer_s": 4.331564,
|
|
"total_s": 6.285412
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1750.042757,
|
|
"decode_tps": 130.008633
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.6884,
|
|
"decode_s": 4.3914,
|
|
"ttft_s": 5.708805,
|
|
"first_answer_s": 8.557563,
|
|
"total_s": 10.089332
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2880.77491,
|
|
"decode_tps": 116.59152
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "sanity-I1",
|
|
"phase": "sanity",
|
|
"case_id": "I1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 82,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 82,
|
|
"output_tokens": 74,
|
|
"decode_tokens": 74
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4793,
|
|
"decode_s": 0.5857,
|
|
"ttft_s": 0.49328,
|
|
"first_answer_s": 0.952282,
|
|
"total_s": 1.067794
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 171.082829,
|
|
"decode_tps": 126.344545
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I2",
|
|
"phase": "sanity",
|
|
"case_id": "I2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 90,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 90,
|
|
"output_tokens": 60,
|
|
"decode_tokens": 60
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3707,
|
|
"decode_s": 0.4525,
|
|
"ttft_s": 0.385405,
|
|
"first_answer_s": 0.723455,
|
|
"total_s": 0.827721
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 242.783922,
|
|
"decode_tps": 132.596685
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I3",
|
|
"phase": "sanity",
|
|
"case_id": "I3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 85,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 85,
|
|
"output_tokens": 94,
|
|
"decode_tokens": 94
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3641,
|
|
"decode_s": 0.6838,
|
|
"ttft_s": 0.378431,
|
|
"first_answer_s": 0.951449,
|
|
"total_s": 1.05066
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 233.452348,
|
|
"decode_tps": 137.467096
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I4",
|
|
"phase": "sanity",
|
|
"case_id": "I4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 71,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 71,
|
|
"output_tokens": 42,
|
|
"decode_tokens": 42
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.374,
|
|
"decode_s": 0.3441,
|
|
"ttft_s": 0.390642,
|
|
"first_answer_s": 0.684641,
|
|
"total_s": 0.722948
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 189.839572,
|
|
"decode_tps": 122.057541
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R1",
|
|
"phase": "sanity",
|
|
"case_id": "R1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 70,
|
|
"decode_tokens": 70
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3876,
|
|
"decode_s": 0.5892,
|
|
"ttft_s": 2.407549,
|
|
"first_answer_s": 2.958609,
|
|
"total_s": 2.985849
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1712.598425,
|
|
"decode_tps": 118.80516
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R2",
|
|
"phase": "sanity",
|
|
"case_id": "R2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 64,
|
|
"decode_tokens": 64
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3926,
|
|
"decode_s": 0.5141,
|
|
"ttft_s": 2.409517,
|
|
"first_answer_s": 2.843578,
|
|
"total_s": 2.911854
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1709.019477,
|
|
"decode_tps": 124.489399
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R3",
|
|
"phase": "sanity",
|
|
"case_id": "R3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16382,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16382,
|
|
"output_tokens": 69,
|
|
"decode_tokens": 69
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.6407,
|
|
"decode_s": 0.55,
|
|
"ttft_s": 5.67147,
|
|
"first_answer_s": 6.153456,
|
|
"total_s": 6.209676
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2904.249473,
|
|
"decode_tps": 125.454545
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R4",
|
|
"phase": "sanity",
|
|
"case_id": "R4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16390,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16390,
|
|
"output_tokens": 223,
|
|
"decode_tokens": 223
|
|
},
|
|
"timings": {
|
|
"prompt_s": 5.8144,
|
|
"decode_s": 1.734,
|
|
"ttft_s": 5.844614,
|
|
"first_answer_s": 7.42764,
|
|
"total_s": 7.566945
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2818.863511,
|
|
"decode_tps": 128.604383
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T1",
|
|
"phase": "sanity",
|
|
"case_id": "T1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 325,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 325,
|
|
"output_tokens": 64,
|
|
"decode_tokens": 64
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.3114,
|
|
"decode_s": 0.5058,
|
|
"ttft_s": 1.329494,
|
|
"first_answer_s": 1.674545,
|
|
"total_s": 1.82352
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 247.82675,
|
|
"decode_tps": 126.532226
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T2",
|
|
"phase": "sanity",
|
|
"case_id": "T2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 335,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 335,
|
|
"output_tokens": 91,
|
|
"decode_tokens": 91
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9674,
|
|
"decode_s": 0.7432,
|
|
"ttft_s": 0.98384,
|
|
"first_answer_s": 1.460856,
|
|
"total_s": 1.715904
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 346.289022,
|
|
"decode_tps": 122.443488
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T3",
|
|
"phase": "sanity",
|
|
"case_id": "T3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 352,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 352,
|
|
"output_tokens": 87,
|
|
"decode_tokens": 87
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.912,
|
|
"decode_s": 0.6195,
|
|
"ttft_s": 0.928926,
|
|
"first_answer_s": 1.246938,
|
|
"total_s": 1.536986
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 385.964912,
|
|
"decode_tps": 140.435835
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T4",
|
|
"phase": "sanity",
|
|
"case_id": "T4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 339,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 339,
|
|
"output_tokens": 90,
|
|
"decode_tokens": 90
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9106,
|
|
"decode_s": 0.6629,
|
|
"ttft_s": 0.926929,
|
|
"first_answer_s": 1.37395,
|
|
"total_s": 1.579982
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 372.282012,
|
|
"decode_tps": 135.767084
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
}
|
|
],
|
|
"summaries": [
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 169.454225,
|
|
"min": 169.082126,
|
|
"max": 208.446129,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 122.452475,
|
|
"min": 106.471892,
|
|
"max": 132.740826,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 0.470504,
|
|
"min": 0.385169,
|
|
"max": 0.470604,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 1.12217,
|
|
"min": 0.993512,
|
|
"max": 1.816585,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 3.946793,
|
|
"min": 3.125534,
|
|
"max": 3.962074,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 1712.194102,
|
|
"min": 1540.575128,
|
|
"max": 1750.042757,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 110.121736,
|
|
"min": 93.760873,
|
|
"max": 130.008633,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 2.407603,
|
|
"min": 2.357393,
|
|
"max": 2.674452,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 4.331564,
|
|
"min": 4.331564,
|
|
"max": 4.331564,
|
|
"n": 1,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": [
|
|
{
|
|
"request_id": "performance-prompt-4k-r1",
|
|
"reason": "first_answer_s is not available for this request"
|
|
},
|
|
{
|
|
"request_id": "performance-prompt-4k-r2",
|
|
"reason": "first_answer_s is not available for this request"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 7.046894,
|
|
"min": 6.285412,
|
|
"max": 8.122705,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 2880.77491,
|
|
"min": 2838.165506,
|
|
"max": 2880.926847,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 116.59152,
|
|
"min": 114.26531,
|
|
"max": 124.6561,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 5.710381,
|
|
"min": 5.708805,
|
|
"max": 5.794524,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 8.557563,
|
|
"min": 7.454714,
|
|
"max": 8.833015,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 10.089332,
|
|
"min": 9.805629,
|
|
"max": 10.262911,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
}
|
|
],
|
|
"concurrency_rounds": [],
|
|
"quality": {
|
|
"passed": 12,
|
|
"total": 12,
|
|
"categories": {
|
|
"instruction": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"retrieval": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"tool_call": {
|
|
"passed": 4,
|
|
"total": 4
|
|
}
|
|
}
|
|
},
|
|
"artifacts": [
|
|
{
|
|
"kind": "outputs",
|
|
"path": "outputs.jsonl",
|
|
"sha256": "15861985448097f12a664fe1c0a37dcac5c183c3fc7b73c2de75ffe8d781aa0a"
|
|
},
|
|
{
|
|
"kind": "config",
|
|
"path": "config.json",
|
|
"sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-manifest.sha256",
|
|
"sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-build.json",
|
|
"sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2"
|
|
}
|
|
],
|
|
"missing": {
|
|
"/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)",
|
|
"/run/optional_duration_s": "the optional concurrency stage was not selected",
|
|
"/hardware/storage": "the drive holding the model files could not be identified",
|
|
"/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background",
|
|
"/software/build_options": "the server does not report build options; pass --build-options for a source build",
|
|
"/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",
|
|
"/model/source_revision": "not given: pass --source-revision",
|
|
"/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)",
|
|
"/model/tokenizer_revision": "not given: pass --tokenizer-revision",
|
|
"/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)",
|
|
"/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/requests/0/round": "not applicable outside the concurrency stage",
|
|
"/requests/0/error": "none: the request ended normally",
|
|
"/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/0/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/1/round": "not applicable outside the concurrency stage",
|
|
"/requests/1/error": "none: the request ended normally",
|
|
"/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/1/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/2/round": "not applicable outside the concurrency stage",
|
|
"/requests/2/error": "none: the request ended normally",
|
|
"/requests/2/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/2/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/3/round": "not applicable outside the concurrency stage",
|
|
"/requests/3/error": "none: the request ended normally",
|
|
"/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/3/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/4/round": "not applicable outside the concurrency stage",
|
|
"/requests/4/error": "none: the request ended normally",
|
|
"/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/4/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/5/round": "not applicable outside the concurrency stage",
|
|
"/requests/5/error": "none: the request ended normally",
|
|
"/requests/5/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/5/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/6/round": "not applicable outside the concurrency stage",
|
|
"/requests/6/error": "none: the request ended normally",
|
|
"/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/6/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/7/round": "not applicable outside the concurrency stage",
|
|
"/requests/7/error": "none: the request ended normally",
|
|
"/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/7/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/8/round": "not applicable outside the concurrency stage",
|
|
"/requests/8/error": "none: the request ended normally",
|
|
"/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/8/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/9/round": "not applicable outside the concurrency stage",
|
|
"/requests/9/error": "none: the request ended normally",
|
|
"/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/9/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/10/round": "not applicable outside the concurrency stage",
|
|
"/requests/10/error": "none: the request ended normally",
|
|
"/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/11/round": "not applicable outside the concurrency stage",
|
|
"/requests/11/error": "none: the request ended normally",
|
|
"/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/12/round": "not applicable outside the concurrency stage",
|
|
"/requests/12/error": "none: the request ended normally",
|
|
"/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/13/round": "not applicable outside the concurrency stage",
|
|
"/requests/13/error": "none: the request ended normally",
|
|
"/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/14/round": "not applicable outside the concurrency stage",
|
|
"/requests/14/error": "none: the request ended normally",
|
|
"/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/15/round": "not applicable outside the concurrency stage",
|
|
"/requests/15/error": "none: the request ended normally",
|
|
"/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/16/round": "not applicable outside the concurrency stage",
|
|
"/requests/16/error": "none: the request ended normally",
|
|
"/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/17/round": "not applicable outside the concurrency stage",
|
|
"/requests/17/error": "none: the request ended normally",
|
|
"/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/18/round": "not applicable outside the concurrency stage",
|
|
"/requests/18/error": "none: the request ended normally",
|
|
"/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/19/round": "not applicable outside the concurrency stage",
|
|
"/requests/19/error": "none: the request ended normally",
|
|
"/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/20/round": "not applicable outside the concurrency stage",
|
|
"/requests/20/error": "none: the request ended normally",
|
|
"/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/21/round": "not applicable outside the concurrency stage",
|
|
"/requests/21/error": "none: the request ended normally",
|
|
"/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists"
|
|
},
|
|
"extensions": {
|
|
"suite": "standard-v1",
|
|
"runner": "strata_cli.bench_standard",
|
|
"status_reasons": [
|
|
"no model file could be hashed"
|
|
],
|
|
"started_engine": false,
|
|
"server_requests_before_run": 0,
|
|
"other_engine_options": {},
|
|
"request_checks": {
|
|
"warmup": {
|
|
"max_shared_prefix_tokens": 47,
|
|
"frozen_input_tokens": 75,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r1": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r1": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r1": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r2": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r2": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r2": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r3": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r3": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r3": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 82,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 90,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 85,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 71,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16382,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16390,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T1": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 325,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T2": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 335,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T3": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 352,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T4": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 339,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
}
|
|
},
|
|
"optional_stage_skipped": false,
|
|
"api_key_sent": true
|
|
}
|
|
}
|