jason/lovey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
lovey/benchmarks/standard/2026-10-26-standard-rtx-3090-0f07e24d/results.json
T
Jason WittyandClaude Opus 5.5 d525f315f4 Add standard benchmark baseline and tuning arms 
standard-v1 on the idle server: ~2,900 prompt tok/s at 16K, decode
~110-131. --adapt-async 1 neutral, --pipeline-windows 2 slower decode,
--prefill auto:32768 22% slower prompt at 16K. Setup defaults kept and
the original config restored.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-26 17:11:43 -07:00

2481 lines
83 KiB
JSON
Raw Permalink Blame History

{
"format": "strata-standard-benchmark",
"schema_version": 1,
"protocol": {
"id": "strata-qwen38-thinking-v1-draft.1",
"git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
"profile": "qwen3.8-flash-next-thinking",
"corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407",
"model_card_revision": null
},
"run": {
"id": "20261026T234702Z-0f07e24d",
"started_at": "2026-10-26T23:47:02Z",
"status": "complete",
"required_duration_s": 91.197,
"optional_duration_s": 47.974,
"optional_concurrency": true,
"command": "<user> bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --optional-concurrency --track default --track-notes lovey 2x3090, 250 W cap, gpu-unlock, setup defaults + --max-context 131072 --quantization IQ3_S"
},
"hardware": {
"gpus": [
{
"device_index": 0,
"name": "NVIDIA GeForce RTX 3090",
"memory_bytes": 25769803776,
"memory_kind": "dedicated",
"power_limit_w": 250.0
}
],
"cpu": "Intel(R) Core(TM) i9-14900KS",
"ram_bytes": 118089846784,
"storage": null,
"background_workloads": null
},
"software": {
"os": {
"name": "NixOS 26.05 (Yarara)",
"version": "6.18.55"
},
"drivers": [
{
"name": "gpu-driver",
"version": "595.71.05"
}
],
"runtime": [
{
"name": "cuda",
"version": "13.2"
}
],
"strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
"engine_version": "0.1.51",
"build_kind": "source",
"runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
"build_options": null
},
"model": {
"id": "qwen3.8-flash-next-iq3_s",
"revision": null,
"source_revision": null,
"quantization": "IQ3_S",
"files": [
{
"path": "Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",
"sha256": "4c1eb2ceb4915e1192f4f386021897bde56a97f40a0bb78bb86465e0f7d2aca3"
},
{
"path": "Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",
"sha256": "316b46f3a2dbd68c900f43136ab9449f9dcc3725dfd8c794847c204bc161e113"
}
],
"tokenizer_revision": null,
"custom_pack": null
},
"configuration": {
"track": "default",
"track_notes": "lovey 2x3090, 250 W cap, gpu-unlock, setup defaults + --max-context 131072",
"context_tokens": 131072,
"server_capacity": 1,
"kv_cache": null,
"expert_cache": null,
"offloading": null,
"prefill": null,
"mtp": null,
"calibration": null,
"experimental_options": null,
"template": {
"path": "provenance/chat_template.jinja",
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
},
"sampling": {
"temperature": 1.0,
"top_p": 0.95,
"top_k": 20,
"min_p": 0.0,
"presence_penalty": 0.0,
"repetition_penalty": 1.0,
"reasoning_effort": "xhigh"
},
"chat_template_kwargs": {
"enable_thinking": true,
"preserve_thinking": true
}
},
"request_counts": {
"warmup": {
"planned": 1,
"attempted": 1,
"completed": 0,
"truncated": 1,
"failed": 0,
"skipped": 0
},
"performance": {
"planned": 9,
"attempted": 9,
"completed": 2,
"truncated": 7,
"failed": 0,
"skipped": 0
},
"sanity": {
"planned": 12,
"attempted": 12,
"completed": 12,
"truncated": 0,
"failed": 0,
"skipped": 0
},
"concurrency": {
"planned": 12,
"attempted": 12,
"completed": 3,
"truncated": 9,
"failed": 0,
"skipped": 0
}
},
"requests": [
{
"id": "warmup",
"phase": "warmup",
"case_id": "warmup",
"repetition": 1,
"seed": 1,
"max_output_tokens": 128,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 75,
"cached_tokens": 0,
"read_tokens": 75,
"output_tokens": 128,
"decode_tokens": 128
},
"timings": {
"prompt_s": 0.5378,
"decode_s": 1.208,
"ttft_s": 0.561074,
"first_answer_s": 1.308051,
"total_s": 1.750101
},
"metrics": {
"prompt_tps": 139.457047,
"decode_tps": 105.960265
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-short-r1",
"phase": "performance",
"case_id": "short",
"repetition": 1,
"seed": 1,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 77,
"cached_tokens": 0,
"read_tokens": 77,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.3929,
"decode_s": 3.7771,
"ttft_s": 0.409799,
"first_answer_s": 0.89164,
"total_s": 4.174695
},
"metrics": {
"prompt_tps": 195.978621,
"decode_tps": 135.553732
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-prompt-4k-r1",
"phase": "performance",
"case_id": "prompt-4k",
"repetition": 1,
"seed": 1,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 4093,
"cached_tokens": 0,
"read_tokens": 4093,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 2.307,
"decode_s": 4.5458,
"ttft_s": 2.326971,
"first_answer_s": null,
"total_s": 6.861012
},
"metrics": {
"prompt_tps": 1774.165583,
"decode_tps": 112.63144
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-prompt-16k-r1",
"phase": "performance",
"case_id": "prompt-16k",
"repetition": 1,
"seed": 1,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 16387,
"cached_tokens": 0,
"read_tokens": 16387,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 5.5643,
"decode_s": 4.1108,
"ttft_s": 5.591829,
"first_answer_s": 8.142807,
"total_s": 9.690426
},
"metrics": {
"prompt_tps": 2945.024531,
"decode_tps": 124.549966
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-short-r2",
"phase": "performance",
"case_id": "short",
"repetition": 2,
"seed": 2,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 77,
"cached_tokens": 0,
"read_tokens": 77,
"output_tokens": 338,
"decode_tokens": 338
},
"timings": {
"prompt_s": 0.3396,
"decode_s": 2.7229,
"ttft_s": 0.353313,
"first_answer_s": 1.171217,
"total_s": 3.065101
},
"metrics": {
"prompt_tps": 226.737338,
"decode_tps": 124.132359
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-prompt-4k-r2",
"phase": "performance",
"case_id": "prompt-4k",
"repetition": 2,
"seed": 2,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 4093,
"cached_tokens": 0,
"read_tokens": 4093,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 2.302,
"decode_s": 3.9869,
"ttft_s": 2.320082,
"first_answer_s": 4.956982,
"total_s": 6.296036
},
"metrics": {
"prompt_tps": 1778.019114,
"decode_tps": 128.420577
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-prompt-16k-r2",
"phase": "performance",
"case_id": "prompt-16k",
"repetition": 2,
"seed": 2,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 16387,
"cached_tokens": 0,
"read_tokens": 16387,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 5.5777,
"decode_s": 3.8954,
"ttft_s": 5.608843,
"first_answer_s": 7.790823,
"total_s": 9.493855
},
"metrics": {
"prompt_tps": 2937.949334,
"decode_tps": 131.43708
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-short-r3",
"phase": "performance",
"case_id": "short",
"repetition": 3,
"seed": 3,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 77,
"cached_tokens": 0,
"read_tokens": 77,
"output_tokens": 510,
"decode_tokens": 510
},
"timings": {
"prompt_s": 0.3254,
"decode_s": 3.9007,
"ttft_s": 0.341909,
"first_answer_s": 2.546901,
"total_s": 4.23085
},
"metrics": {
"prompt_tps": 236.631838,
"decode_tps": 130.745764
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-prompt-4k-r3",
"phase": "performance",
"case_id": "prompt-4k",
"repetition": 3,
"seed": 3,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 4093,
"cached_tokens": 0,
"read_tokens": 4093,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 2.3045,
"decode_s": 4.1569,
"ttft_s": 2.323905,
"first_answer_s": 5.64188,
"total_s": 6.469928
},
"metrics": {
"prompt_tps": 1776.090258,
"decode_tps": 123.168707
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "performance-prompt-16k-r3",
"phase": "performance",
"case_id": "prompt-16k",
"repetition": 3,
"seed": 3,
"max_output_tokens": 512,
"concurrency": 1,
"round": null,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 16387,
"cached_tokens": 0,
"read_tokens": 16387,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 5.5843,
"decode_s": 4.452,
"ttft_s": 5.613231,
"first_answer_s": null,
"total_s": 10.055156
},
"metrics": {
"prompt_tps": 2934.477016,
"decode_tps": 115.004492
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "sanity-I1",
"phase": "sanity",
"case_id": "I1",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 82,
"cached_tokens": 0,
"read_tokens": 82,
"output_tokens": 68,
"decode_tokens": 68
},
"timings": {
"prompt_s": 0.3304,
"decode_s": 0.5127,
"ttft_s": 0.346933,
"first_answer_s": 0.741939,
"total_s": 0.84901
},
"metrics": {
"prompt_tps": 248.184019,
"decode_tps": 132.631168
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "instruction",
"passed": true,
"reason": "the expected JSON value"
}
},
{
"id": "sanity-I2",
"phase": "sanity",
"case_id": "I2",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 90,
"cached_tokens": 0,
"read_tokens": 90,
"output_tokens": 60,
"decode_tokens": 60
},
"timings": {
"prompt_s": 0.3295,
"decode_s": 0.3899,
"ttft_s": 0.345914,
"first_answer_s": 0.634915,
"total_s": 0.725301
},
"metrics": {
"prompt_tps": 273.141123,
"decode_tps": 153.885612
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "instruction",
"passed": true,
"reason": "the expected JSON value"
}
},
{
"id": "sanity-I3",
"phase": "sanity",
"case_id": "I3",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 85,
"cached_tokens": 0,
"read_tokens": 85,
"output_tokens": 77,
"decode_tokens": 77
},
"timings": {
"prompt_s": 0.3485,
"decode_s": 0.4826,
"ttft_s": 0.365442,
"first_answer_s": 0.756434,
"total_s": 0.835819
},
"metrics": {
"prompt_tps": 243.902439,
"decode_tps": 159.552424
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "instruction",
"passed": true,
"reason": "the expected JSON value"
}
},
{
"id": "sanity-I4",
"phase": "sanity",
"case_id": "I4",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 71,
"cached_tokens": 0,
"read_tokens": 71,
"output_tokens": 44,
"decode_tokens": 44
},
"timings": {
"prompt_s": 0.2982,
"decode_s": 0.309,
"ttft_s": 0.315554,
"first_answer_s": 0.614546,
"total_s": 0.614778
},
"metrics": {
"prompt_tps": 238.095238,
"decode_tps": 142.394822
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "instruction",
"passed": true,
"reason": "the exact expected text"
}
},
{
"id": "sanity-R1",
"phase": "sanity",
"case_id": "R1",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 4089,
"cached_tokens": 0,
"read_tokens": 4089,
"output_tokens": 72,
"decode_tokens": 72
},
"timings": {
"prompt_s": 2.3058,
"decode_s": 0.5465,
"ttft_s": 2.324606,
"first_answer_s": 2.804535,
"total_s": 2.859877
},
"metrics": {
"prompt_tps": 1773.35415,
"decode_tps": 131.747484
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "retrieval",
"passed": true,
"reason": "the exact expected text"
}
},
{
"id": "sanity-R2",
"phase": "sanity",
"case_id": "R2",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 4089,
"cached_tokens": 0,
"read_tokens": 4089,
"output_tokens": 59,
"decode_tokens": 59
},
"timings": {
"prompt_s": 2.3156,
"decode_s": 0.4081,
"ttft_s": 2.335515,
"first_answer_s": 2.678503,
"total_s": 2.732544
},
"metrics": {
"prompt_tps": 1765.849024,
"decode_tps": 144.572409
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "retrieval",
"passed": true,
"reason": "the exact expected text"
}
},
{
"id": "sanity-R3",
"phase": "sanity",
"case_id": "R3",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 16382,
"cached_tokens": 0,
"read_tokens": 16382,
"output_tokens": 63,
"decode_tokens": 63
},
"timings": {
"prompt_s": 5.5868,
"decode_s": 0.4552,
"ttft_s": 5.615875,
"first_answer_s": 6.008867,
"total_s": 6.059926
},
"metrics": {
"prompt_tps": 2932.26892,
"decode_tps": 138.400703
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "retrieval",
"passed": true,
"reason": "the exact expected text"
}
},
{
"id": "sanity-R4",
"phase": "sanity",
"case_id": "R4",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 16390,
"cached_tokens": 0,
"read_tokens": 16390,
"output_tokens": 382,
"decode_tokens": 382
},
"timings": {
"prompt_s": 5.6744,
"decode_s": 2.9063,
"ttft_s": 5.703869,
"first_answer_s": 8.408836,
"total_s": 8.598088
},
"metrics": {
"prompt_tps": 2888.41111,
"decode_tps": 131.438599
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "retrieval",
"passed": true,
"reason": "the expected JSON value"
}
},
{
"id": "sanity-T1",
"phase": "sanity",
"case_id": "T1",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "tool_calls",
"error": null,
"tokens": {
"input_tokens": 325,
"cached_tokens": 0,
"read_tokens": 325,
"output_tokens": 77,
"decode_tokens": 77
},
"timings": {
"prompt_s": 0.8607,
"decode_s": 0.4764,
"ttft_s": 0.876618,
"first_answer_s": 1.20064,
"total_s": 1.342668
},
"metrics": {
"prompt_tps": 377.599628,
"decode_tps": 161.628883
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "tool_call",
"passed": true,
"reason": "the expected tool call"
}
},
{
"id": "sanity-T2",
"phase": "sanity",
"case_id": "T2",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "tool_calls",
"error": null,
"tokens": {
"input_tokens": 335,
"cached_tokens": 0,
"read_tokens": 335,
"output_tokens": 91,
"decode_tokens": 91
},
"timings": {
"prompt_s": 0.8719,
"decode_s": 0.691,
"ttft_s": 0.888903,
"first_answer_s": 1.368934,
"total_s": 1.569968
},
"metrics": {
"prompt_tps": 384.218374,
"decode_tps": 131.693198
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "tool_call",
"passed": true,
"reason": "the expected tool call"
}
},
{
"id": "sanity-T3",
"phase": "sanity",
"case_id": "T3",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "tool_calls",
"error": null,
"tokens": {
"input_tokens": 352,
"cached_tokens": 0,
"read_tokens": 352,
"output_tokens": 87,
"decode_tokens": 87
},
"timings": {
"prompt_s": 0.8466,
"decode_s": 0.609,
"ttft_s": 0.863912,
"first_answer_s": 1.190931,
"total_s": 1.461985
},
"metrics": {
"prompt_tps": 415.78077,
"decode_tps": 142.857143
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "tool_call",
"passed": true,
"reason": "the expected tool call"
}
},
{
"id": "sanity-T4",
"phase": "sanity",
"case_id": "T4",
"repetition": 1,
"seed": 1,
"max_output_tokens": 2048,
"concurrency": 1,
"round": null,
"status": "completed",
"finish_reason": "tool_calls",
"error": null,
"tokens": {
"input_tokens": 339,
"cached_tokens": 0,
"read_tokens": 339,
"output_tokens": 90,
"decode_tokens": 90
},
"timings": {
"prompt_s": 0.849,
"decode_s": 0.5909,
"ttft_s": 0.866735,
"first_answer_s": 1.25176,
"total_s": 1.446792
},
"metrics": {
"prompt_tps": 399.293286,
"decode_tps": 152.310036
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": {
"category": "tool_call",
"passed": true,
"reason": "the expected tool call"
}
},
{
"id": "concurrency-r1-c1",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 1,
"seed": 1,
"max_output_tokens": 512,
"concurrency": 4,
"round": 1,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 395,
"decode_tokens": 395
},
"timings": {
"prompt_s": 0.2999,
"decode_s": 2.8552,
"ttft_s": 8.740826,
"first_answer_s": 9.478704,
"total_s": 11.584859
},
"metrics": {
"prompt_tps": 260.086696,
"decode_tps": 138.344074
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r1-c2",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 1,
"seed": 2,
"max_output_tokens": 512,
"concurrency": 4,
"round": 1,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 458,
"decode_tokens": 458
},
"timings": {
"prompt_s": 0.3293,
"decode_s": 3.372,
"ttft_s": 0.345774,
"first_answer_s": 0.889766,
"total_s": 3.706825
},
"metrics": {
"prompt_tps": 236.86608,
"decode_tps": 135.824437
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r1-c3",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 1,
"seed": 3,
"max_output_tokens": 512,
"concurrency": 4,
"round": 1,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.3092,
"decode_s": 4.4084,
"ttft_s": 4.030741,
"first_answer_s": 7.659896,
"total_s": 8.42778
},
"metrics": {
"prompt_tps": 252.263907,
"decode_tps": 116.141911
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r1-c4",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 1,
"seed": 4,
"max_output_tokens": 512,
"concurrency": 4,
"round": 1,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.3034,
"decode_s": 4.3636,
"ttft_s": 11.901639,
"first_answer_s": 15.534626,
"total_s": 16.253254
},
"metrics": {
"prompt_tps": 257.086355,
"decode_tps": 117.334311
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r2-c1",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 2,
"seed": 5,
"max_output_tokens": 512,
"concurrency": 4,
"round": 2,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.302,
"decode_s": 3.4163,
"ttft_s": 11.905763,
"first_answer_s": 12.623437,
"total_s": 15.310805
},
"metrics": {
"prompt_tps": 258.278146,
"decode_tps": 149.869742
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r2-c2",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 2,
"seed": 6,
"max_output_tokens": 512,
"concurrency": 4,
"round": 2,
"status": "completed",
"finish_reason": "stop",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 401,
"decode_tokens": 401
},
"timings": {
"prompt_s": 0.2983,
"decode_s": 2.785,
"ttft_s": 0.314868,
"first_answer_s": 0.917163,
"total_s": 3.088908
},
"metrics": {
"prompt_tps": 261.48173,
"decode_tps": 143.985637
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r2-c3",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 2,
"seed": 7,
"max_output_tokens": 512,
"concurrency": 4,
"round": 2,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.2985,
"decode_s": 4.1389,
"ttft_s": 7.461829,
"first_answer_s": 10.869258,
"total_s": 11.589246
},
"metrics": {
"prompt_tps": 261.306533,
"decode_tps": 123.704366
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r2-c4",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 2,
"seed": 8,
"max_output_tokens": 512,
"concurrency": 4,
"round": 2,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.2994,
"decode_s": 3.7582,
"ttft_s": 3.402808,
"first_answer_s": 4.669764,
"total_s": 7.14983
},
"metrics": {
"prompt_tps": 260.521042,
"decode_tps": 136.235432
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r3-c1",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 3,
"seed": 9,
"max_output_tokens": 512,
"concurrency": 4,
"round": 3,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.302,
"decode_s": 3.7219,
"ttft_s": 4.174711,
"first_answer_s": 5.235479,
"total_s": 7.885758
},
"metrics": {
"prompt_tps": 258.278146,
"decode_tps": 137.564147
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r3-c2",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 3,
"seed": 10,
"max_output_tokens": 512,
"concurrency": 4,
"round": 3,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.2994,
"decode_s": 4.0292,
"ttft_s": 12.211817,
"first_answer_s": 14.241092,
"total_s": 16.230166
},
"metrics": {
"prompt_tps": 260.521042,
"decode_tps": 127.072372
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r3-c3",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 3,
"seed": 11,
"max_output_tokens": 512,
"concurrency": 4,
"round": 3,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.303,
"decode_s": 3.7087,
"ttft_s": 8.201769,
"first_answer_s": 9.483954,
"total_s": 11.899125
},
"metrics": {
"prompt_tps": 257.425743,
"decode_tps": 138.053765
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
},
{
"id": "concurrency-r3-c4",
"phase": "concurrency",
"case_id": "short-burst",
"repetition": 3,
"seed": 12,
"max_output_tokens": 512,
"concurrency": 4,
"round": 3,
"status": "truncated",
"finish_reason": "length",
"error": null,
"tokens": {
"input_tokens": 78,
"cached_tokens": 0,
"read_tokens": 78,
"output_tokens": 512,
"decode_tokens": 512
},
"timings": {
"prompt_s": 0.2973,
"decode_s": 3.5563,
"ttft_s": 0.312613,
"first_answer_s": 1.198234,
"total_s": 3.85865
},
"metrics": {
"prompt_tps": 262.361251,
"decode_tps": 143.969856
},
"sources": {
"input_tokens": "usage.prompt_tokens of the final stream chunk",
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
"read_tokens": "timings.prompt_n",
"output_tokens": "usage.completion_tokens of the final stream chunk",
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
},
"first_delta_kind": "reasoning",
"token_accounting_note": null,
"evaluation": null
}
],
"summaries": [
{
"phase": "performance",
"case_id": "short",
"concurrency": 1,
"metric": "prompt_tps",
"unit": "tokens/s",
"median": 226.737338,
"min": 195.978621,
"max": 236.631838,
"n": 3,
"included_request_ids": [
"performance-short-r1",
"performance-short-r2",
"performance-short-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "short",
"concurrency": 1,
"metric": "decode_tps",
"unit": "tokens/s",
"median": 130.745764,
"min": 124.132359,
"max": 135.553732,
"n": 3,
"included_request_ids": [
"performance-short-r1",
"performance-short-r2",
"performance-short-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "short",
"concurrency": 1,
"metric": "ttft_s",
"unit": "s",
"median": 0.353313,
"min": 0.341909,
"max": 0.409799,
"n": 3,
"included_request_ids": [
"performance-short-r1",
"performance-short-r2",
"performance-short-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "short",
"concurrency": 1,
"metric": "first_answer_s",
"unit": "s",
"median": 1.171217,
"min": 0.89164,
"max": 2.546901,
"n": 3,
"included_request_ids": [
"performance-short-r1",
"performance-short-r2",
"performance-short-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "short",
"concurrency": 1,
"metric": "total_s",
"unit": "s",
"median": 4.174695,
"min": 3.065101,
"max": 4.23085,
"n": 3,
"included_request_ids": [
"performance-short-r1",
"performance-short-r2",
"performance-short-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-4k",
"concurrency": 1,
"metric": "prompt_tps",
"unit": "tokens/s",
"median": 1776.090258,
"min": 1774.165583,
"max": 1778.019114,
"n": 3,
"included_request_ids": [
"performance-prompt-4k-r1",
"performance-prompt-4k-r2",
"performance-prompt-4k-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-4k",
"concurrency": 1,
"metric": "decode_tps",
"unit": "tokens/s",
"median": 123.168707,
"min": 112.63144,
"max": 128.420577,
"n": 3,
"included_request_ids": [
"performance-prompt-4k-r1",
"performance-prompt-4k-r2",
"performance-prompt-4k-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-4k",
"concurrency": 1,
"metric": "ttft_s",
"unit": "s",
"median": 2.323905,
"min": 2.320082,
"max": 2.326971,
"n": 3,
"included_request_ids": [
"performance-prompt-4k-r1",
"performance-prompt-4k-r2",
"performance-prompt-4k-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-4k",
"concurrency": 1,
"metric": "first_answer_s",
"unit": "s",
"median": 5.299431,
"min": 4.956982,
"max": 5.64188,
"n": 2,
"included_request_ids": [
"performance-prompt-4k-r2",
"performance-prompt-4k-r3"
],
"excluded": [
{
"request_id": "performance-prompt-4k-r1",
"reason": "first_answer_s is not available for this request"
}
]
},
{
"phase": "performance",
"case_id": "prompt-4k",
"concurrency": 1,
"metric": "total_s",
"unit": "s",
"median": 6.469928,
"min": 6.296036,
"max": 6.861012,
"n": 3,
"included_request_ids": [
"performance-prompt-4k-r1",
"performance-prompt-4k-r2",
"performance-prompt-4k-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-16k",
"concurrency": 1,
"metric": "prompt_tps",
"unit": "tokens/s",
"median": 2937.949334,
"min": 2934.477016,
"max": 2945.024531,
"n": 3,
"included_request_ids": [
"performance-prompt-16k-r1",
"performance-prompt-16k-r2",
"performance-prompt-16k-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-16k",
"concurrency": 1,
"metric": "decode_tps",
"unit": "tokens/s",
"median": 124.549966,
"min": 115.004492,
"max": 131.43708,
"n": 3,
"included_request_ids": [
"performance-prompt-16k-r1",
"performance-prompt-16k-r2",
"performance-prompt-16k-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-16k",
"concurrency": 1,
"metric": "ttft_s",
"unit": "s",
"median": 5.608843,
"min": 5.591829,
"max": 5.613231,
"n": 3,
"included_request_ids": [
"performance-prompt-16k-r1",
"performance-prompt-16k-r2",
"performance-prompt-16k-r3"
],
"excluded": []
},
{
"phase": "performance",
"case_id": "prompt-16k",
"concurrency": 1,
"metric": "first_answer_s",
"unit": "s",
"median": 7.966815,
"min": 7.790823,
"max": 8.142807,
"n": 2,
"included_request_ids": [
"performance-prompt-16k-r1",
"performance-prompt-16k-r2"
],
"excluded": [
{
"request_id": "performance-prompt-16k-r3",
"reason": "first_answer_s is not available for this request"
}
]
},
{
"phase": "performance",
"case_id": "prompt-16k",
"concurrency": 1,
"metric": "total_s",
"unit": "s",
"median": 9.690426,
"min": 9.493855,
"max": 10.055156,
"n": 3,
"included_request_ids": [
"performance-prompt-16k-r1",
"performance-prompt-16k-r2",
"performance-prompt-16k-r3"
],
"excluded": []
},
{
"phase": "concurrency",
"case_id": "short-burst",
"concurrency": 4,
"metric": "prompt_tps",
"unit": "tokens/s",
"median": 259.182421,
"min": 236.86608,
"max": 262.361251,
"n": 12,
"included_request_ids": [
"concurrency-r1-c1",
"concurrency-r1-c2",
"concurrency-r1-c3",
"concurrency-r1-c4",
"concurrency-r2-c1",
"concurrency-r2-c2",
"concurrency-r2-c3",
"concurrency-r2-c4",
"concurrency-r3-c1",
"concurrency-r3-c2",
"concurrency-r3-c3",
"concurrency-r3-c4"
],
"excluded": []
},
{
"phase": "concurrency",
"case_id": "short-burst",
"concurrency": 4,
"metric": "decode_tps",
"unit": "tokens/s",
"median": 136.899789,
"min": 116.141911,
"max": 149.869742,
"n": 12,
"included_request_ids": [
"concurrency-r1-c1",
"concurrency-r1-c2",
"concurrency-r1-c3",
"concurrency-r1-c4",
"concurrency-r2-c1",
"concurrency-r2-c2",
"concurrency-r2-c3",
"concurrency-r2-c4",
"concurrency-r3-c1",
"concurrency-r3-c2",
"concurrency-r3-c3",
"concurrency-r3-c4"
],
"excluded": []
},
{
"phase": "concurrency",
"case_id": "short-burst",
"concurrency": 4,
"metric": "ttft_s",
"unit": "s",
"median": 5.81827,
"min": 0.312613,
"max": 12.211817,
"n": 12,
"included_request_ids": [
"concurrency-r1-c1",
"concurrency-r1-c2",
"concurrency-r1-c3",
"concurrency-r1-c4",
"concurrency-r2-c1",
"concurrency-r2-c2",
"concurrency-r2-c3",
"concurrency-r2-c4",
"concurrency-r3-c1",
"concurrency-r3-c2",
"concurrency-r3-c3",
"concurrency-r3-c4"
],
"excluded": []
},
{
"phase": "concurrency",
"case_id": "short-burst",
"concurrency": 4,
"metric": "first_answer_s",
"unit": "s",
"median": 8.5693,
"min": 0.889766,
"max": 15.534626,
"n": 12,
"included_request_ids": [
"concurrency-r1-c1",
"concurrency-r1-c2",
"concurrency-r1-c3",
"concurrency-r1-c4",
"concurrency-r2-c1",
"concurrency-r2-c2",
"concurrency-r2-c3",
"concurrency-r2-c4",
"concurrency-r3-c1",
"concurrency-r3-c2",
"concurrency-r3-c3",
"concurrency-r3-c4"
],
"excluded": []
},
{
"phase": "concurrency",
"case_id": "short-burst",
"concurrency": 4,
"metric": "total_s",
"unit": "s",
"median": 10.00632,
"min": 3.088908,
"max": 16.253254,
"n": 12,
"included_request_ids": [
"concurrency-r1-c1",
"concurrency-r1-c2",
"concurrency-r1-c3",
"concurrency-r1-c4",
"concurrency-r2-c1",
"concurrency-r2-c2",
"concurrency-r2-c3",
"concurrency-r2-c4",
"concurrency-r3-c1",
"concurrency-r3-c2",
"concurrency-r3-c3",
"concurrency-r3-c4"
],
"excluded": []
}
],
"concurrency_rounds": [
{
"round": 1,
"request_ids": [
"concurrency-r1-c1",
"concurrency-r1-c2",
"concurrency-r1-c3",
"concurrency-r1-c4"
],
"wall_s": 16.25344,
"output_tokens": 1877,
"aggregate_tps": 115.483245,
"failed": 0
},
{
"round": 2,
"request_ids": [
"concurrency-r2-c1",
"concurrency-r2-c2",
"concurrency-r2-c3",
"concurrency-r2-c4"
],
"wall_s": 15.310917,
"output_tokens": 1937,
"aggregate_tps": 126.511038,
"failed": 0
},
{
"round": 3,
"request_ids": [
"concurrency-r3-c1",
"concurrency-r3-c2",
"concurrency-r3-c3",
"concurrency-r3-c4"
],
"wall_s": 16.230166,
"output_tokens": 2048,
"aggregate_tps": 126.184785,
"failed": 0
}
],
"quality": {
"passed": 12,
"total": 12,
"categories": {
"instruction": {
"passed": 4,
"total": 4
},
"retrieval": {
"passed": 4,
"total": 4
},
"tool_call": {
"passed": 4,
"total": 4
}
}
},
"artifacts": [
{
"kind": "outputs",
"path": "outputs.jsonl",
"sha256": "d6aae509aa76e077f381700178931e540314b3f2b15916a83cf4735d6f463a4b"
},
{
"kind": "config",
"path": "config.json",
"sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410"
},
{
"kind": "provenance",
"path": "provenance/chat_template.jinja",
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
},
{
"kind": "provenance",
"path": "provenance/corpus-manifest.sha256",
"sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407"
},
{
"kind": "provenance",
"path": "provenance/corpus-build.json",
"sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2"
}
],
"missing": {
"/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)",
"/hardware/storage": "the drive holding the model files could not be identified",
"/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background",
"/software/build_options": "the server does not report build options; pass --build-options for a source build",
"/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",
"/model/source_revision": "not given: pass --source-revision",
"/model/tokenizer_revision": "not given: pass --tokenizer-revision",
"/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)",
"/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
"/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
"/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config",
"/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config",
"/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config",
"/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config",
"/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config",
"/requests/0/round": "not applicable outside the concurrency stage",
"/requests/0/error": "none: the request ended normally",
"/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/0/evaluation": "not applicable outside the sanity stage",
"/requests/1/round": "not applicable outside the concurrency stage",
"/requests/1/error": "none: the request ended normally",
"/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/1/evaluation": "not applicable outside the sanity stage",
"/requests/2/round": "not applicable outside the concurrency stage",
"/requests/2/error": "none: the request ended normally",
"/requests/2/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
"/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/2/evaluation": "not applicable outside the sanity stage",
"/requests/3/round": "not applicable outside the concurrency stage",
"/requests/3/error": "none: the request ended normally",
"/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/3/evaluation": "not applicable outside the sanity stage",
"/requests/4/round": "not applicable outside the concurrency stage",
"/requests/4/error": "none: the request ended normally",
"/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/4/evaluation": "not applicable outside the sanity stage",
"/requests/5/round": "not applicable outside the concurrency stage",
"/requests/5/error": "none: the request ended normally",
"/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/5/evaluation": "not applicable outside the sanity stage",
"/requests/6/round": "not applicable outside the concurrency stage",
"/requests/6/error": "none: the request ended normally",
"/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/6/evaluation": "not applicable outside the sanity stage",
"/requests/7/round": "not applicable outside the concurrency stage",
"/requests/7/error": "none: the request ended normally",
"/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/7/evaluation": "not applicable outside the sanity stage",
"/requests/8/round": "not applicable outside the concurrency stage",
"/requests/8/error": "none: the request ended normally",
"/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/8/evaluation": "not applicable outside the sanity stage",
"/requests/9/round": "not applicable outside the concurrency stage",
"/requests/9/error": "none: the request ended normally",
"/requests/9/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
"/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/9/evaluation": "not applicable outside the sanity stage",
"/requests/10/round": "not applicable outside the concurrency stage",
"/requests/10/error": "none: the request ended normally",
"/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/11/round": "not applicable outside the concurrency stage",
"/requests/11/error": "none: the request ended normally",
"/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/12/round": "not applicable outside the concurrency stage",
"/requests/12/error": "none: the request ended normally",
"/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/13/round": "not applicable outside the concurrency stage",
"/requests/13/error": "none: the request ended normally",
"/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/14/round": "not applicable outside the concurrency stage",
"/requests/14/error": "none: the request ended normally",
"/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/15/round": "not applicable outside the concurrency stage",
"/requests/15/error": "none: the request ended normally",
"/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/16/round": "not applicable outside the concurrency stage",
"/requests/16/error": "none: the request ended normally",
"/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/17/round": "not applicable outside the concurrency stage",
"/requests/17/error": "none: the request ended normally",
"/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/18/round": "not applicable outside the concurrency stage",
"/requests/18/error": "none: the request ended normally",
"/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/19/round": "not applicable outside the concurrency stage",
"/requests/19/error": "none: the request ended normally",
"/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/20/round": "not applicable outside the concurrency stage",
"/requests/20/error": "none: the request ended normally",
"/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/21/round": "not applicable outside the concurrency stage",
"/requests/21/error": "none: the request ended normally",
"/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/22/error": "none: the request ended normally",
"/requests/22/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/22/evaluation": "not applicable outside the sanity stage",
"/requests/23/error": "none: the request ended normally",
"/requests/23/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/23/evaluation": "not applicable outside the sanity stage",
"/requests/24/error": "none: the request ended normally",
"/requests/24/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/24/evaluation": "not applicable outside the sanity stage",
"/requests/25/error": "none: the request ended normally",
"/requests/25/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/25/evaluation": "not applicable outside the sanity stage",
"/requests/26/error": "none: the request ended normally",
"/requests/26/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/26/evaluation": "not applicable outside the sanity stage",
"/requests/27/error": "none: the request ended normally",
"/requests/27/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/27/evaluation": "not applicable outside the sanity stage",
"/requests/28/error": "none: the request ended normally",
"/requests/28/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/28/evaluation": "not applicable outside the sanity stage",
"/requests/29/error": "none: the request ended normally",
"/requests/29/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/29/evaluation": "not applicable outside the sanity stage",
"/requests/30/error": "none: the request ended normally",
"/requests/30/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/30/evaluation": "not applicable outside the sanity stage",
"/requests/31/error": "none: the request ended normally",
"/requests/31/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/31/evaluation": "not applicable outside the sanity stage",
"/requests/32/error": "none: the request ended normally",
"/requests/32/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/32/evaluation": "not applicable outside the sanity stage",
"/requests/33/error": "none: the request ended normally",
"/requests/33/token_accounting_note": "none: the API and engine counts agree or only one exists",
"/requests/33/evaluation": "not applicable outside the sanity stage"
},
"extensions": {
"suite": "standard-v1",
"runner": "strata_cli.bench_standard",
"status_reasons": [],
"started_engine": false,
"server_requests_before_run": 1122,
"other_engine_options": {},
"request_checks": {
"warmup": {
"max_shared_prefix_tokens": 47,
"frozen_input_tokens": 75,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-short-r1": {
"max_shared_prefix_tokens": 50,
"frozen_input_tokens": 77,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-prompt-4k-r1": {
"max_shared_prefix_tokens": 54,
"frozen_input_tokens": 4093,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-prompt-16k-r1": {
"max_shared_prefix_tokens": 55,
"frozen_input_tokens": 16387,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-short-r2": {
"max_shared_prefix_tokens": 50,
"frozen_input_tokens": 77,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-prompt-4k-r2": {
"max_shared_prefix_tokens": 54,
"frozen_input_tokens": 4093,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-prompt-16k-r2": {
"max_shared_prefix_tokens": 55,
"frozen_input_tokens": 16387,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-short-r3": {
"max_shared_prefix_tokens": 50,
"frozen_input_tokens": 77,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-prompt-4k-r3": {
"max_shared_prefix_tokens": 54,
"frozen_input_tokens": 4093,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"performance-prompt-16k-r3": {
"max_shared_prefix_tokens": 55,
"frozen_input_tokens": 16387,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-I1": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 82,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-I2": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 90,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-I3": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 85,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-I4": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 71,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-R1": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 4089,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-R2": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 4089,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-R3": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 16382,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-R4": {
"max_shared_prefix_tokens": 49,
"frozen_input_tokens": 16390,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-T1": {
"max_shared_prefix_tokens": 61,
"frozen_input_tokens": 325,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-T2": {
"max_shared_prefix_tokens": 62,
"frozen_input_tokens": 335,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-T3": {
"max_shared_prefix_tokens": 61,
"frozen_input_tokens": 352,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"sanity-T4": {
"max_shared_prefix_tokens": 62,
"frozen_input_tokens": 339,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r1-c1": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r1-c2": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r1-c3": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r1-c4": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r2-c1": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r2-c2": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r2-c3": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r2-c4": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r3-c1": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r3-c2": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r3-c3": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
},
"concurrency-r3-c4": {
"max_shared_prefix_tokens": 51,
"frozen_input_tokens": 78,
"unexpected_payload_reuse": false,
"length_outside_tolerance": false
}
},
"optional_stage_skipped": false,
"api_key_sent": true
}
}
Reference in New Issue View Git Blame Copy Permalink