Jason WittyandClaude Opus 5.5
d525f315f4
standard-v1 on the idle server: ~2,900 prompt tok/s at 16K, decode ~110-131. --adapt-async 1 neutral, --pipeline-windows 2 slower decode, --prefill auto:32768 22% slower prompt at 16K. Setup defaults kept and the original config restored. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1682 lines
56 KiB
JSON
1682 lines
56 KiB
JSON
{
|
|
"format": "strata-standard-benchmark",
|
|
"schema_version": 1,
|
|
"protocol": {
|
|
"id": "strata-qwen38-thinking-v1-draft.1",
|
|
"git_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"profile": "qwen3.8-flash-next-thinking",
|
|
"corpus_sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407",
|
|
"model_card_revision": null
|
|
},
|
|
"run": {
|
|
"id": "20261027T000315Z-92972582",
|
|
"started_at": "2026-10-27T00:03:15Z",
|
|
"status": "nonconforming",
|
|
"required_duration_s": 102.34,
|
|
"optional_duration_s": null,
|
|
"optional_concurrency": false,
|
|
"command": "<user> bench run --url http://127.0.0.1:8080 --credential-env STRATA_BENCH_KEY --skip-model-hash --track custom --track-notes arm prefill-32k: [\"--prefill\",\"auto:32768\"] --quantization IQ3_S"
|
|
},
|
|
"hardware": {
|
|
"gpus": [
|
|
{
|
|
"device_index": 0,
|
|
"name": "NVIDIA GeForce RTX 3090",
|
|
"memory_bytes": 25769803776,
|
|
"memory_kind": "dedicated",
|
|
"power_limit_w": 250.0
|
|
}
|
|
],
|
|
"cpu": "Intel(R) Core(TM) i9-14900KS",
|
|
"ram_bytes": 118089846784,
|
|
"storage": null,
|
|
"background_workloads": null
|
|
},
|
|
"software": {
|
|
"os": {
|
|
"name": "NixOS 26.05 (Yarara)",
|
|
"version": "6.18.55"
|
|
},
|
|
"drivers": [
|
|
{
|
|
"name": "gpu-driver",
|
|
"version": "595.71.05"
|
|
}
|
|
],
|
|
"runtime": [
|
|
{
|
|
"name": "cuda",
|
|
"version": "13.2"
|
|
}
|
|
],
|
|
"strata_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"engine_version": "0.1.51",
|
|
"build_kind": "source",
|
|
"runner_commit": "4b9e05a2a806a3fe24f7b2560d98626c37dade35",
|
|
"build_options": null
|
|
},
|
|
"model": {
|
|
"id": "qwen3.8-flash-next-iq3_s",
|
|
"revision": null,
|
|
"source_revision": null,
|
|
"quantization": "IQ3_S",
|
|
"files": null,
|
|
"tokenizer_revision": null,
|
|
"custom_pack": null
|
|
},
|
|
"configuration": {
|
|
"track": "custom",
|
|
"track_notes": "arm prefill-32k: [\"--prefill\",\"auto:32768\"]",
|
|
"context_tokens": 131072,
|
|
"server_capacity": 1,
|
|
"kv_cache": null,
|
|
"expert_cache": null,
|
|
"offloading": null,
|
|
"prefill": null,
|
|
"mtp": null,
|
|
"calibration": null,
|
|
"experimental_options": null,
|
|
"template": {
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
"sampling": {
|
|
"temperature": 1.0,
|
|
"top_p": 0.95,
|
|
"top_k": 20,
|
|
"min_p": 0.0,
|
|
"presence_penalty": 0.0,
|
|
"repetition_penalty": 1.0,
|
|
"reasoning_effort": "xhigh"
|
|
},
|
|
"chat_template_kwargs": {
|
|
"enable_thinking": true,
|
|
"preserve_thinking": true
|
|
}
|
|
},
|
|
"request_counts": {
|
|
"warmup": {
|
|
"planned": 1,
|
|
"attempted": 1,
|
|
"completed": 0,
|
|
"truncated": 1,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"performance": {
|
|
"planned": 9,
|
|
"attempted": 9,
|
|
"completed": 3,
|
|
"truncated": 6,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"sanity": {
|
|
"planned": 12,
|
|
"attempted": 12,
|
|
"completed": 12,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
},
|
|
"concurrency": {
|
|
"planned": 0,
|
|
"attempted": 0,
|
|
"completed": 0,
|
|
"truncated": 0,
|
|
"failed": 0,
|
|
"skipped": 0
|
|
}
|
|
},
|
|
"requests": [
|
|
{
|
|
"id": "warmup",
|
|
"phase": "warmup",
|
|
"case_id": "warmup",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 128,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 75,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 75,
|
|
"output_tokens": 128,
|
|
"decode_tokens": 128
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4055,
|
|
"decode_s": 1.1719,
|
|
"ttft_s": 0.433842,
|
|
"first_answer_s": 1.042827,
|
|
"total_s": 1.581193
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 184.956843,
|
|
"decode_tps": 109.224337
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r1",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 447,
|
|
"decode_tokens": 447
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.3761,
|
|
"decode_s": 3.6417,
|
|
"ttft_s": 0.394492,
|
|
"first_answer_s": 0.892493,
|
|
"total_s": 4.024395
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 204.732784,
|
|
"decode_tps": 122.744872
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.6489,
|
|
"decode_s": 4.588,
|
|
"ttft_s": 2.666856,
|
|
"first_answer_s": 6.134936,
|
|
"total_s": 7.243141
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1545.169693,
|
|
"decode_tps": 111.595466
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r1",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.6395,
|
|
"decode_s": 4.329,
|
|
"ttft_s": 7.669504,
|
|
"first_answer_s": 9.770124,
|
|
"total_s": 11.988469
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2145.03567,
|
|
"decode_tps": 118.272118
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r2",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 449,
|
|
"decode_tokens": 449
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4712,
|
|
"decode_s": 3.7583,
|
|
"ttft_s": 0.488926,
|
|
"first_answer_s": 1.709907,
|
|
"total_s": 4.236185
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 163.412564,
|
|
"decode_tps": 119.468909
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.4042,
|
|
"decode_s": 4.0489,
|
|
"ttft_s": 2.422599,
|
|
"first_answer_s": 4.841742,
|
|
"total_s": 6.460611
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1702.437401,
|
|
"decode_tps": 126.454099
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r2",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 2,
|
|
"seed": 2,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.2906,
|
|
"decode_s": 4.8339,
|
|
"ttft_s": 7.315847,
|
|
"first_answer_s": 11.621347,
|
|
"total_s": 12.137885
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2247.688805,
|
|
"decode_tps": 105.918616
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-short-r3",
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 77,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 77,
|
|
"output_tokens": 260,
|
|
"decode_tokens": 260
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4674,
|
|
"decode_s": 2.0275,
|
|
"ttft_s": 0.481843,
|
|
"first_answer_s": 1.113049,
|
|
"total_s": 2.497374
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 164.741121,
|
|
"decode_tps": 128.236745
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-4k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4093,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4093,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3474,
|
|
"decode_s": 4.9717,
|
|
"ttft_s": 2.363708,
|
|
"first_answer_s": null,
|
|
"total_s": 7.325637
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1743.631252,
|
|
"decode_tps": 102.982883
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "performance-prompt-16k-r3",
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"repetition": 3,
|
|
"seed": 3,
|
|
"max_output_tokens": 512,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "truncated",
|
|
"finish_reason": "length",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16387,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16387,
|
|
"output_tokens": 512,
|
|
"decode_tokens": 512
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.3176,
|
|
"decode_s": 5.1227,
|
|
"ttft_s": 7.347428,
|
|
"first_answer_s": null,
|
|
"total_s": 12.457752
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2239.39543,
|
|
"decode_tps": 99.947293
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": null
|
|
},
|
|
{
|
|
"id": "sanity-I1",
|
|
"phase": "sanity",
|
|
"case_id": "I1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 82,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 82,
|
|
"output_tokens": 74,
|
|
"decode_tokens": 74
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.4924,
|
|
"decode_s": 0.5651,
|
|
"ttft_s": 0.507508,
|
|
"first_answer_s": 0.976536,
|
|
"total_s": 1.061626
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 166.531275,
|
|
"decode_tps": 130.950274
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I2",
|
|
"phase": "sanity",
|
|
"case_id": "I2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 90,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 90,
|
|
"output_tokens": 60,
|
|
"decode_tokens": 60
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.366,
|
|
"decode_s": 0.4516,
|
|
"ttft_s": 0.38275,
|
|
"first_answer_s": 0.714752,
|
|
"total_s": 0.823786
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 245.901639,
|
|
"decode_tps": 132.860939
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I3",
|
|
"phase": "sanity",
|
|
"case_id": "I3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 85,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 85,
|
|
"output_tokens": 90,
|
|
"decode_tokens": 90
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.368,
|
|
"decode_s": 0.6786,
|
|
"ttft_s": 0.3848,
|
|
"first_answer_s": 0.858799,
|
|
"total_s": 1.052106
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 230.978261,
|
|
"decode_tps": 132.625995
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-I4",
|
|
"phase": "sanity",
|
|
"case_id": "I4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 71,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 71,
|
|
"output_tokens": 44,
|
|
"decode_tokens": 44
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.39,
|
|
"decode_s": 0.3196,
|
|
"ttft_s": 0.40651,
|
|
"first_answer_s": 0.714583,
|
|
"total_s": 0.714734
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 182.051282,
|
|
"decode_tps": 137.67209
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "instruction",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R1",
|
|
"phase": "sanity",
|
|
"case_id": "R1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 79,
|
|
"decode_tokens": 79
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.3997,
|
|
"decode_s": 0.6404,
|
|
"ttft_s": 2.418636,
|
|
"first_answer_s": 3.020719,
|
|
"total_s": 3.047915
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1703.962995,
|
|
"decode_tps": 123.3604
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R2",
|
|
"phase": "sanity",
|
|
"case_id": "R2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 4089,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 4089,
|
|
"output_tokens": 59,
|
|
"decode_tokens": 59
|
|
},
|
|
"timings": {
|
|
"prompt_s": 2.4058,
|
|
"decode_s": 0.4686,
|
|
"ttft_s": 2.423593,
|
|
"first_answer_s": 2.822058,
|
|
"total_s": 2.880385
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 1699.642531,
|
|
"decode_tps": 125.906957
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R3",
|
|
"phase": "sanity",
|
|
"case_id": "R3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16382,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16382,
|
|
"output_tokens": 68,
|
|
"decode_tokens": 68
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.3172,
|
|
"decode_s": 0.5111,
|
|
"ttft_s": 7.346867,
|
|
"first_answer_s": 7.788916,
|
|
"total_s": 7.846141
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2238.834527,
|
|
"decode_tps": 133.046371
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the exact expected text"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-R4",
|
|
"phase": "sanity",
|
|
"case_id": "R4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "stop",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 16390,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 16390,
|
|
"output_tokens": 126,
|
|
"decode_tokens": 126
|
|
},
|
|
"timings": {
|
|
"prompt_s": 7.4424,
|
|
"decode_s": 0.9913,
|
|
"ttft_s": 7.472494,
|
|
"first_answer_s": 8.260505,
|
|
"total_s": 8.452904
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 2202.246587,
|
|
"decode_tps": 127.105821
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "retrieval",
|
|
"passed": true,
|
|
"reason": "the expected JSON value"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T1",
|
|
"phase": "sanity",
|
|
"case_id": "T1",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 325,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 325,
|
|
"output_tokens": 64,
|
|
"decode_tokens": 64
|
|
},
|
|
"timings": {
|
|
"prompt_s": 1.2811,
|
|
"decode_s": 0.499,
|
|
"ttft_s": 1.29742,
|
|
"first_answer_s": 1.636457,
|
|
"total_s": 1.784789
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 253.688237,
|
|
"decode_tps": 128.256513
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T2",
|
|
"phase": "sanity",
|
|
"case_id": "T2",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 335,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 335,
|
|
"output_tokens": 92,
|
|
"decode_tokens": 92
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9724,
|
|
"decode_s": 0.7489,
|
|
"ttft_s": 0.989003,
|
|
"first_answer_s": 1.466015,
|
|
"total_s": 1.727261
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 344.508433,
|
|
"decode_tps": 122.846842
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T3",
|
|
"phase": "sanity",
|
|
"case_id": "T3",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 352,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 352,
|
|
"output_tokens": 87,
|
|
"decode_tokens": 87
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9077,
|
|
"decode_s": 0.612,
|
|
"ttft_s": 0.92355,
|
|
"first_answer_s": 1.23857,
|
|
"total_s": 1.523853
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 387.793324,
|
|
"decode_tps": 142.156863
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
},
|
|
{
|
|
"id": "sanity-T4",
|
|
"phase": "sanity",
|
|
"case_id": "T4",
|
|
"repetition": 1,
|
|
"seed": 1,
|
|
"max_output_tokens": 2048,
|
|
"concurrency": 1,
|
|
"round": null,
|
|
"status": "completed",
|
|
"finish_reason": "tool_calls",
|
|
"error": null,
|
|
"tokens": {
|
|
"input_tokens": 339,
|
|
"cached_tokens": 0,
|
|
"read_tokens": 339,
|
|
"output_tokens": 77,
|
|
"decode_tokens": 77
|
|
},
|
|
"timings": {
|
|
"prompt_s": 0.9077,
|
|
"decode_s": 0.551,
|
|
"ttft_s": 0.924561,
|
|
"first_answer_s": 1.237569,
|
|
"total_s": 1.464608
|
|
},
|
|
"metrics": {
|
|
"prompt_tps": 373.471411,
|
|
"decode_tps": 139.745917
|
|
},
|
|
"sources": {
|
|
"input_tokens": "usage.prompt_tokens of the final stream chunk",
|
|
"cached_tokens": "usage.prompt_tokens_details.cached_tokens",
|
|
"read_tokens": "timings.prompt_n",
|
|
"output_tokens": "usage.completion_tokens of the final stream chunk",
|
|
"decode_tokens": "timings.predicted_n (agrees with timings.predicted_per_second x predicted_ms)",
|
|
"engine_timings": "timings.prompt_ms, timings.predicted_ms, timings.predicted_per_second of the final stream chunk (raw values in outputs.jsonl)",
|
|
"client_timings": "client monotonic clock from dispatch; event offsets in outputs.jsonl"
|
|
},
|
|
"first_delta_kind": "reasoning",
|
|
"token_accounting_note": null,
|
|
"evaluation": {
|
|
"category": "tool_call",
|
|
"passed": true,
|
|
"reason": "the expected tool call"
|
|
}
|
|
}
|
|
],
|
|
"summaries": [
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 164.741121,
|
|
"min": 163.412564,
|
|
"max": 204.732784,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 122.744872,
|
|
"min": 119.468909,
|
|
"max": 128.236745,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 0.481843,
|
|
"min": 0.394492,
|
|
"max": 0.488926,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 1.113049,
|
|
"min": 0.892493,
|
|
"max": 1.709907,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "short",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 4.024395,
|
|
"min": 2.497374,
|
|
"max": 4.236185,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-short-r1",
|
|
"performance-short-r2",
|
|
"performance-short-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 1702.437401,
|
|
"min": 1545.169693,
|
|
"max": 1743.631252,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 111.595466,
|
|
"min": 102.982883,
|
|
"max": 126.454099,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 2.422599,
|
|
"min": 2.363708,
|
|
"max": 2.666856,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 5.488339,
|
|
"min": 4.841742,
|
|
"max": 6.134936,
|
|
"n": 2,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2"
|
|
],
|
|
"excluded": [
|
|
{
|
|
"request_id": "performance-prompt-4k-r3",
|
|
"reason": "first_answer_s is not available for this request"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-4k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 7.243141,
|
|
"min": 6.460611,
|
|
"max": 7.325637,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-4k-r1",
|
|
"performance-prompt-4k-r2",
|
|
"performance-prompt-4k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "prompt_tps",
|
|
"unit": "tokens/s",
|
|
"median": 2239.39543,
|
|
"min": 2145.03567,
|
|
"max": 2247.688805,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "decode_tps",
|
|
"unit": "tokens/s",
|
|
"median": 105.918616,
|
|
"min": 99.947293,
|
|
"max": 118.272118,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "ttft_s",
|
|
"unit": "s",
|
|
"median": 7.347428,
|
|
"min": 7.315847,
|
|
"max": 7.669504,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "first_answer_s",
|
|
"unit": "s",
|
|
"median": 10.695735,
|
|
"min": 9.770124,
|
|
"max": 11.621347,
|
|
"n": 2,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2"
|
|
],
|
|
"excluded": [
|
|
{
|
|
"request_id": "performance-prompt-16k-r3",
|
|
"reason": "first_answer_s is not available for this request"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"phase": "performance",
|
|
"case_id": "prompt-16k",
|
|
"concurrency": 1,
|
|
"metric": "total_s",
|
|
"unit": "s",
|
|
"median": 12.137885,
|
|
"min": 11.988469,
|
|
"max": 12.457752,
|
|
"n": 3,
|
|
"included_request_ids": [
|
|
"performance-prompt-16k-r1",
|
|
"performance-prompt-16k-r2",
|
|
"performance-prompt-16k-r3"
|
|
],
|
|
"excluded": []
|
|
}
|
|
],
|
|
"concurrency_rounds": [],
|
|
"quality": {
|
|
"passed": 12,
|
|
"total": 12,
|
|
"categories": {
|
|
"instruction": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"retrieval": {
|
|
"passed": 4,
|
|
"total": 4
|
|
},
|
|
"tool_call": {
|
|
"passed": 4,
|
|
"total": 4
|
|
}
|
|
}
|
|
},
|
|
"artifacts": [
|
|
{
|
|
"kind": "outputs",
|
|
"path": "outputs.jsonl",
|
|
"sha256": "478f818041a3f774094ce94acb120db85063a6a6d536e0eecfd4e6619d2e31c7"
|
|
},
|
|
{
|
|
"kind": "config",
|
|
"path": "config.json",
|
|
"sha256": "c04e49e3e279b102073f14b16842effb0afa63556469259b0ca6b2b32632c410"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/chat_template.jinja",
|
|
"sha256": "12827f24b742ea4e80cdc12dbcf9622227056b9f797252a3149263d4f9aaadce"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-manifest.sha256",
|
|
"sha256": "e75eff8b9feea04f745520b392c4c17d97aefc1ba185e222aa6740efa8f68407"
|
|
},
|
|
{
|
|
"kind": "provenance",
|
|
"path": "provenance/corpus-build.json",
|
|
"sha256": "bc6c1f81d86863c3dc36a31f58010701d8ab05180f69f86c6978214409e43af2"
|
|
}
|
|
],
|
|
"missing": {
|
|
"/protocol/model_card_revision": "the profile's model-card revision is not pinned yet (protocol draft)",
|
|
"/run/optional_duration_s": "the optional concurrency stage was not selected",
|
|
"/hardware/storage": "the drive holding the model files could not be identified",
|
|
"/hardware/background_workloads": "the runner does not inspect other programs; none were declared with --background",
|
|
"/software/build_options": "the server does not report build options; pass --build-options for a source build",
|
|
"/model/revision": "not given: the quantization repository's immutable revision cannot be read from the engine; pass --model-revision",
|
|
"/model/source_revision": "not given: pass --source-revision",
|
|
"/model/files": "no local model file could be found to hash (the server reports no model_path or it is not on this host)",
|
|
"/model/tokenizer_revision": "not given: pass --tokenizer-revision",
|
|
"/model/custom_pack": "no custom pack or expert profile was declared by the operator (--custom-pack)",
|
|
"/configuration/kv_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/expert_cache": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/offloading": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/prefill": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/mtp": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/calibration": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/configuration/experimental_options": "engine options are not exposed by a running server; start it from a profile or pass --config",
|
|
"/requests/0/round": "not applicable outside the concurrency stage",
|
|
"/requests/0/error": "none: the request ended normally",
|
|
"/requests/0/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/0/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/1/round": "not applicable outside the concurrency stage",
|
|
"/requests/1/error": "none: the request ended normally",
|
|
"/requests/1/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/1/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/2/round": "not applicable outside the concurrency stage",
|
|
"/requests/2/error": "none: the request ended normally",
|
|
"/requests/2/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/2/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/3/round": "not applicable outside the concurrency stage",
|
|
"/requests/3/error": "none: the request ended normally",
|
|
"/requests/3/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/3/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/4/round": "not applicable outside the concurrency stage",
|
|
"/requests/4/error": "none: the request ended normally",
|
|
"/requests/4/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/4/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/5/round": "not applicable outside the concurrency stage",
|
|
"/requests/5/error": "none: the request ended normally",
|
|
"/requests/5/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/5/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/6/round": "not applicable outside the concurrency stage",
|
|
"/requests/6/error": "none: the request ended normally",
|
|
"/requests/6/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/6/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/7/round": "not applicable outside the concurrency stage",
|
|
"/requests/7/error": "none: the request ended normally",
|
|
"/requests/7/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/7/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/8/round": "not applicable outside the concurrency stage",
|
|
"/requests/8/error": "none: the request ended normally",
|
|
"/requests/8/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/8/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/8/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/9/round": "not applicable outside the concurrency stage",
|
|
"/requests/9/error": "none: the request ended normally",
|
|
"/requests/9/timings/first_answer_s": "no answer text or tool call arrived (reasoning alone does not count)",
|
|
"/requests/9/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/9/evaluation": "not applicable outside the sanity stage",
|
|
"/requests/10/round": "not applicable outside the concurrency stage",
|
|
"/requests/10/error": "none: the request ended normally",
|
|
"/requests/10/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/11/round": "not applicable outside the concurrency stage",
|
|
"/requests/11/error": "none: the request ended normally",
|
|
"/requests/11/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/12/round": "not applicable outside the concurrency stage",
|
|
"/requests/12/error": "none: the request ended normally",
|
|
"/requests/12/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/13/round": "not applicable outside the concurrency stage",
|
|
"/requests/13/error": "none: the request ended normally",
|
|
"/requests/13/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/14/round": "not applicable outside the concurrency stage",
|
|
"/requests/14/error": "none: the request ended normally",
|
|
"/requests/14/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/15/round": "not applicable outside the concurrency stage",
|
|
"/requests/15/error": "none: the request ended normally",
|
|
"/requests/15/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/16/round": "not applicable outside the concurrency stage",
|
|
"/requests/16/error": "none: the request ended normally",
|
|
"/requests/16/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/17/round": "not applicable outside the concurrency stage",
|
|
"/requests/17/error": "none: the request ended normally",
|
|
"/requests/17/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/18/round": "not applicable outside the concurrency stage",
|
|
"/requests/18/error": "none: the request ended normally",
|
|
"/requests/18/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/19/round": "not applicable outside the concurrency stage",
|
|
"/requests/19/error": "none: the request ended normally",
|
|
"/requests/19/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/20/round": "not applicable outside the concurrency stage",
|
|
"/requests/20/error": "none: the request ended normally",
|
|
"/requests/20/token_accounting_note": "none: the API and engine counts agree or only one exists",
|
|
"/requests/21/round": "not applicable outside the concurrency stage",
|
|
"/requests/21/error": "none: the request ended normally",
|
|
"/requests/21/token_accounting_note": "none: the API and engine counts agree or only one exists"
|
|
},
|
|
"extensions": {
|
|
"suite": "standard-v1",
|
|
"runner": "strata_cli.bench_standard",
|
|
"status_reasons": [
|
|
"no model file could be hashed"
|
|
],
|
|
"started_engine": false,
|
|
"server_requests_before_run": 0,
|
|
"other_engine_options": {},
|
|
"request_checks": {
|
|
"warmup": {
|
|
"max_shared_prefix_tokens": 47,
|
|
"frozen_input_tokens": 75,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r1": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r1": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r1": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r2": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r2": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r2": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-short-r3": {
|
|
"max_shared_prefix_tokens": 50,
|
|
"frozen_input_tokens": 77,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-4k-r3": {
|
|
"max_shared_prefix_tokens": 54,
|
|
"frozen_input_tokens": 4093,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"performance-prompt-16k-r3": {
|
|
"max_shared_prefix_tokens": 55,
|
|
"frozen_input_tokens": 16387,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 82,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 90,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 85,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-I4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 71,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R1": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R2": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 4089,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R3": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16382,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-R4": {
|
|
"max_shared_prefix_tokens": 49,
|
|
"frozen_input_tokens": 16390,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T1": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 325,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T2": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 335,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T3": {
|
|
"max_shared_prefix_tokens": 61,
|
|
"frozen_input_tokens": 352,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
},
|
|
"sanity-T4": {
|
|
"max_shared_prefix_tokens": 62,
|
|
"frozen_input_tokens": 339,
|
|
"unexpected_payload_reuse": false,
|
|
"length_outside_tolerance": false
|
|
}
|
|
},
|
|
"optional_stage_skipped": false,
|
|
"api_key_sent": true
|
|
}
|
|
}
|