{
  "generatedAtUtc": "2026-07-17T06:26:25Z",
  "benchmark": {
    "repository": "https://github.com/ciru-ai/invoice-sandbox-benchmark",
    "commit": "b8cb58ff9747486e86c623e867b76fad5d8d075f",
    "workspaceFiles": 148,
    "pdfFiles": 112,
    "expectedValidInvoices": 105,
    "expectedCustomers": 18
  },
  "harness": {
    "name": "Pi",
    "version": "0.78.0",
    "tools": ["read", "bash", "edit", "write", "grep", "find", "ls"],
    "pdfParseVersion": "1.1.1",
    "promptSha256": "0595615047a4819809b394c8de59eb2b252d2a5e573bb16d61d17dacf2c92479"
  },
  "runs": [
    {
      "id": "hy3-iq1m",
      "label": "AngelSlim Hy3 IQ1_M MTP",
      "profile": "hy3-iq1-m-mtp-64k",
      "sourceRepository": "https://huggingface.co/AngelSlim/Hy3-GGUF",
      "sourceRevision": "218c93f0fb5227553b67e556b01dfe70fb70cf30",
      "artifactSha256": "f3b9ab6394d9de03394b9d95aa75af42ca7025711cf8418857eddd0d213e5f13",
      "contextTokens": 65536,
      "completed": true,
      "dashboardProduced": true,
      "wallSeconds": 1544.38,
      "assistantTurns": 29,
      "toolCalls": 36,
      "toolErrors": 2,
      "piInputTokens": 27001,
      "piOutputTokens": 11547,
      "piCacheReadTokens": 593114,
      "serverPromptTokens": 27001,
      "serverOutputTokens": 11547,
      "promptTokensPerSecond": 77.6525,
      "generatedTokensPerSecond": 9.68415,
      "customerNetExact": 18,
      "customerNetPossible": 18,
      "validDocumentsExact": 105,
      "validDocumentsPossible": 105,
      "invoiceTotalsExact": 105,
      "invoiceTotalsPossible": 105,
      "invoiceCountsExact": 18,
      "invoiceCountsPossible": 18,
      "trapsPassed": 6,
      "trapsPossible": 6,
      "expectedNetUsd": 3566177.15,
      "observedNetUsd": 3566177.15,
      "netAbsoluteErrorUsd": 0,
      "dashboardSha256": "31cbefc3a84f324e8464431d8569fbb3ef72d7ef3bf55f650364c4727e30aa82"
    },
    {
      "id": "bonsai-27b",
      "label": "Ternary Bonsai 27B Q2_0",
      "profile": "lan-ternary-bonsai-27b-q2_0",
      "apiModelId": "ternary-bonsai-27b",
      "sourceRepository": "https://huggingface.co/prism-ml/Ternary-Bonsai-27B-gguf",
      "endpointClass": "LAN OpenAI-compatible API",
      "contextTokens": 262144,
      "sampling": {
        "temperature": 1.0,
        "topK": 20,
        "topP": 0.95,
        "minP": 0.05,
        "seed": "random"
      },
      "completed": true,
      "dashboardProduced": true,
      "wallSeconds": 637.42,
      "assistantTurns": 21,
      "toolCalls": 23,
      "toolErrors": 1,
      "piInputTokens": 272858,
      "piOutputTokens": 25690,
      "piCacheReadTokens": 445410,
      "serverPromptTokens": 272858,
      "serverOutputTokens": 25690,
      "promptTokensPerSecond": 1805.547836,
      "generatedTokensPerSecond": 53.232601,
      "customerNetExact": 0,
      "customerNetPossible": 18,
      "customerNetExactAfterIdPrefixDiagnostic": 16,
      "validDocumentsExact": 105,
      "validDocumentsPossible": 105,
      "invoiceTotalsExact": 105,
      "invoiceTotalsPossible": 105,
      "invoiceCountsExact": 0,
      "invoiceCountsPossible": 18,
      "trapsPassed": 5,
      "trapsPossible": 6,
      "expectedNetUsd": 3566177.15,
      "observedNetUsd": 3568582.15,
      "netAbsoluteErrorUsd": 2405.0,
      "instructionAdherence": {
        "requiredPdfParseUsed": true,
        "pageOneOnly": true,
        "prohibitedPdfDefaultUsed": false,
        "prohibitedPdfParserUsed": false,
        "manualStreamDecompressionUsed": false,
        "exploratoryPdfMinerUsed": true
      },
      "dashboardSha256": "1c7bbaa129dea512026349433fdef533e72e8ff629cfa141379ba6d11bc7f61d"
    },
    {
      "id": "hy3-ifp2",
      "label": "Hy3 Chadrock ROCmFPX-iFP2 MTP",
      "profile": "hy3-chadrock-fpx-ifp2-speed-mtp-64k",
      "contextTokens": 65536,
      "completed": true,
      "dashboardProduced": true,
      "wallSeconds": 1937.09,
      "assistantTurns": 22,
      "toolCalls": 29,
      "toolErrors": 0,
      "serverPromptTokens": 192221,
      "serverOutputTokens": 9685,
      "promptTokensPerSecond": 161.914,
      "generatedTokensPerSecond": 12.9803,
      "customerNetExact": 1,
      "customerNetPossible": 18,
      "validDocumentsExact": 105,
      "validDocumentsPossible": 105,
      "invoiceTotalsExact": 9,
      "invoiceTotalsPossible": 105,
      "invoiceCountsExact": 18,
      "invoiceCountsPossible": 18,
      "trapsPassed": 5,
      "trapsPossible": 6,
      "expectedNetUsd": 3566177.15,
      "observedNetUsd": 3413274.81,
      "netAbsoluteErrorUsd": 152902.34,
      "dashboardSha256": "1af3c8ad04aec0aa0c493091fcb0ba8124d2321b7bcf7ff4a86d2a09cb8ab88d"
    },
    {
      "id": "step37-fp3",
      "label": "Step 3.7 Flash ROCmFPX FP3 QualityPlus MTP",
      "profile": "step3.7-flash-rocmfpx-q3-qualityplus-mtp-256k",
      "contextTokens": 262144,
      "completed": false,
      "dashboardProduced": false,
      "wallSeconds": 297.87,
      "assistantTurns": 48,
      "toolCalls": 48,
      "toolErrors": 1,
      "serverPromptTokens": 25839,
      "serverOutputTokens": 4960,
      "promptTokensPerSecond": 179.303,
      "generatedTokensPerSecond": 34.2904,
      "partialParserRecords": 112,
      "resolvedCustomerIds": 0,
      "repeatedSampleCalls": 29,
      "failure": "Stable repeated-tool loop after the batch parser failed to resolve customer IDs; no HTML dashboard was written."
    }
  ]
}