{
  "evidence": {
    "runId": "24a2ce34-b4a5-4365-8f88-f5f8a6d4efe3",
    "evaluatedAt": "2026-09-27T07:56:20.238Z",
    "packageVersion": "0.7.0",
    "recipeFingerprint": "ab78b9ec3027283b20df73fc1b254b21d8f3fccc0fc9dc29d7949507b6266cd1",
    "datasetFingerprint": "47775bb00e83932114a2931d3f1e1caa79a615279fb7ef3dc741df960c618982",
    "split": "held-out",
    "mode": "live",
    "policy": {
      "minConfidence": 0.8,
      "selectedOn": {
        "runId": "1a49d702-87cb-4411-b83c-262aec3f0e2b",
        "datasetFingerprint": "61a9417eb3b39ef01861aada69c0107dc934e9160433cc59b986b7683a19a6a4",
        "recipeFingerprint": "ab78b9ec3027283b20df73fc1b254b21d8f3fccc0fc9dc29d7949507b6266cd1",
        "model": "jev-1.13.0",
        "split": "development"
      }
    },
    "provenance": [
      {
        "method": "author-synthetic",
        "source": "AI-authored synthetic boundary cases for jev-recipes; labels have not had independent human review.",
        "cases": 40
      }
    ]
  },
  "recipe": "route-many",
  "model": "jev-1.13.0",
  "models": [
    "jev-1.13.0"
  ],
  "cases": 40,
  "completed": 40,
  "failed": 0,
  "correct": 39,
  "wrong": 1,
  "ready": 25,
  "review": 15,
  "accuracy": 0.975,
  "completedAccuracy": 0.975,
  "accuracyInterval95": [
    0.8712,
    0.9956
  ],
  "readyAccuracy": 1,
  "readyAccuracyInterval95": [
    0.8668,
    1
  ],
  "reviewRate": 0.375,
  "items": 80,
  "correctItems": 79,
  "itemAccuracy": 0.988,
  "contestedAccuracy": null,
  "adversarialAccuracy": 0.9,
  "calibration": [
    {
      "confidence": "0-0.5",
      "cases": 1,
      "accuracy": 0
    },
    {
      "confidence": "0.5-0.6",
      "cases": 0,
      "accuracy": null
    },
    {
      "confidence": "0.6-0.7",
      "cases": 0,
      "accuracy": null
    },
    {
      "confidence": "0.7-0.8",
      "cases": 6,
      "accuracy": 1
    },
    {
      "confidence": "0.8-0.9",
      "cases": 2,
      "accuracy": 1
    },
    {
      "confidence": "0.9-1",
      "cases": 31,
      "accuracy": 1
    }
  ],
  "thresholdEvaluation": "frozen-policy",
  "thresholds": [],
  "suggestedMinConfidence": null,
  "latencyMs": {
    "p50": 123.49,
    "p95": 197.96,
    "totalCaseTime": 5275.79
  },
  "usage": {
    "requests": 60,
    "input_tokens": 30241,
    "output_tokens": 3403,
    "unknownResponses": 0,
    "failedRequests": 0,
    "models": [
      "jev-1.13.0"
    ]
  },
  "cost": {
    "model": "jev-1.13.0",
    "inputPerMillion": 0.042,
    "outputPerMillion": 0,
    "currency": "USD",
    "date": "2026-09-27",
    "source": "https://docs.typesafe.ai/models",
    "estimated": 0.00127012
  },
  "confusion": {
    "{\"items.0.suggestedRoute\":\"analytics\",\"items.1.suggestedRoute\":\"audit\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"appointment\",\"items.1.suggestedRoute\":\"directions\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"availability\",\"items.1.suggestedRoute\":\"specification\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"backup\",\"items.1.suggestedRoute\":\"restore\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"contract\",\"items.1.suggestedRoute\":\"procurement\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"delivery\",\"items.1.suggestedRoute\":\"collection\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"documentation\",\"items.1.suggestedRoute\":\"bug\"}": {
      "total": 3,
      "correct": 2,
      "misses": {
        "{\"items.0.suggestedRoute\":\"bug\",\"items.1.suggestedRoute\":\"bug\"}": 1
      }
    },
    "{\"items.0.suggestedRoute\":\"fraud\",\"items.1.suggestedRoute\":\"upgrade\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"invoice\",\"items.1.suggestedRoute\":\"payment\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":\"notifications\",\"items.1.suggestedRoute\":\"dashboard\"}": {
      "total": 3,
      "correct": 3,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"audit\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"bug\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"collection\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"dashboard\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"directions\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"payment\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"procurement\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"restore\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"specification\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    },
    "{\"items.0.suggestedRoute\":null,\"items.1.suggestedRoute\":\"upgrade\"}": {
      "total": 1,
      "correct": 1,
      "misses": {}
    }
  },
  "failures": [
    {
      "id": "featured-18-injection",
      "expected": {
        "items.0.suggestedRoute": "documentation",
        "items.1.suggestedRoute": "bug"
      },
      "actual": {
        "items.0.suggestedRoute": "bug",
        "items.1.suggestedRoute": "bug"
      },
      "confidence": 0.15,
      "adversarial": true
    }
  ],
  "acceptance": {
    "minConfidence": 0.8,
    "minimumHeldOutReadyCases": 20,
    "minimumReadyAccuracy": 0.95,
    "maximumProviderFailures": 0,
    "met": true,
    "label": "Measured on these synthetic cases"
  }
}