{ "accuracy": 0.8831168831168831, "brier_mean": 0.17388059732042813, "calibration_n": 231, "complete": true, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.375, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.25595494387562745, "n": 8 }, { "accuracy": 0.14285714285714285, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3297851132949228, "n": 7 }, { "accuracy": 0.6, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.43664669932552147, "n": 10 }, { "accuracy": 0.7058823529411765, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.544120172237359, "n": 17 }, { "accuracy": 0.8125, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6546157628392665, "n": 16 }, { "accuracy": 0.95, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7476094342235899, "n": 20 }, { "accuracy": 0.9259259259259259, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8653082190143647, "n": 27 }, { "accuracy": 0.9920634920634921, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9633948871674165, "n": 126 } ], "ece": 0.07994470880686899, "n": 231 }, "latency": { "n": 231, "p50_s": 0.21858379500008596, "p95_s": 1.5345631805002995 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": 0.0, "macro_accuracy": 0.8852705019629684, "model_identities": [ "decision-medium" ], "n_attempted": 231, "n_correct": 204, "n_planned": 231, "n_renormalized": 0, "n_scorable": 231, "n_valid": 231, "operational_success": 1.0, "ordinal_mae": 0.15935163111560383, "paraphrase_consistency": { "agree": 35, "agreement": 0.9722222222222222, "both_correct_rate_all_pairs": 0.9722222222222222, "both_valid": 36, "pairs": 36 }, "per_family": { "adequacy": { "accuracy": 0.9166666666666666, "brier_mean": 0.16264464567147735, "calibration_n": 12, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.6666666666666666, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5620667012484599, "n": 3 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6926419831347361, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7486761108579822, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8933094060543488, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9600966390610968, "n": 5 } ], "ece": 0.1191677575036739, "n": 12 }, "latency": { "n": 12, "p50_s": 0.21708322949962167, "p95_s": 0.21958271000085006 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 11, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "adversarial": { "accuracy": 1.0, "brier_mean": 0.02147913858770016, "calibration_n": 6, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7729575833605401, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8436883703095561, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9397783116556646, "n": 4 } ], "ece": 0.10404013328454087, "n": 6 }, "latency": { "n": 6, "p50_s": 0.687402278999798, "p95_s": 1.2077261487502255 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 6, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": 0.12498961013257048, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ambiguous": { "accuracy": 0.8571428571428571, "brier_mean": 0.21781633320694474, "calibration_n": 7, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.46074320252123346, "n": 1 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5964432224654873, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.758214144226258, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8725905572715923, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9669272131950774, "n": 2 } ], "ece": 0.2589752496899987, "n": 7 }, "latency": { "n": 7, "p50_s": 0.21855992199925822, "p95_s": 0.9147610416002856 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 7, "n_correct": 6, "n_planned": 7, "n_renormalized": 0, "n_scorable": 7, "n_valid": 7, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "brier_mean": 0.0013421247567715038, "calibration_n": 24, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9715361228706377, "n": 24 } ], "ece": 0.02846387712936227, "n": 24 }, "latency": { "n": 24, "p50_s": 0.21347228000104224, "p95_s": 0.22208953325143738 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 24, "n_correct": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_valid": 24, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "fact": { "accuracy": 1.0, "brier_mean": 0.002724952817468707, "calibration_n": 12, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.965341067263609, "n": 12 } ], "ece": 0.03465893273639098, "n": 12 }, "latency": { "n": 12, "p50_s": 0.2177023165004357, "p95_s": 0.22845289710048747 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "brier_mean": 0.009758254186063013, "calibration_n": 24, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6993083638267495, "n": 1 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8675616866841231, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.964999862451638, "n": 21 } ], "ece": 0.05419046463835855, "n": 24 }, "latency": { "n": 24, "p50_s": 0.2140641610003513, "p95_s": 0.2231446215008873 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 24, "n_correct": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_valid": 24, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "judge_hard": { "accuracy": 0.9411764705882353, "brier_mean": 0.2330549578604239, "calibration_n": 17, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5360955052256952, "n": 3 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6605400174675301, "n": 3 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7310585786300049, "n": 2 }, { "accuracy": 0.8333333333333334, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8502355969264647, "n": 6 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9376951392579799, "n": 3 } ], "ece": 0.1903708493203037, "n": 17 }, "latency": { "n": 17, "p50_s": 0.20071710599950165, "p95_s": 0.40699049959912303 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 17, "n_correct": 16, "n_planned": 17, "n_renormalized": 0, "n_scorable": 17, "n_valid": 17, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "long_policy": { "accuracy": 0.8421052631578947, "brier_mean": 0.37287762317711176, "calibration_n": 19, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.22442754228684655, "n": 1 }, { "accuracy": 1.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.31283638571410965, "n": 1 }, { "accuracy": 0.6, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4193208964205217, "n": 5 }, { "accuracy": 0.8, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5417293375335148, "n": 5 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6090125731919687, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7392890512295184, "n": 3 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8626221731586058, "n": 3 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.2759336134669747, "n": 19 }, "latency": { "n": 19, "p50_s": 1.3997882029998436, "p95_s": 2.098419081499196 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 19, "n_correct": 16, "n_planned": 19, "n_renormalized": 0, "n_scorable": 19, "n_valid": 19, "operational_success": 1.0, "ordinal_mae": 0.45926189423191066, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "multi_hop": { "accuracy": 0.7777777777777778, "brier_mean": 0.3339428580035161, "calibration_n": 18, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.2687583256416153, "n": 2 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3200120674268267, "n": 2 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4352711293911737, "n": 1 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5091602097092707, "n": 2 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6464075608078717, "n": 2 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7472210815029732, "n": 5 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8793432025803274, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9604376252440875, "n": 3 } ], "ece": 0.24694275391938864, "n": 18 }, "latency": { "n": 18, "p50_s": 0.3213424514997314, "p95_s": 1.32010343535012 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 18, "n_correct": 14, "n_planned": 18, "n_renormalized": 0, "n_scorable": 18, "n_valid": 18, "operational_success": 1.0, "ordinal_mae": 0.21521003292817387, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "brier_mean": 0.00520444465125537, "calibration_n": 12, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9430657463170716, "n": 12 } ], "ece": 0.05693425368292837, "n": 12 }, "latency": { "n": 12, "p50_s": 0.21340739050083357, "p95_s": 0.2468797694506065 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": 0.06209043374233167, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 1.0, "brier_mean": 0.04023651543090801, "calibration_n": 12, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6859103411550646, "n": 2 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8888360061630532, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9719461047045929, "n": 7 } ], "ece": 0.09650404718904676, "n": 12 }, "latency": { "n": 12, "p50_s": 0.21283611899980315, "p95_s": 13.76410637669986 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "probability": { "accuracy": 0.8, "brier_mean": 0.4404549320196618, "calibration_n": 10, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5541984537026645, "n": 1 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6350351933614518, "n": 2 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7632029218849571, "n": 2 }, { "accuracy": 0.6666666666666666, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8354966434681792, "n": 3 }, { "accuracy": 0.5, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.916099769988311, "n": 2 } ], "ece": 0.29880147861856776, "n": 10 }, "latency": { "n": 10, "p50_s": 1.3118648220006435, "p95_s": 1.674036793000232 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 10, "n_correct": 8, "n_planned": 10, "n_renormalized": 0, "n_scorable": 10, "n_valid": 10, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 1.0, "brier_mean": 0.009240877440215378, "calibration_n": 12, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8787478975111437, "n": 4 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9502562117534098, "n": 8 } ], "ece": 0.07357989299401224, "n": 12 }, "latency": { "n": 12, "p50_s": 0.23803264799971657, "p95_s": 1.4357894898999803 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing_hard": { "accuracy": 1.0, "brier_mean": 0.0015139575377800941, "calibration_n": 5, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9671320340975906, "n": 5 } ], "ece": 0.03286796590240937, "n": 5 }, "latency": { "n": 5, "p50_s": 0.2117115230012132, "p95_s": 0.21383038779931668 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 5, "n_correct": 5, "n_planned": 5, "n_renormalized": 0, "n_scorable": 5, "n_valid": 5, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "temporal_numeric": { "accuracy": 0.13333333333333333, "brier_mean": 0.7832557743071475, "calibration_n": 15, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.2, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.2571390714869885, "n": 5 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.33890881812417417, "n": 4 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4471442911532385, "n": 2 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5467381519846138, "n": 1 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6441928295732188, "n": 2 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7549149868676283, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.27504386734911973, "n": 15 }, "latency": { "n": 15, "p50_s": 0.23321098899941717, "p95_s": 2.3723114913995196 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 15, "n_correct": 2, "n_planned": 15, "n_renormalized": 0, "n_scorable": 15, "n_valid": 15, "operational_success": 1.0, "ordinal_mae": 0.3727883693553482, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tool_selection": { "accuracy": 1.0, "brier_mean": 0.0004656692263069889, "calibration_n": 12, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.981743734359022, "n": 12 } ], "ece": 0.018256265640978042, "n": 12 }, "latency": { "n": 12, "p50_s": 0.21899352399941563, "p95_s": 0.22537494965008592 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tradeoff": { "accuracy": 0.6666666666666666, "brier_mean": 0.3610780784089173, "calibration_n": 6, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4795595969337222, "n": 1 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5312093733737562, "n": 1 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6340886915385033, "n": 2 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8809986819154472, "n": 2 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.2596382992810244, "n": 6 }, "latency": { "n": 6, "p50_s": 0.724976662500012, "p95_s": 1.3624804334995133 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 4, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "trap": { "accuracy": 1.0, "brier_mean": 0.0269012123203273, "calibration_n": 8, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7340200209209016, "n": 2 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9771173080316468, "n": 6 } ], "ece": 0.08365701374603954, "n": 8 }, "latency": { "n": 8, "p50_s": 0.20171799050058326, "p95_s": 0.21115064904943212 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 8, "n_correct": 8, "n_planned": 8, "n_renormalized": 0, "n_scorable": 8, "n_valid": 8, "operational_success": 1.0, "ordinal_mae": 0.49173235429297507, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": null, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 0.8831168831168831, "brier_mean": 0.17388059732042813, "calibration_n": 231, "cost_basis": [ "no_billable_account_public_endpoint" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.375, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.25595494387562745, "n": 8 }, { "accuracy": 0.14285714285714285, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3297851132949228, "n": 7 }, { "accuracy": 0.6, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.43664669932552147, "n": 10 }, { "accuracy": 0.7058823529411765, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.544120172237359, "n": 17 }, { "accuracy": 0.8125, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6546157628392665, "n": 16 }, { "accuracy": 0.95, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7476094342235899, "n": 20 }, { "accuracy": 0.9259259259259259, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8653082190143647, "n": 27 }, { "accuracy": 0.9920634920634921, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9633948871674165, "n": 126 } ], "ece": 0.07994470880686899, "n": 231 }, "latency": { "n": 231, "p50_s": 0.21858379500008596, "p95_s": 1.5345631805002995 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 231, "n_correct": 204, "n_planned": 231, "n_renormalized": 0, "n_scorable": 231, "n_valid": 231, "operational_success": 1.0, "ordinal_mae": 0.15935163111560383, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" }