{"task_id": "original-policy-01-0", "family": "policy", "split": "public", "group": "original-policy-01", "ts": 1791912581.964942, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.01590639171181471, "no": 0.9840936082881853}, "probs_as_returned": {"yes": 0.01590639171181471, "no": 0.9840936082881853}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 5.152902279000045, "usage": {"input_tokens": 184, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "4a3f3e8d5c60ec05f5e2f23173830b69bdafb078fd72a6f1d35d43903e08c411", "runtime": null}
{"task_id": "original-policy-01-1", "family": "policy", "split": "public", "group": "original-policy-01", "ts": 1791912606.2565937, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.017986209962091555, "no": 0.9820137900379085}, "probs_as_returned": {"yes": 0.017986209962091555, "no": 0.9820137900379085}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 24.288911384999665, "usage": {"input_tokens": 187, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9f6a9cc90d1009f8473d94cb1d2106917ddbb4793baab3b933e9443128d61cdb", "runtime": null}
{"task_id": "original-policy-02-0", "family": "policy", "split": "public", "group": "original-policy-02", "ts": 1791912606.5124485, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9433475746920261, "no": 0.056652425307973875}, "probs_as_returned": {"yes": 0.9433475746920261, "no": 0.056652425307973875}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.25094059299954097, "usage": {"input_tokens": 173, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "013edf91a158c2257ee55c21e106e3b8c8ba93f74ee3be48ddc6114d325bcaca", "runtime": null}
{"task_id": "original-policy-02-1", "family": "policy", "split": "public", "group": "original-policy-02", "ts": 1791912606.7366817, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8933094060543487, "no": 0.10669059394565128}, "probs_as_returned": {"yes": 0.8933094060543487, "no": 0.10669059394565128}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2173203789989202, "usage": {"input_tokens": 172, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "da15de2f139f9a10d0c89e8381a0d72bdb2038abce725cee4aab288316d72fd3", "runtime": null}
{"task_id": "original-policy-03-0", "family": "policy", "split": "public", "group": "original-policy-03", "ts": 1791912606.9507475, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.679178699175393, "no": 0.320821300824607}, "probs_as_returned": {"yes": 0.679178699175393, "no": 0.320821300824607}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20785250900007668, "usage": {"input_tokens": 175, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "36830da049c44a19281222185b0f40db25d068e8a0c1d518e8671d947ae47781", "runtime": null}
{"task_id": "original-policy-03-1", "family": "policy", "split": "public", "group": "original-policy-03", "ts": 1791912607.1605067, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8740772351648677, "no": 0.12592276483513232}, "probs_as_returned": {"yes": 0.8740772351648677, "no": 0.12592276483513232}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20431526500033215, "usage": {"input_tokens": 174, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "8959c4503a47470b639bdf32e0ac31b83c62b7ff6f9eac496ba7e8f0c77a0a24", "runtime": null}
{"task_id": "original-policy-04-0", "family": "policy", "split": "public", "group": "original-policy-04", "ts": 1791912607.3736463, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.02297736991002561, "no": 0.9770226300899744}, "probs_as_returned": {"yes": 0.02297736991002561, "no": 0.9770226300899744}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2079292800008261, "usage": {"input_tokens": 181, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2dd116f87c1b62fd0e2f3bec29ea51f13698c7980124bc3834023e25886d588b", "runtime": null}
{"task_id": "original-policy-04-1", "family": "policy", "split": "public", "group": "original-policy-04", "ts": 1791912607.5974545, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.027585282226789985, "no": 0.97241471777321}, "probs_as_returned": {"yes": 0.027585282226789985, "no": 0.97241471777321}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21908740999970178, "usage": {"input_tokens": 178, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "753706a5786c8cb7ff6762ab9287d758a9e5603fa86cb3a6071287b1e98e0c70", "runtime": null}
{"task_id": "original-policy-05-0", "family": "policy", "split": "public", "group": "original-policy-05", "ts": 1791912607.8098261, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.02595735719779685, "no": 0.9740426428022031}, "probs_as_returned": {"yes": 0.02595735719779685, "no": 0.9740426428022031}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2074612899996282, "usage": {"input_tokens": 185, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "67742186d47503cc6e3af8eff2964accd9b5e57a048f0ad1e08181c8a261325e", "runtime": null}
{"task_id": "original-policy-05-1", "family": "policy", "split": "public", "group": "original-policy-05", "ts": 1791912608.020555, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.029312230751356316, "no": 0.9706877692486436}, "probs_as_returned": {"yes": 0.029312230751356316, "no": 0.9706877692486436}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20582477599964477, "usage": {"input_tokens": 177, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0a9f2bb4f474f68e714605ca4e41a7312f7335a2699712397ec92ceaed096fc1", "runtime": null}
{"task_id": "original-policy-06-0", "family": "policy", "split": "public", "group": "original-policy-06", "ts": 1791912608.2420459, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8991213772699436, "no": 0.10087862273005643}, "probs_as_returned": {"yes": 0.8991213772699436, "no": 0.10087862273005643}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21554586999991443, "usage": {"input_tokens": 187, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "ccd9afd3999fe1a8aaafa0b00ca285d4087473804e2aacc13b62bd995217e947", "runtime": null}
{"task_id": "original-policy-06-1", "family": "policy", "split": "public", "group": "original-policy-06", "ts": 1791912608.4578, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.6926419831347361, "no": 0.30735801686526387}, "probs_as_returned": {"yes": 0.6926419831347361, "no": 0.30735801686526387}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21012636799969187, "usage": {"input_tokens": 177, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9d386af1149cad3d8e5fa2745e27aa5af15cf71100ea7725dda828f436fc3842", "runtime": null}
{"task_id": "original-intent-01-0", "family": "intent", "split": "public", "group": "original-intent-01", "ts": 1791912608.6818075, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "refund", "ordinal_ev": null, "probs": {"cancel": 0.02882802717242703, "change_address": 0.008792051543524446, "other": 0.10710911483118023, "refund": 0.8424784469401595, "status": 0.012792359512708774}, "probs_as_returned": {"cancel": 0.02882802717242703, "change_address": 0.008792051543524446, "other": 0.10710911483118023, "refund": 0.8424784469401595, "status": 0.012792359512708774}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21844180799962487, "usage": {"input_tokens": 218, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "50f6b3f72cd123a77b1c674c5829cac6042d456077d964326ceb31510debc666", "runtime": null}
{"task_id": "original-intent-01-1", "family": "intent", "split": "public", "group": "original-intent-01", "ts": 1791912608.895875, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "refund", "ordinal_ev": null, "probs": {"cancel": 0.009211124272194627, "change_address": 0.0055868292815092575, "other": 0.03643070320088571, "refund": 0.9395602189732158, "status": 0.009211124272194627}, "probs_as_returned": {"cancel": 0.009211124272194627, "change_address": 0.0055868292815092575, "other": 0.03643070320088571, "refund": 0.9395602189732158, "status": 0.009211124272194627}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20869200700144575, "usage": {"input_tokens": 219, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "13e6cde5843503add433a43239de7c2f35ff691c826c5162e4bb875d4b5eb513", "runtime": null}
{"task_id": "original-intent-02-0", "family": "intent", "split": "public", "group": "original-intent-02", "ts": 1791912609.1212218, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "status", "ordinal_ev": null, "probs": {"cancel": 0.011108549970115802, "change_address": 0.008127188773071472, "other": 0.03421674278499859, "refund": 0.0071722189189558725, "status": 0.9393752995528581}, "probs_as_returned": {"cancel": 0.011108549970115802, "change_address": 0.008127188773071472, "other": 0.03421674278499859, "refund": 0.0071722189189558725, "status": 0.9393752995528581}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21950632200059772, "usage": {"input_tokens": 219, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "35de5e31c268275e3bf7635f7f617cac3288a1a24e001cb325eb8954ac0f17bc", "runtime": null}
{"task_id": "original-intent-02-1", "family": "intent", "split": "public", "group": "original-intent-02", "ts": 1791912609.3571086, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "status", "ordinal_ev": null, "probs": {"cancel": 0.012672026907698883, "change_address": 0.009271052936104893, "other": 0.023674463042780786, "refund": 0.008709348234212159, "status": 0.9456731088792033}, "probs_as_returned": {"cancel": 0.012672026907698883, "change_address": 0.009271052936104893, "other": 0.023674463042780786, "refund": 0.008709348234212159, "status": 0.9456731088792033}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22995570599960047, "usage": {"input_tokens": 218, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "eb01c34b24c3e8beb54a2db760a8e10f52ee5ddfcd45c2fdc177e77531241921", "runtime": null}
{"task_id": "original-intent-03-0", "family": "intent", "split": "public", "group": "original-intent-03", "ts": 1791912609.5803933, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "change_address", "ordinal_ev": null, "probs": {"cancel": 0.014556951576754974, "change_address": 0.9586908454522287, "other": 0.016495187160568595, "refund": 0.0037974077052821344, "status": 0.006459608105165554}, "probs_as_returned": {"cancel": 0.014556951576754974, "change_address": 0.9586908454522287, "other": 0.016495187160568595, "refund": 0.0037974077052821344, "status": 0.006459608105165554}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21820193599887716, "usage": {"input_tokens": 220, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5bfa403fe8ef2d670f13059732970e50dde6836bc237791894d50eea43aeb5b0", "runtime": null}
{"task_id": "original-intent-03-1", "family": "intent", "split": "public", "group": "original-intent-03", "ts": 1791912609.8048635, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "change_address", "ordinal_ev": null, "probs": {"cancel": 0.016524744731645016, "change_address": 0.9604087145815046, "other": 0.012869484137060211, "refund": 0.003924970904003681, "status": 0.006272085645786506}, "probs_as_returned": {"cancel": 0.016524744731645016, "change_address": 0.9604087145815046, "other": 0.012869484137060211, "refund": 0.003924970904003681, "status": 0.006272085645786506}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21848109700113127, "usage": {"input_tokens": 218, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "a78651330f2d8b2e34cb2df122c4ad6c9c43502cd09b2db87e69976d20cc5de3", "runtime": null}
{"task_id": "original-intent-04-0", "family": "intent", "split": "public", "group": "original-intent-04", "ts": 1791912610.0206282, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cancel", "ordinal_ev": null, "probs": {"cancel": 0.9599418573211413, "change_address": 0.009410938823622567, "other": 0.015516015015764165, "refund": 0.00780194231419751, "status": 0.007329246525274341}, "probs_as_returned": {"cancel": 0.9599418573211413, "change_address": 0.009410938823622567, "other": 0.015516015015764165, "refund": 0.00780194231419751, "status": 0.007329246525274341}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21116346700000577, "usage": {"input_tokens": 223, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "d154c368b929bdddbaedff80860c35107417d601119f14b0160b2c9b842c8bc1", "runtime": null}
{"task_id": "original-intent-04-1", "family": "intent", "split": "public", "group": "original-intent-04", "ts": 1791912610.235847, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cancel", "ordinal_ev": null, "probs": {"cancel": 0.9562728652884915, "change_address": 0.008806968615821323, "other": 0.019846810145648312, "refund": 0.007772122524622059, "status": 0.007301233425416813}, "probs_as_returned": {"cancel": 0.9562728652884915, "change_address": 0.008806968615821323, "other": 0.019846810145648312, "refund": 0.007772122524622059, "status": 0.007301233425416813}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20948871000109648, "usage": {"input_tokens": 218, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9e88220307cddf8e8ebbcebce1ad868663b9afb6a6d0740a024a647a823935f5", "runtime": null}
{"task_id": "original-intent-05-0", "family": "intent", "split": "public", "group": "original-intent-05", "ts": 1791912610.4486563, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "other", "ordinal_ev": null, "probs": {"cancel": 0.009542835501381472, "change_address": 0.0030981050911183523, "other": 0.973395684212945, "refund": 0.006981687597277548, "status": 0.006981687597277548}, "probs_as_returned": {"cancel": 0.009542835501381472, "change_address": 0.0030981050911183523, "other": 0.973395684212945, "refund": 0.006981687597277548, "status": 0.006981687597277548}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20623202299975674, "usage": {"input_tokens": 218, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "34cdb259c1ab7c80964f16c22315377f6917e353fd007386ee5fcb96b18de0c2", "runtime": null}
{"task_id": "original-intent-05-1", "family": "intent", "split": "public", "group": "original-intent-05", "ts": 1791912610.662022, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "other", "ordinal_ev": null, "probs": {"cancel": 0.01665579966429837, "change_address": 0.003716411246495059, "other": 0.9680255523271513, "refund": 0.005079731896714449, "status": 0.006522504865340781}, "probs_as_returned": {"cancel": 0.01665579966429837, "change_address": 0.003716411246495059, "other": 0.9680255523271513, "refund": 0.005079731896714449, "status": 0.006522504865340781}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20729865400062408, "usage": {"input_tokens": 217, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "73d17d6e85231ae63153c7323779fbca897a395aa835cc6773628a4b6edfaacc", "runtime": null}
{"task_id": "original-intent-06-0", "family": "intent", "split": "public", "group": "original-intent-06", "ts": 1791912610.878374, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "status", "ordinal_ev": null, "probs": {"cancel": 0.06924090459981717, "change_address": 0.009370737435575534, "other": 0.21327700098268285, "refund": 0.00880299315517491, "status": 0.6993083638267495}, "probs_as_returned": {"cancel": 0.06924090459981717, "change_address": 0.009370737435575534, "other": 0.21327700098268285, "refund": 0.00880299315517491, "status": 0.6993083638267495}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20999049000056402, "usage": {"input_tokens": 216, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "7ad09474079f9ab5edb2c283c8f6613ba8852227f2c5376e3145fbed5e7d4901", "runtime": null}
{"task_id": "original-intent-06-1", "family": "intent", "split": "public", "group": "original-intent-06", "ts": 1791912611.0908055, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "status", "ordinal_ev": null, "probs": {"cancel": 0.02378818153118597, "change_address": 0.004986273728554017, "other": 0.07327275755747975, "refund": 0.005307860754693446, "status": 0.8926449264280868}, "probs_as_returned": {"cancel": 0.02378818153118597, "change_address": 0.004986273728554017, "other": 0.07327275755747975, "refund": 0.005307860754693446, "status": 0.8926449264280868}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2065572629999224, "usage": {"input_tokens": 221, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "f91d6e5e5b201ada359cf89653671af422111c1e28242ce592868036416fe17c", "runtime": null}
{"task_id": "original-ordinal-01-0", "family": "ordinal", "split": "public", "group": "original-ordinal-01", "ts": 1791912611.3047194, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.055126708161598134, "probs": {"0": 0.9615888059098497, "1": 0.025625473686241428, "2": 0.00885592673636964, "3": 0.00392979366753914}, "probs_as_returned": {"0": 0.9615888059098497, "1": 0.025625473686241428, "2": 0.00885592673636964, "3": 0.00392979366753914}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20733239600122033, "usage": {"input_tokens": 196, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c152f07edc0782fd312c3da3cb684f84326a390ed884f214ed1c777f23bc54af", "runtime": null}
{"task_id": "original-ordinal-01-1", "family": "ordinal", "split": "public", "group": "original-ordinal-01", "ts": 1791912611.52474, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.06487275281290578, "probs": {"0": 0.9585984301647877, "1": 0.023998042129696244, "2": 0.011335872433338253, "3": 0.006067655272177675}, "probs_as_returned": {"0": 0.9585984301647877, "1": 0.023998042129696244, "2": 0.011335872433338253, "3": 0.006067655272177675}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2138822139986587, "usage": {"input_tokens": 197, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9a6413f89a94ba905be36633871971bd264cd9125d858cbe8b2abbd574fe1881", "runtime": null}
{"task_id": "original-ordinal-02-0", "family": "ordinal", "split": "public", "group": "original-ordinal-02", "ts": 1791912611.7483227, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "1", "ordinal_ev": 0.9894541655016668, "probs": {"0": 0.036520155857686594, "1": 0.9418672333986066, "2": 0.01725090012805999, "3": 0.004361710615646737}, "probs_as_returned": {"0": 0.036520155857686594, "1": 0.9418672333986066, "2": 0.01725090012805999, "3": 0.004361710615646737}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.218263979000767, "usage": {"input_tokens": 200, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "03ee7d374c111c7bd2a576088f8fe24da4422eab9541d1478560dbf57c53f6c5", "runtime": null}
{"task_id": "original-ordinal-02-1", "family": "ordinal", "split": "public", "group": "original-ordinal-02", "ts": 1791912611.9660676, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "1", "ordinal_ev": 0.993009238015512, "probs": {"0": 0.028780772286580842, "1": 0.953088281990331, "2": 0.014471881144083772, "3": 0.003659064579004473}, "probs_as_returned": {"0": 0.028780772286580842, "1": 0.953088281990331, "2": 0.014471881144083772, "3": 0.003659064579004473}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.211610534999636, "usage": {"input_tokens": 201, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c6b33455cd22cf3b49d2795608d3d69d873187132439a11e50f52fa85dfd82a9", "runtime": null}
{"task_id": "original-ordinal-03-0", "family": "ordinal", "split": "public", "group": "original-ordinal-03", "ts": 1791912612.1893253, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.932679433741297, "probs": {"0": 0.029453013310619003, "1": 0.03135256946758789, "2": 0.9162563873916706, "3": 0.022938029830122594}, "probs_as_returned": {"0": 0.029453013310619003, "1": 0.03135256946758789, "2": 0.9162563873916706, "3": 0.022938029830122594}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21772415899977204, "usage": {"input_tokens": 197, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "24e1f7f921004b5cd6802ffd5a9901d695da5c63ebced5497ecd90c365498ccd", "runtime": null}
{"task_id": "original-ordinal-03-1", "family": "ordinal", "split": "public", "group": "original-ordinal-03", "ts": 1791912612.4742153, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.9642370209846887, "probs": {"0": 0.01844010422587882, "1": 0.01732287478943253, "2": 0.9457969167588098, "3": 0.01844010422587882}, "probs_as_returned": {"0": 0.01844010422587882, "1": 0.01732287478943253, "2": 0.9457969167588098, "3": 0.01844010422587882}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.27944605100128683, "usage": {"input_tokens": 199, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "deb18dc0dec4dc1a763a497398c806509941c96075749d02b0e6109f291e0fc0", "runtime": null}
{"task_id": "original-ordinal-04-0", "family": "ordinal", "split": "public", "group": "original-ordinal-04", "ts": 1791912612.693732, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "3", "ordinal_ev": 2.939408722868747, "probs": {"0": 0.008327686666365141, "1": 0.006485608896938469, "2": 0.022636999338281047, "3": 0.9625497050984154}, "probs_as_returned": {"0": 0.008327686666365141, "1": 0.006485608896938469, "2": 0.022636999338281047, "3": 0.9625497050984154}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21366905200011388, "usage": {"input_tokens": 199, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "90db53519186790d0be5fd44d2c123286a2821a5989e9fb853cf513ce64b9954", "runtime": null}
{"task_id": "original-ordinal-04-1", "family": "ordinal", "split": "public", "group": "original-ordinal-04", "ts": 1791912612.9125364, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "3", "ordinal_ev": 2.9378483627391523, "probs": {"0": 0.007808933493590185, "1": 0.007335814130520291, "2": 0.024053208519036834, "3": 0.9608020438568527}, "probs_as_returned": {"0": 0.007808933493590185, "1": 0.007335814130520291, "2": 0.024053208519036834, "3": 0.9608020438568527}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21314572900155326, "usage": {"input_tokens": 198, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5fc2e95b727b0c4e3b5d116e966eccbe3614dbc89d503b7a12cdc7350ae29746", "runtime": null}
{"task_id": "original-ordinal-05-0", "family": "ordinal", "split": "public", "group": "original-ordinal-05", "ts": 1791912613.138888, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.09261201394183338, "probs": {"0": 0.9450131409844158, "1": 0.028536924157435934, "2": 0.015274714790047252, "3": 0.011175220068100983}, "probs_as_returned": {"0": 0.9450131409844158, "1": 0.028536924157435934, "2": 0.015274714790047252, "3": 0.011175220068100983}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22023463000004995, "usage": {"input_tokens": 202, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5ce965fe88d7d9891b564eb9199cbb2cb35339cba7fe80bf3b063b8640ad2c70", "runtime": null}
{"task_id": "original-ordinal-05-1", "family": "ordinal", "split": "public", "group": "original-ordinal-05", "ts": 1791912613.353172, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.16144891171153158, "probs": {"0": 0.9004829031372495, "1": 0.0540782016933103, "2": 0.02894597549009933, "3": 0.01649291967934087}, "probs_as_returned": {"0": 0.9004829031372495, "1": 0.0540782016933103, "2": 0.02894597549009933, "3": 0.01649291967934087}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2079424139992625, "usage": {"input_tokens": 201, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "f768f96561de3750adf06f2cde7638601d460309868b66bb8ce7e9b40316c266", "runtime": null}
{"task_id": "original-ordinal-06-0", "family": "ordinal", "split": "public", "group": "original-ordinal-06", "ts": 1791912613.567179, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.9027036158936557, "probs": {"0": 0.031065571372513527, "1": 0.048115298186130105, "2": 0.9078690736165436, "3": 0.012950056824812843}, "probs_as_returned": {"0": 0.031065571372513527, "1": 0.048115298186130105, "2": 0.9078690736165436, "3": 0.012950056824812843}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20815279499947792, "usage": {"input_tokens": 202, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "79e91508ba177d11d8d38a842e08302252621411b1c6bc0b905d9fc3f2b3494f", "runtime": null}
{"task_id": "original-ordinal-06-1", "family": "ordinal", "split": "public", "group": "original-ordinal-06", "ts": 1791912613.7797675, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.9696346219751697, "probs": {"0": 0.01212082047149501, "1": 0.015563441556508687, "2": 0.9628760334973282, "3": 0.009439704474668228}, "probs_as_returned": {"0": 0.01212082047149501, "1": 0.015563441556508687, "2": 0.9628760334973282, "3": 0.009439704474668228}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2059316859995306, "usage": {"input_tokens": 209, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "7370c6ca92f4b2ae00fb93066d7c7dcbdd190c19ffd7f11bd372817b891b1b1b", "runtime": null}
{"task_id": "original-extraction-01-0", "family": "extraction", "split": "public", "group": "original-extraction-01", "ts": 1791912613.990049, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "pickup", "ordinal_ev": null, "probs": {"courier": 0.015714323893226054, "pickup": 0.972210793124608, "post": 0.005101694024804107, "unknown": 0.006973188957361904}, "probs_as_returned": {"courier": 0.015714323893226054, "pickup": 0.972210793124608, "post": 0.005101694024804107, "unknown": 0.006973188957361904}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20462053499977628, "usage": {"input_tokens": 197, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0da56836322820376998ad0c3b3934173e926624107915bc0783c066bb41a9bb", "runtime": null}
{"task_id": "original-extraction-01-1", "family": "extraction", "split": "public", "group": "original-extraction-01", "ts": 1791912614.2229004, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "pickup", "ordinal_ev": null, "probs": {"courier": 0.019869110835437295, "pickup": 0.9573473726962775, "post": 0.007309437390714121, "unknown": 0.015474079077571099}, "probs_as_returned": {"courier": 0.019869110835437295, "pickup": 0.9573473726962775, "post": 0.007309437390714121, "unknown": 0.015474079077571099}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2271388000008301, "usage": {"input_tokens": 199, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "dabcae6858db8aedd1a1ad280a12a1f522dc73f8cd68525472874ba696571d5a", "runtime": null}
{"task_id": "original-extraction-02-0", "family": "extraction", "split": "public", "group": "original-extraction-02", "ts": 1791912614.4375007, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.017472092244236192, "pickup": 0.008785519713623126, "post": 0.019798474298397137, "unknown": 0.9539439137437435}, "probs_as_returned": {"courier": 0.017472092244236192, "pickup": 0.008785519713623126, "post": 0.019798474298397137, "unknown": 0.9539439137437435}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2089694389997021, "usage": {"input_tokens": 200, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1b700336464bbe4fb842e89950e7f5c47a76c58d6a4390e2e17ecd3ba06d2ea8", "runtime": null}
{"task_id": "original-extraction-02-1", "family": "extraction", "split": "public", "group": "original-extraction-02", "ts": 1791912614.658322, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.005163504742566916, "pickup": 0.003333809228241761, "post": 0.007512855069775229, "unknown": 0.9839898309594161}, "probs_as_returned": {"courier": 0.005163504742566916, "pickup": 0.003333809228241761, "post": 0.007512855069775229, "unknown": 0.9839898309594161}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2153606949996174, "usage": {"input_tokens": 199, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "8eaa41fb227e62220aad22fd3ec050d13001f16dab8d4799bb346d78849e4e81", "runtime": null}
{"task_id": "original-extraction-03-0", "family": "extraction", "split": "public", "group": "original-extraction-03", "ts": 1791912614.8812575, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9742044602550028, "pickup": 0.009550764462699044, "post": 0.003981350963394447, "unknown": 0.012263424318903614}, "probs_as_returned": {"courier": 0.9742044602550028, "pickup": 0.009550764462699044, "post": 0.003981350963394447, "unknown": 0.012263424318903614}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2164891449992865, "usage": {"input_tokens": 203, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "3140f8582fdc259e114829a1c0dfde656e85dcbe4351a4e1236d92db4244510b", "runtime": null}
{"task_id": "original-extraction-03-1", "family": "extraction", "split": "public", "group": "original-extraction-03", "ts": 1791912615.1065698, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9853348473391993, "pickup": 0.006639133977723494, "post": 0.0028554559423102136, "unknown": 0.005170562740767029}, "probs_as_returned": {"courier": 0.9853348473391993, "pickup": 0.006639133977723494, "post": 0.0028554559423102136, "unknown": 0.005170562740767029}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21929472099873237, "usage": {"input_tokens": 202, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "562be7156f47145e091069f5ad6e473f21a64e8b0c2c3f8f1cb5a9aef597b4d6", "runtime": null}
{"task_id": "original-extraction-04-0", "family": "extraction", "split": "public", "group": "original-extraction-04", "ts": 1791912615.3288572, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "post", "ordinal_ev": null, "probs": {"courier": 0.006527588329641671, "pickup": 0.006948581607000444, "post": 0.9687800053232315, "unknown": 0.017743824740126295}, "probs_as_returned": {"courier": 0.006527588329641671, "pickup": 0.006948581607000444, "post": 0.9687800053232315, "unknown": 0.017743824740126295}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21620036800050002, "usage": {"input_tokens": 200, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "80f39af9d64be86910766eb613e6d1e1d357cd198cc4949539f3040643fb924e", "runtime": null}
{"task_id": "original-extraction-04-1", "family": "extraction", "split": "public", "group": "original-extraction-04", "ts": 1791912615.5561454, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "post", "ordinal_ev": null, "probs": {"courier": 0.01002497341081251, "pickup": 0.009417590976475038, "post": 0.9606203953590634, "unknown": 0.01993704025364903}, "probs_as_returned": {"courier": 0.01002497341081251, "pickup": 0.009417590976475038, "post": 0.9606203953590634, "unknown": 0.01993704025364903}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22167432100104634, "usage": {"input_tokens": 197, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "d6034e6339c874005a7efd76c076bdddf6adc9ce690cf4a633e0a6f6ddd1aa2c", "runtime": null}
{"task_id": "original-extraction-05-0", "family": "extraction", "split": "public", "group": "original-extraction-05", "ts": 1791912615.770271, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.006190654229801513, "pickup": 0.006190654229801513, "post": 0.009588272839484135, "unknown": 0.9780304187009129}, "probs_as_returned": {"courier": 0.006190654229801513, "pickup": 0.006190654229801513, "post": 0.009588272839484135, "unknown": 0.9780304187009129}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20853290100058075, "usage": {"input_tokens": 200, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "dcb08a99f0dbae9c791a01450f3ee4c1ada5bd763a506bd83cba9ea686f20a4a", "runtime": null}
{"task_id": "original-extraction-05-1", "family": "extraction", "split": "public", "group": "original-extraction-05", "ts": 1791912615.9826958, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.0074578329790909404, "pickup": 0.0061827606981924655, "post": 0.009576047098564817, "unknown": 0.9767833592241517}, "probs_as_returned": {"courier": 0.0074578329790909404, "pickup": 0.0061827606981924655, "post": 0.009576047098564817, "unknown": 0.9767833592241517}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2068135669996991, "usage": {"input_tokens": 196, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "f32780c7901a9a2f4bf1e510c7a037894c5329041f84af3859fa608e96b7dd5a", "runtime": null}
{"task_id": "original-extraction-06-0", "family": "extraction", "split": "public", "group": "original-extraction-06", "ts": 1791912616.1958225, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9803016255726286, "pickup": 0.007967416646019878, "post": 0.0037635411353317844, "unknown": 0.007967416646019878}, "probs_as_returned": {"courier": 0.9803016255726286, "pickup": 0.007967416646019878, "post": 0.0037635411353317844, "unknown": 0.007967416646019878}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2072897029993328, "usage": {"input_tokens": 200, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "48a3e3c5502079b00ed888f4494929d6bbcaaa55be38bfc7243c2fddc0e56004", "runtime": null}
{"task_id": "original-extraction-06-1", "family": "extraction", "split": "public", "group": "original-extraction-06", "ts": 1791912616.4227855, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9706809230429699, "pickup": 0.012219069531282035, "post": 0.004092875616566332, "unknown": 0.01300713180918177}, "probs_as_returned": {"courier": 0.9706809230429699, "pickup": 0.012219069531282035, "post": 0.004092875616566332, "unknown": 0.01300713180918177}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22031887099910819, "usage": {"input_tokens": 199, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "adab7385ed7c7c7ac1df3769c9e8c7a583b2f9b1a7411a056841ddc9a6d2999a", "runtime": null}
{"task_id": "original-adequacy-01-0", "family": "adequacy", "split": "public", "group": "original-adequacy-01", "ts": 1791912616.6484675, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9149009549929797, "no": 0.08509904500702026}, "probs_as_returned": {"yes": 0.9149009549929797, "no": 0.08509904500702026}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21941594000054465, "usage": {"input_tokens": 167, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c2bce41fcf69f89010c44abb7945f15275d7103932446f300c0d406350b017b3", "runtime": null}
{"task_id": "original-adequacy-01-1", "family": "adequacy", "split": "public", "group": "original-adequacy-01", "ts": 1791912616.8591537, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7310585786300049, "no": 0.2689414213699951}, "probs_as_returned": {"yes": 0.7310585786300049, "no": 0.2689414213699951}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2056481649997295, "usage": {"input_tokens": 163, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "fa44e25e86bccfb50f5dad1f4c6d8ed7c17ea8f13705da5b1742b9b913260b6e", "runtime": null}
{"task_id": "original-adequacy-02-0", "family": "adequacy", "split": "public", "group": "original-adequacy-02", "ts": 1791912617.0828433, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.017986209962091555, "no": 0.9820137900379085}, "probs_as_returned": {"yes": 0.017986209962091555, "no": 0.9820137900379085}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2170573309995234, "usage": {"input_tokens": 164, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "16992d8aad99d03d14fa7af68ae1327f6cd9e569c64a527055d49b96e25c03bc", "runtime": null}
{"task_id": "original-adequacy-02-1", "family": "adequacy", "split": "public", "group": "original-adequacy-02", "ts": 1791912617.307837, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.01495708659314999, "no": 0.98504291340685}, "probs_as_returned": {"yes": 0.01495708659314999, "no": 0.98504291340685}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2194050830003107, "usage": {"input_tokens": 165, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "ac05876b44a6ad1d3372224bf71d1f1605a5dd8fab12c03a506e4a7caeadd186", "runtime": null}
{"task_id": "original-adequacy-03-0", "family": "adequacy", "split": "public", "group": "original-adequacy-03", "ts": 1791912617.5219877, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.10669059394565118, "no": 0.8933094060543488}, "probs_as_returned": {"yes": 0.10669059394565118, "no": 0.8933094060543488}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20841913299955195, "usage": {"input_tokens": 167, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "bd26a025b76470362204c22e25a7305e832ee4fbe03901d1c3e4e383eff4a324", "runtime": null}
{"task_id": "original-adequacy-03-1", "family": "adequacy", "split": "public", "group": "original-adequacy-03", "ts": 1791912617.7460194, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.30735801686526387, "no": 0.6926419831347361}, "probs_as_returned": {"yes": 0.30735801686526387, "no": 0.6926419831347361}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21800626699950953, "usage": {"input_tokens": 171, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c3f6d1fdb24664de8025ff97e5df33fe11f3d4be10704e1325ee3acdd63a870b", "runtime": null}
{"task_id": "original-adequacy-04-0", "family": "adequacy", "split": "public", "group": "original-adequacy-04", "ts": 1791912617.972185, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9688561694652216, "no": 0.031143830534778427}, "probs_as_returned": {"yes": 0.9688561694652216, "no": 0.031143830534778427}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21978654000122333, "usage": {"input_tokens": 171, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "14b74d2d2f6b8d4575fe5a0847ac35d4d7816ff91db739364166055425603d8f", "runtime": null}
{"task_id": "original-adequacy-04-1", "family": "adequacy", "split": "public", "group": "original-adequacy-04", "ts": 1791912618.1947086, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9496693674025232, "no": 0.05033063259747683}, "probs_as_returned": {"yes": 0.9496693674025232, "no": 0.05033063259747683}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21710912799971993, "usage": {"input_tokens": 168, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "aa1bfadd3896c2a945d178ab8cc8dba38be35d014748d3b342fdaf399ca143ac", "runtime": null}
{"task_id": "original-adequacy-05-0", "family": "adequacy", "split": "public", "group": "original-adequacy-05", "ts": 1791912618.4067636, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5774953651858118, "no": 0.4225046348141882}, "probs_as_returned": {"yes": 0.5774953651858118, "no": 0.4225046348141882}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20524841500082402, "usage": {"input_tokens": 160, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "aee9bc012267d7f34612548163a542a178974e062d8c7006c265f6edf702ec29", "runtime": null}
{"task_id": "original-adequacy-05-1", "family": "adequacy", "split": "public", "group": "original-adequacy-05", "ts": 1791912618.6254456, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.42250463481418826, "no": 0.5774953651858117}, "probs_as_returned": {"yes": 0.42250463481418826, "no": 0.5774953651858117}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21281000499948277, "usage": {"input_tokens": 161, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "a34c99c601e494dcd935decde402353cf9ccfbf045bbbf7a2b6e80a2d6f7fdc2", "runtime": null}
{"task_id": "original-adequacy-06-0", "family": "adequacy", "split": "public", "group": "original-adequacy-06", "ts": 1791912618.8396251, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7662936430859597, "no": 0.2337063569140403}, "probs_as_returned": {"yes": 0.7662936430859597, "no": 0.2337063569140403}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20826722899982997, "usage": {"input_tokens": 176, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "766be48c9a2e79f19e62b917703a1829924a9dc6992332ebbd04b5f8128169fc", "runtime": null}
{"task_id": "original-adequacy-06-1", "family": "adequacy", "split": "public", "group": "original-adequacy-06", "ts": 1791912619.0626762, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5312093733737563, "no": 0.4687906266262437}, "probs_as_returned": {"yes": 0.5312093733737563, "no": 0.4687906266262437}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21754225099903124, "usage": {"input_tokens": 178, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "981e90fdbd5e8457194ddeeee6d436f4617f89a2da03deacab42fc33df39c428", "runtime": null}
{"task_id": "original-routing-01-0", "family": "routing", "split": "public", "group": "original-routing-01", "ts": 1791912620.5313606, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "math", "ordinal_ev": null, "probs": {"coding": 0.007345734510540175, "coding_agent": 0.005374254174003903, "document": 0.006900678955161194, "general": 0.006900678955161194, "math": 0.9621013572294865, "tools": 0.011377296175646997}, "probs_as_returned": {"coding": 0.007345734510540175, "coding_agent": 0.005374254174003903, "document": 0.006900678955161194, "general": 0.006900678955161194, "math": 0.9621013572294865, "tools": 0.011377296175646997}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.4641905120006413, "usage": {"input_tokens": 259, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "4cb04cec20db9071ce8bcd3081a0f797e19578ca5fe717ca7aa45a10b283105e", "runtime": null}
{"task_id": "original-routing-01-1", "family": "routing", "split": "public", "group": "original-routing-01", "ts": 1791912620.7918935, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "math", "ordinal_ev": null, "probs": {"coding": 0.012732471280249057, "coding_agent": 0.006815200766043127, "document": 0.00822070131987826, "general": 0.009315274983744468, "math": 0.9501838803698359, "tools": 0.012732471280249057}, "probs_as_returned": {"coding": 0.012732471280249057, "coding_agent": 0.006815200766043127, "document": 0.00822070131987826, "general": 0.009315274983744468, "math": 0.9501838803698359, "tools": 0.012732471280249057}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.25415638399863383, "usage": {"input_tokens": 255, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "39e5589c349246d5602442766f90fee9d7ed936346ae2e8001b18d2e0fafb8ac", "runtime": null}
{"task_id": "original-routing-02-0", "family": "routing", "split": "public", "group": "original-routing-02", "ts": 1791912622.2116885, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding", "ordinal_ev": null, "probs": {"coding": 0.9520906953736354, "coding_agent": 0.022391027005668144, "document": 0.005318335074774821, "general": 0.007269302164077605, "math": 0.007269302164077605, "tools": 0.0056613382177662964}, "probs_as_returned": {"coding": 0.9520906953736354, "coding_agent": 0.022391027005668144, "document": 0.005318335074774821, "general": 0.007269302164077605, "math": 0.007269302164077605, "tools": 0.0056613382177662964}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.4125522899994394, "usage": {"input_tokens": 260, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "f13f92dc5a9381da6eaf9f890ef28553a5e64dfb9e3db70c67bacd8614f75671", "runtime": null}
{"task_id": "original-routing-02-1", "family": "routing", "split": "public", "group": "original-routing-02", "ts": 1791912622.4338775, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding", "ordinal_ev": null, "probs": {"coding": 0.943238175238013, "coding_agent": 0.01839021670827909, "document": 0.007201712338767772, "general": 0.012639398778242822, "math": 0.009843573666047267, "tools": 0.008686923270650005}, "probs_as_returned": {"coding": 0.943238175238013, "coding_agent": 0.01839021670827909, "document": 0.007201712338767772, "general": 0.012639398778242822, "math": 0.009843573666047267, "tools": 0.008686923270650005}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21435886199833476, "usage": {"input_tokens": 254, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "4573d20d5a16df17954d0bfa67558931dcc3ed7d14306cdb311fbffc434ca184", "runtime": null}
{"task_id": "original-routing-03-0", "family": "routing", "split": "public", "group": "original-routing-03", "ts": 1791912622.6775966, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding_agent", "ordinal_ev": null, "probs": {"coding": 0.01850053770291334, "coding_agent": 0.9488965628100527, "document": 0.0105412886314069, "general": 0.004979352172265726, "math": 0.005137413777351499, "tools": 0.01194484490600965}, "probs_as_returned": {"coding": 0.01850053770291334, "coding_agent": 0.9488965628100527, "document": 0.0105412886314069, "general": 0.004979352172265726, "math": 0.005137413777351499, "tools": 0.01194484490600965}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.23770265199891583, "usage": {"input_tokens": 259, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "05d47dc976bcac415bc2b3cf1328feae34bbbdc98abd35a7e07f91c865d954ff", "runtime": null}
{"task_id": "original-routing-03-1", "family": "routing", "split": "public", "group": "original-routing-03", "ts": 1791912622.9248366, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding_agent", "ordinal_ev": null, "probs": {"coding": 0.010721787866510405, "coding_agent": 0.9651445861560088, "document": 0.008888674340290439, "general": 0.0030718436546726146, "math": 0.0038229711961537587, "tools": 0.008350136786363791}, "probs_as_returned": {"coding": 0.010721787866510405, "coding_agent": 0.9651445861560088, "document": 0.008888674340290439, "general": 0.0030718436546726146, "math": 0.0038229711961537587, "tools": 0.008350136786363791}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.24067505600032746, "usage": {"input_tokens": 261, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "16deedf7082cb9324f797961d075ec097e46dfd7205ab1133b60b59ec96ab4f4", "runtime": null}
{"task_id": "original-routing-04-0", "family": "routing", "split": "public", "group": "original-routing-04", "ts": 1791912623.1637497, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "document", "ordinal_ev": null, "probs": {"coding": 0.024866633638513103, "coding_agent": 0.009737913649352295, "document": 0.8765790515668891, "general": 0.06349917351682002, "math": 0.007124409819198966, "tools": 0.018192817809226484}, "probs_as_returned": {"coding": 0.024866633638513103, "coding_agent": 0.009737913649352295, "document": 0.8765790515668891, "general": 0.06349917351682002, "math": 0.007124409819198966, "tools": 0.018192817809226484}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22964875800062146, "usage": {"input_tokens": 255, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "f5bf9f4f1d1de031c27b6f951a1a1bb01e72833d2b1f6c525b87376305cefb0a", "runtime": null}
{"task_id": "original-routing-04-1", "family": "routing", "split": "public", "group": "original-routing-04", "ts": 1791912623.3918514, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "document", "ordinal_ev": null, "probs": {"coding": 0.023304218510243654, "coding_agent": 0.0062722453220191415, "document": 0.8744843480477424, "general": 0.08133971495638083, "math": 0.0048848295684044635, "tools": 0.00971464359520945}, "probs_as_returned": {"coding": 0.023304218510243654, "coding_agent": 0.0062722453220191415, "document": 0.8744843480477424, "general": 0.08133971495638083, "math": 0.0048848295684044635, "tools": 0.00971464359520945}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22080730000016047, "usage": {"input_tokens": 254, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "efb78b45af4dc5e886e520b0d6ea8d8d436ffa81d784bc798a1f4e8b6b7ea5b2", "runtime": null}
{"task_id": "original-routing-05-0", "family": "routing", "split": "public", "group": "original-routing-05", "ts": 1791912623.6227922, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "tools", "ordinal_ev": null, "probs": {"coding": 0.0037391800376907268, "coding_agent": 0.0065624653383621, "document": 0.005440474852551775, "general": 0.004510312069493637, "math": 0.005791355334423317, "tools": 0.9739562123674785}, "probs_as_returned": {"coding": 0.0037391800376907268, "coding_agent": 0.0065624653383621, "document": 0.005440474852551775, "general": 0.004510312069493637, "math": 0.005791355334423317, "tools": 0.9739562123674785}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.221713490998809, "usage": {"input_tokens": 256, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2d0c929dc613bd602a735f0fbcff604465a217851e2585171db4c765df6ab6c6", "runtime": null}
{"task_id": "original-routing-05-1", "family": "routing", "split": "public", "group": "original-routing-05", "ts": 1791912623.8773637, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "tools", "ordinal_ev": null, "probs": {"coding": 0.012379797740960092, "coding_agent": 0.014932886516859586, "document": 0.014932886516859586, "general": 0.07583545516726947, "math": 0.014028148659449118, "tools": 0.8678908253986021}, "probs_as_returned": {"coding": 0.012379797740960092, "coding_agent": 0.014932886516859586, "document": 0.014932886516859586, "general": 0.07583545516726947, "math": 0.014028148659449118, "tools": 0.8678908253986021}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2473705729989888, "usage": {"input_tokens": 256, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1d8ad2b724c361f67ddccd88007d9fd103466e1a775824471789663b30cdf8da", "runtime": null}
{"task_id": "original-routing-06-0", "family": "routing", "split": "public", "group": "original-routing-06", "ts": 1791912624.1234257, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "general", "ordinal_ev": null, "probs": {"coding": 0.04512890185356874, "coding_agent": 0.014651209335933894, "document": 0.017672731891602542, "general": 0.9064382244827673, "math": 0.00538987870298918, "tools": 0.010719053733138187}, "probs_as_returned": {"coding": 0.04512890185356874, "coding_agent": 0.014651209335933894, "document": 0.017672731891602542, "general": 0.9064382244827673, "math": 0.00538987870298918, "tools": 0.010719053733138187}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.23836264400051732, "usage": {"input_tokens": 255, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "814aa7e538abc66c4df7c021b60ce96bad325306533622c61a52abb014a10a25", "runtime": null}
{"task_id": "original-routing-06-1", "family": "routing", "split": "public", "group": "original-routing-06", "ts": 1791912624.3550458, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "general", "ordinal_ev": null, "probs": {"coding": 0.05728175230774756, "coding_agent": 0.012781286566010405, "document": 0.019796043888962327, "general": 0.8960373650313415, "math": 0.004149476219669406, "tools": 0.009954075986268929}, "probs_as_returned": {"coding": 0.05728175230774756, "coding_agent": 0.012781286566010405, "document": 0.019796043888962327, "general": 0.8960373650313415, "math": 0.004149476219669406, "tools": 0.009954075986268929}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22402690699891536, "usage": {"input_tokens": 254, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "eecb09f4749700011ecf6b145a3a52b1fe1067fa154176b2ae376830d4caccb2", "runtime": null}
{"task_id": "easy-intent-00", "family": "intent", "split": "public", "group": null, "ts": 1791912624.5854666, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "track_order", "ordinal_ev": null, "probs": {"billing_question": 0.013786427191723485, "cancel_order": 0.007379342712814964, "change_address": 0.007855269428247413, "report_damage": 0.004475797603849274, "track_order": 0.9665031630633648}, "probs_as_returned": {"billing_question": 0.013786427191723485, "cancel_order": 0.007379342712814964, "change_address": 0.007855269428247413, "report_damage": 0.004475797603849274, "track_order": 0.9665031630633648}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22219949800091854, "usage": {"input_tokens": 245, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "65fc9c27afb4c518496d82eba523889a45b6a50861dca6627fa7503621a438ae", "runtime": null}
{"task_id": "easy-intent-01", "family": "intent", "split": "public", "group": null, "ts": 1791912624.8117712, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cancel_order", "ordinal_ev": null, "probs": {"billing_question": 0.018839661104342757, "cancel_order": 0.9662902750984347, "change_address": 0.008360048937891246, "report_damage": 0.0028002656309957423, "track_order": 0.003709749228335429}, "probs_as_returned": {"billing_question": 0.018839661104342757, "cancel_order": 0.9662902750984347, "change_address": 0.008360048937891246, "report_damage": 0.0028002656309957423, "track_order": 0.003709749228335429}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21477718599999207, "usage": {"input_tokens": 245, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "69152c7cd5cff606f1761880742486c629192defca94de0c4f2d486e228b9ffc", "runtime": null}
{"task_id": "easy-intent-02", "family": "intent", "split": "public", "group": null, "ts": 1791912625.0372386, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "change_address", "ordinal_ev": null, "probs": {"billing_question": 0.011504617642382052, "cancel_order": 0.006977903328375587, "change_address": 0.9728680766731138, "report_damage": 0.003853564210096113, "track_order": 0.004795838146032382}, "probs_as_returned": {"billing_question": 0.011504617642382052, "cancel_order": 0.006977903328375587, "change_address": 0.9728680766731138, "report_damage": 0.003853564210096113, "track_order": 0.004795838146032382}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21776380899973447, "usage": {"input_tokens": 248, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "de0bfaef2a3f3ffc8ae530489ae6e73982b8120c026691682a5977aa8b3bae54", "runtime": null}
{"task_id": "easy-intent-03", "family": "intent", "split": "public", "group": null, "ts": 1791912625.2630346, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "report_damage", "ordinal_ev": null, "probs": {"billing_question": 0.0063312077031251095, "cancel_order": 0.00372192500290584, "change_address": 0.01472049894344899, "report_damage": 0.9694617381367755, "track_order": 0.005764630213744498}, "probs_as_returned": {"billing_question": 0.0063312077031251095, "cancel_order": 0.00372192500290584, "change_address": 0.01472049894344899, "report_damage": 0.9694617381367755, "track_order": 0.005764630213744498}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21958713599997282, "usage": {"input_tokens": 237, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "81a6a50b9ab22f8be4d3d860e6d70830e165c77719fe40536dbc8055b5f00342", "runtime": null}
{"task_id": "easy-intent-04", "family": "intent", "split": "public", "group": null, "ts": 1791912625.4797177, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "billing_question", "ordinal_ev": null, "probs": {"billing_question": 0.9808307699551909, "cancel_order": 0.007726453318958719, "change_address": 0.005146927511823108, "report_damage": 0.001893448816799407, "track_order": 0.004402400397227914}, "probs_as_returned": {"billing_question": 0.9808307699551909, "cancel_order": 0.007726453318958719, "change_address": 0.005146927511823108, "report_damage": 0.001893448816799407, "track_order": 0.004402400397227914}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20992651300002763, "usage": {"input_tokens": 242, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "36a60ee85df3cabd12dd312f9777439f97dcc552d7d76568b399ed7ce8e03676", "runtime": null}
{"task_id": "easy-intent-05", "family": "intent", "split": "public", "group": null, "ts": 1791912625.6992135, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "track_order", "ordinal_ev": null, "probs": {"billing_question": 0.016526619865564297, "cancel_order": 0.008846062157832268, "change_address": 0.008573897437614556, "report_damage": 0.005535724246339717, "track_order": 0.9605176962926492}, "probs_as_returned": {"billing_question": 0.016526619865564297, "cancel_order": 0.008846062157832268, "change_address": 0.008573897437614556, "report_damage": 0.005535724246339717, "track_order": 0.9605176962926492}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21098394099863071, "usage": {"input_tokens": 241, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "6a202df1bf03c87d1f4bc1d0ef86a58bbb1c64f0c8c46de001c432bc16ac297b", "runtime": null}
{"task_id": "easy-intent-06", "family": "intent", "split": "public", "group": null, "ts": 1791912625.9185746, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_alarm", "ordinal_ev": null, "probs": {"play_music": 0.011571401468822753, "send_message": 0.004823677786804312, "set_alarm": 0.9785155353546511, "turn_off_lights": 0.0026638878765596163, "weather": 0.0024254975131621885}, "probs_as_returned": {"play_music": 0.011571401468822753, "send_message": 0.004823677786804312, "set_alarm": 0.9785155353546511, "turn_off_lights": 0.0026638878765596163, "weather": 0.0024254975131621885}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21217866000006325, "usage": {"input_tokens": 231, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1189a6b75bdcd023eb9e7fcddcca4b458756bf728931be1107597ae47b6581ca", "runtime": null}
{"task_id": "easy-intent-07", "family": "intent", "split": "public", "group": null, "ts": 1791912626.145797, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "play_music", "ordinal_ev": null, "probs": {"play_music": 0.9816179114743363, "send_message": 0.005483272753308888, "set_alarm": 0.005151058051427584, "turn_off_lights": 0.002433187534721696, "weather": 0.0053145701862055824}, "probs_as_returned": {"play_music": 0.9816179114743363, "send_message": 0.005483272753308888, "set_alarm": 0.005151058051427584, "turn_off_lights": 0.002433187534721696, "weather": 0.0053145701862055824}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22097225499965134, "usage": {"input_tokens": 224, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9e44054ad95797db443afd29ac11939e50d7349c0e3c09ec7913ae91a143dbf8", "runtime": null}
{"task_id": "easy-intent-08", "family": "intent", "split": "public", "group": null, "ts": 1791912626.3754895, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "weather", "ordinal_ev": null, "probs": {"play_music": 0.007927716766767371, "send_message": 0.004378101008645739, "set_alarm": 0.0069961854912401695, "turn_off_lights": 0.005280997871648997, "weather": 0.9754169988616979}, "probs_as_returned": {"play_music": 0.007927716766767371, "send_message": 0.004378101008645739, "set_alarm": 0.0069961854912401695, "turn_off_lights": 0.005280997871648997, "weather": 0.9754169988616979}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22331140800088178, "usage": {"input_tokens": 226, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "3946414046bc0c46a628568c0aed333d641fec086066a3ce790c426d9dff8013", "runtime": null}
{"task_id": "easy-intent-09", "family": "intent", "split": "public", "group": null, "ts": 1791912626.5955834, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "send_message", "ordinal_ev": null, "probs": {"play_music": 0.018848615133123477, "send_message": 0.9667495291628173, "set_alarm": 0.006934018002029056, "turn_off_lights": 0.00255088266765917, "weather": 0.0049169550343710205}, "probs_as_returned": {"play_music": 0.018848615133123477, "send_message": 0.9667495291628173, "set_alarm": 0.006934018002029056, "turn_off_lights": 0.00255088266765917, "weather": 0.0049169550343710205}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21335113600071054, "usage": {"input_tokens": 229, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "35dfbc784671e14a9f11fbbe1fd797e759fa1f62f76c7f15a82d416bf8acd6cd", "runtime": null}
{"task_id": "easy-intent-10", "family": "intent", "split": "public", "group": null, "ts": 1791912626.811175, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "turn_off_lights", "ordinal_ev": null, "probs": {"play_music": 0.008344491555621152, "send_message": 0.005061189968197397, "set_alarm": 0.013757740720930197, "turn_off_lights": 0.9644920861996301, "weather": 0.008344491555621152}, "probs_as_returned": {"play_music": 0.008344491555621152, "send_message": 0.005061189968197397, "set_alarm": 0.013757740720930197, "turn_off_lights": 0.9644920861996301, "weather": 0.008344491555621152}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20854645300096308, "usage": {"input_tokens": 227, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "f571cd2088fe223079363c2b936dccab4cadccc8ccc6dfb4bbf769d6ca19075b", "runtime": null}
{"task_id": "easy-intent-11", "family": "intent", "split": "public", "group": null, "ts": 1791912627.0371907, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_alarm", "ordinal_ev": null, "probs": {"play_music": 0.010891140058107054, "send_message": 0.004006626518296333, "set_alarm": 0.9803891846229955, "turn_off_lights": 0.0024301418253644064, "weather": 0.002282906975236708}, "probs_as_returned": {"play_music": 0.010891140058107054, "send_message": 0.004006626518296333, "set_alarm": 0.9803891846229955, "turn_off_lights": 0.0024301418253644064, "weather": 0.002282906975236708}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2191638749991398, "usage": {"input_tokens": 227, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "370be3d6c42361de18d9932fafb4e524aaff480cc487f15becee5e682fc6cafd", "runtime": null}
{"task_id": "easy-fact-00", "family": "fact", "split": "public", "group": null, "ts": 1791912627.2537148, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9724147177732099, "no": 0.02758528222679013}, "probs_as_returned": {"yes": 0.9724147177732099, "no": 0.02758528222679013}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2088159260001703, "usage": {"input_tokens": 165, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e8c70fb9018857ed5f1757d49e3d267f5e500cfc3fe110a011754465785c21ab", "runtime": null}
{"task_id": "easy-fact-01", "family": "fact", "split": "public", "group": null, "ts": 1791912627.4793549, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.04742587317756679, "no": 0.9525741268224333}, "probs_as_returned": {"yes": 0.04742587317756679, "no": 0.9525741268224333}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21858379500008596, "usage": {"input_tokens": 163, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "33e71b617e4d3fac546e6cd2fcbc390703bf8dfea774788659914f280c8873fb", "runtime": null}
{"task_id": "easy-fact-02", "family": "fact", "split": "public", "group": null, "ts": 1791912627.7058537, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9796676466573412, "no": 0.020332353342658815}, "probs_as_returned": {"yes": 0.9796676466573412, "no": 0.020332353342658815}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21682083800078544, "usage": {"input_tokens": 171, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "29de21dba8c538a0067e133d8319d893689392853f55badd1fd072885b0df01c", "runtime": null}
{"task_id": "easy-fact-03", "family": "fact", "split": "public", "group": null, "ts": 1791912627.9404113, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.053403329799824234, "no": 0.9465966702001758}, "probs_as_returned": {"yes": 0.053403329799824234, "no": 0.9465966702001758}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22754783100026543, "usage": {"input_tokens": 174, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5c1333585bf9c5acb8d821da66224e98f484ada0b9d53bf2a6d1128d00a38149", "runtime": null}
{"task_id": "easy-fact-04", "family": "fact", "split": "public", "group": null, "ts": 1791912628.1764243, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9770226300899744, "no": 0.022977369910025636}, "probs_as_returned": {"yes": 0.9770226300899744, "no": 0.022977369910025636}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22955908900075883, "usage": {"input_tokens": 163, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e44ef30c4d777fa09ac82c445cb1358c087af41adf94148e86d7d61f90fabb65", "runtime": null}
{"task_id": "easy-fact-05", "family": "fact", "split": "public", "group": null, "ts": 1791912628.390178, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.06371499425196533, "no": 0.9362850057480346}, "probs_as_returned": {"yes": 0.06371499425196533, "no": 0.9362850057480346}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20793076299923996, "usage": {"input_tokens": 164, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "550eb191be827ad663b4e6f8e1d3263d7ab65627f84692b0f325795f253ca1f4", "runtime": null}
{"task_id": "easy-fact-06", "family": "fact", "split": "public", "group": null, "ts": 1791912628.6027076, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9669140216112958, "no": 0.033085978388704196}, "probs_as_returned": {"yes": 0.9669140216112958, "no": 0.033085978388704196}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20644549000098777, "usage": {"input_tokens": 184, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2974f7452aa640f0845b76c7bbef69d817ce35a5653589f1b1d4f1e914d7f014", "runtime": null}
{"task_id": "easy-fact-07", "family": "fact", "split": "public", "group": null, "ts": 1791912628.8296204, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.027585282226789985, "no": 0.97241471777321}, "probs_as_returned": {"yes": 0.027585282226789985, "no": 0.97241471777321}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22079327299979923, "usage": {"input_tokens": 159, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9565e1c5a5027fdf9f87b9c8b4777d5303b2598758690f9038e2c82a7a355adb", "runtime": null}
{"task_id": "easy-fact-08", "family": "fact", "split": "public", "group": null, "ts": 1791912629.0491784, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9688561694652216, "no": 0.031143830534778427}, "probs_as_returned": {"yes": 0.9688561694652216, "no": 0.031143830534778427}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21297958200011635, "usage": {"input_tokens": 155, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "12cb9be56f6052d5f70bc52025e2876c80a1ad7571e40a4e8287fc3c3d5f4921", "runtime": null}
{"task_id": "easy-fact-09", "family": "fact", "split": "public", "group": null, "ts": 1791912629.27733, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.024423090054107213, "no": 0.9755769099458927}, "probs_as_returned": {"yes": 0.024423090054107213, "no": 0.9755769099458927}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22145563500089338, "usage": {"input_tokens": 155, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "d621c15f75c27c6186117cd12bb7fbf4617aeb861a0969ee4c9f00300e1b47c8", "runtime": null}
{"task_id": "easy-fact-10", "family": "fact", "split": "public", "group": null, "ts": 1791912629.5072737, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9688561694652216, "no": 0.031143830534778427}, "probs_as_returned": {"yes": 0.9688561694652216, "no": 0.031143830534778427}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2226223449997633, "usage": {"input_tokens": 164, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9d68825fcbee0712861464716be2c8342afdb787f65ed4f6d91fdbaca8a87d80", "runtime": null}
{"task_id": "easy-fact-11", "family": "fact", "split": "public", "group": null, "ts": 1791912629.7219584, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.033085978388704126, "no": 0.9669140216112959}, "probs_as_returned": {"yes": 0.033085978388704126, "no": 0.9669140216112959}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20690551699954085, "usage": {"input_tokens": 165, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "bb17745084f04cee5d3d3cce794c17ee85ffc3ed882afa68be086b30edb1de95", "runtime": null}
{"task_id": "easy-extraction-00", "family": "extraction", "split": "public", "group": null, "ts": 1791912629.945052, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "paypal", "ordinal_ev": null, "probs": {"bank_transfer": 0.008982879478184195, "cash": 0.004808188901706793, "credit_card": 0.010835420953488729, "paypal": 0.9753735106666201}, "probs_as_returned": {"bank_transfer": 0.008982879478184195, "cash": 0.004808188901706793, "credit_card": 0.010835420953488729, "paypal": 0.9753735106666201}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21499474800111784, "usage": {"input_tokens": 185, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c98d6fbdc3713efdd9ff0a3761bdc2f3262a6cdc2b0e8e35885d809522e88740", "runtime": null}
{"task_id": "easy-extraction-01", "family": "extraction", "split": "public", "group": null, "ts": 1791912630.1628487, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cash", "ordinal_ev": null, "probs": {"bank_transfer": 0.015713201539826152, "cash": 0.9721413555785513, "credit_card": 0.008953112358928435, "paypal": 0.0031923305226940262}, "probs_as_returned": {"bank_transfer": 0.015713201539826152, "cash": 0.9721413555785513, "credit_card": 0.008953112358928435, "paypal": 0.0031923305226940262}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2081525760004297, "usage": {"input_tokens": 187, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "da6ff6f2c724a1dffd2283d320be5fa0157205390258a15d5840c62468d11fa1", "runtime": null}
{"task_id": "easy-extraction-02", "family": "extraction", "split": "public", "group": null, "ts": 1791912630.394116, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "bank_transfer", "ordinal_ev": null, "probs": {"bank_transfer": 0.9846893550062124, "cash": 0.006845395358507434, "credit_card": 0.005331199265638949, "paypal": 0.0031340503696412627}, "probs_as_returned": {"bank_transfer": 0.9846893550062124, "cash": 0.006845395358507434, "credit_card": 0.005331199265638949, "paypal": 0.0031340503696412627}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21194981200096663, "usage": {"input_tokens": 188, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "6689ce2c34425e657e0e1bef911c9ad367649cff3c3570323482a53617439c00", "runtime": null}
{"task_id": "easy-extraction-03", "family": "extraction", "split": "public", "group": null, "ts": 1791912630.6218252, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "high", "ordinal_ev": null, "probs": {"high": 0.976966956679311, "low": 0.00845242033270976, "medium": 0.01019556508613546, "urgent": 0.004385057901843796}, "probs_as_returned": {"high": 0.976966956679311, "low": 0.00845242033270976, "medium": 0.01019556508613546, "urgent": 0.004385057901843796}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21822797400091076, "usage": {"input_tokens": 187, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "b2e55066cb0ea6c496ac6e620c61c6d0d0b8ac25d0ed8405bf01a2c1937936a2", "runtime": null}
{"task_id": "easy-extraction-04", "family": "extraction", "split": "public", "group": null, "ts": 1791912630.8380384, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "low", "ordinal_ev": null, "probs": {"high": 0.016696055531537263, "low": 0.970365200311863, "medium": 0.010126669576142057, "urgent": 0.0028120745804575166}, "probs_as_returned": {"high": 0.016696055531537263, "low": 0.970365200311863, "medium": 0.010126669576142057, "urgent": 0.0028120745804575166}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20821268699910433, "usage": {"input_tokens": 185, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c29a7f2108be6ff9a22a0e817c5d7e8f96f57b5f28029f8148d225e311026068", "runtime": null}
{"task_id": "easy-extraction-05", "family": "extraction", "split": "public", "group": null, "ts": 1791912631.057358, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "urgent", "ordinal_ev": null, "probs": {"high": 0.007647921101410536, "low": 0.004785946731980246, "medium": 0.016704595470754764, "urgent": 0.9708615366958544}, "probs_as_returned": {"high": 0.007647921101410536, "low": 0.004785946731980246, "medium": 0.016704595470754764, "urgent": 0.9708615366958544}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21192583500123874, "usage": {"input_tokens": 185, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "fd3c9338abf3d9f17230b6dd00bce8bd06aa8cf44e2ed9c90a2eaed1a63e4766", "runtime": null}
{"task_id": "easy-extraction-06", "family": "extraction", "split": "public", "group": null, "ts": 1791912631.2826576, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "M", "ordinal_ev": null, "probs": {"L": 0.01292512651411809, "M": 0.9645611284045675, "S": 0.016596190958032258, "XL": 0.005917554123282112}, "probs_as_returned": {"L": 0.01292512651411809, "M": 0.9645611284045675, "S": 0.016596190958032258, "XL": 0.005917554123282112}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21792216399990139, "usage": {"input_tokens": 177, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "b33d4426bc061e1fde8609c5909abd256187d3c78d0c19d8bb3f5de4aea4de73", "runtime": null}
{"task_id": "easy-extraction-07", "family": "extraction", "split": "public", "group": null, "ts": 1791912631.4971702, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "XL", "ordinal_ev": null, "probs": {"L": 0.02320378743337191, "M": 0.011667597050024688, "S": 0.0382565779022046, "XL": 0.9268720376143988}, "probs_as_returned": {"L": 0.02320378743337191, "M": 0.011667597050024688, "S": 0.0382565779022046, "XL": 0.9268720376143988}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20746643600068637, "usage": {"input_tokens": 175, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e02f5834d8006be2e3a16c4a5a49870b1b6640a75a688c5eaa88192644d6027d", "runtime": null}
{"task_id": "easy-extraction-08", "family": "extraction", "split": "public", "group": null, "ts": 1791912631.7250946, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "thursday", "ordinal_ev": null, "probs": {"friday": 0.005169568976011531, "monday": 0.005010518059468218, "thursday": 0.9851454692980184, "tuesday": 0.002519446702294301, "wednesday": 0.002154996964207525}, "probs_as_returned": {"friday": 0.005169568976011531, "monday": 0.005010518059468218, "thursday": 0.9851454692980184, "tuesday": 0.002519446702294301, "wednesday": 0.002154996964207525}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21901093400083482, "usage": {"input_tokens": 202, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "27abe1c65f6daa0a8e2740aa65bff37fc0e8ef77c8addb8760757dfc63e44005", "runtime": null}
{"task_id": "easy-extraction-09", "family": "extraction", "split": "public", "group": null, "ts": 1791912631.9545841, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "monday", "ordinal_ev": null, "probs": {"friday": 0.016670630378706593, "monday": 0.9688875049680202, "thursday": 0.0061327821876942515, "tuesday": 0.0028969211807126317, "wednesday": 0.005412161284866155}, "probs_as_returned": {"friday": 0.016670630378706593, "monday": 0.9688875049680202, "thursday": 0.0061327821876942515, "tuesday": 0.0028969211807126317, "wednesday": 0.005412161284866155}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2221628060015064, "usage": {"input_tokens": 204, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1272046926f7c8f4029cd4dd0342811773230772cb70e4c0bc397080b3690c5d", "runtime": null}
{"task_id": "easy-extraction-10", "family": "extraction", "split": "public", "group": null, "ts": 1791912632.1694334, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "friday", "ordinal_ev": null, "probs": {"friday": 0.9824220278072218, "monday": 0.004842935182509282, "thursday": 0.004014934283760505, "tuesday": 0.00167367361440489, "wednesday": 0.007046429112103436}, "probs_as_returned": {"friday": 0.9824220278072218, "monday": 0.004842935182509282, "thursday": 0.004014934283760505, "tuesday": 0.00167367361440489, "wednesday": 0.007046429112103436}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20741145099964342, "usage": {"input_tokens": 200, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0206f6205bb96009a1a5f42da44729d120f321468d4386431cf0391e8cf6715e", "runtime": null}
{"task_id": "easy-extraction-11", "family": "extraction", "split": "public", "group": null, "ts": 1791912632.3818138, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "credit_card", "ordinal_ev": null, "probs": {"bank_transfer": 0.01578127514702095, "cash": 0.005123429914541355, "credit_card": 0.9763529205234635, "paypal": 0.0027423744149743337}, "probs_as_returned": {"bank_transfer": 0.01578127514702095, "cash": 0.005123429914541355, "credit_card": 0.9763529205234635, "paypal": 0.0027423744149743337}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20509187199968437, "usage": {"input_tokens": 191, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e5c3c7812e549f59bf54aafb0f2dce3173cbb9701eed4c7e40e97df01bc8c036", "runtime": null}
{"task_id": "easy-tool_selection-00", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912632.6093154, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "get_weather", "ordinal_ev": null, "probs": {"convert_currency": 0.009043726699441566, "create_calendar_event": 0.004547471367018132, "get_weather": 0.9819803863300692, "send_email": 0.001837344522588592, "translate_text": 0.002591071080882583}, "probs_as_returned": {"convert_currency": 0.009043726699441566, "create_calendar_event": 0.004547471367018132, "get_weather": 0.9819803863300692, "send_email": 0.001837344522588592, "translate_text": 0.002591071080882583}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21899547399880248, "usage": {"input_tokens": 234, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2e85881f762d41d0701c1192e7c664f992d65e57c3c476018e20dfd3a4bac3a4", "runtime": null}
{"task_id": "easy-tool_selection-01", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912632.8335145, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "send_email", "ordinal_ev": null, "probs": {"convert_currency": 0.005282634097760304, "create_calendar_event": 0.0036306977791662775, "get_weather": 0.009565614595645375, "send_email": 0.9757192149961745, "translate_text": 0.00580183853125358}, "probs_as_returned": {"convert_currency": 0.005282634097760304, "create_calendar_event": 0.0036306977791662775, "get_weather": 0.009565614595645375, "send_email": 0.9757192149961745, "translate_text": 0.00580183853125358}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.216357142000561, "usage": {"input_tokens": 235, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "22d0a65b71c999c9813e5101cb7dc23c05457ca6f4ccac6401b5520f62c20a22", "runtime": null}
{"task_id": "easy-tool_selection-02", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912633.061555, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "create_calendar_event", "ordinal_ev": null, "probs": {"convert_currency": 0.01393033493832136, "create_calendar_event": 0.9765918749784915, "get_weather": 0.00512468383244068, "send_email": 0.0020068550237233923, "translate_text": 0.002346251227022957}, "probs_as_returned": {"convert_currency": 0.01393033493832136, "create_calendar_event": 0.9765918749784915, "get_weather": 0.00512468383244068, "send_email": 0.0020068550237233923, "translate_text": 0.002346251227022957}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2191826320013206, "usage": {"input_tokens": 238, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1cbaf88cc3d07a1cc4fd53623c8a430c746dabc2cdb382b28b0a4231df56e86f", "runtime": null}
{"task_id": "easy-tool_selection-03", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912633.2924204, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "convert_currency", "ordinal_ev": null, "probs": {"convert_currency": 0.9888907674639486, "create_calendar_event": 0.004579472780466868, "get_weather": 0.0035664969874817805, "send_email": 0.001054254495774708, "translate_text": 0.0019090082723281733}, "probs_as_returned": {"convert_currency": 0.9888907674639486, "create_calendar_event": 0.004579472780466868, "get_weather": 0.0035664969874817805, "send_email": 0.001054254495774708, "translate_text": 0.0019090082723281733}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2237737699997524, "usage": {"input_tokens": 236, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "89b2c5a82093d5f6183eb7485c6e2a53eee9c694e8fe3b0a6dd4de9800e0c39e", "runtime": null}
{"task_id": "easy-tool_selection-04", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912633.5227628, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "translate_text", "ordinal_ev": null, "probs": {"convert_currency": 0.004560393610518057, "create_calendar_event": 0.0033364552396034156, "get_weather": 0.0037806990934833146, "send_email": 0.003551638114985295, "translate_text": 0.9847708139414099}, "probs_as_returned": {"convert_currency": 0.004560393610518057, "create_calendar_event": 0.0033364552396034156, "get_weather": 0.0037806990934833146, "send_email": 0.003551638114985295, "translate_text": 0.9847708139414099}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22196177700061526, "usage": {"input_tokens": 232, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "03a06c2c215047afb159e07d004c48f7fe68296ae76fb6691faeaaa22cd825f6", "runtime": null}
{"task_id": "easy-tool_selection-05", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912633.7398965, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "create_calendar_event", "ordinal_ev": null, "probs": {"convert_currency": 0.012330895117983541, "create_calendar_event": 0.9795643297083751, "get_weather": 0.004536282805147451, "send_email": 0.0016174614455723756, "translate_text": 0.0019510309229215867}, "probs_as_returned": {"convert_currency": 0.012330895117983541, "create_calendar_event": 0.9795643297083751, "get_weather": 0.004536282805147451, "send_email": 0.0016174614455723756, "translate_text": 0.0019510309229215867}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20640916400043352, "usage": {"input_tokens": 238, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "209b491cd8fd4d02c9aa451e77c34c9d7e12397c3b1955487e892af535d71962", "runtime": null}
{"task_id": "easy-tool_selection-06", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912633.9742372, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "search_flights", "ordinal_ev": null, "probs": {"book_restaurant": 0.005812222406624663, "call_taxi": 0.004526563361664453, "get_stock_price": 0.006383477501360058, "search_flights": 0.9774655143237262, "set_reminder": 0.005812222406624663}, "probs_as_returned": {"book_restaurant": 0.005812222406624663, "call_taxi": 0.004526563361664453, "get_stock_price": 0.006383477501360058, "search_flights": 0.9774655143237262, "set_reminder": 0.005812222406624663}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22733194700049353, "usage": {"input_tokens": 247, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0625822b980729e895679d645b26fa00df10a99b90662c4beb1f22123d8e0540", "runtime": null}
{"task_id": "easy-tool_selection-07", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912634.1870058, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "book_restaurant", "ordinal_ev": null, "probs": {"book_restaurant": 0.9887919483999824, "call_taxi": 0.004874336262366302, "get_stock_price": 0.0020964259031212586, "search_flights": 0.0009902814769343574, "set_reminder": 0.0032470079575956314}, "probs_as_returned": {"book_restaurant": 0.9887919483999824, "call_taxi": 0.004874336262366302, "get_stock_price": 0.0020964259031212586, "search_flights": 0.0009902814769343574, "set_reminder": 0.0032470079575956314}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20554294099929393, "usage": {"input_tokens": 247, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "62fc57dba64244893bacccc580ac6789f832682201bfa77052fb25e7393b02b6", "runtime": null}
{"task_id": "easy-tool_selection-08", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912634.4135172, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_reminder", "ordinal_ev": null, "probs": {"book_restaurant": 0.007051972051564175, "call_taxi": 0.0028492542726850363, "get_stock_price": 0.003774648634196235, "search_flights": 0.003129293628648557, "set_reminder": 0.9831948314129061}, "probs_as_returned": {"book_restaurant": 0.007051972051564175, "call_taxi": 0.0028492542726850363, "get_stock_price": 0.003774648634196235, "search_flights": 0.003129293628648557, "set_reminder": 0.9831948314129061}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2196546360009961, "usage": {"input_tokens": 246, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "291cafee4f2f2107640f0a7dd612cf9f91cb5153f24dbd6bb5c7b267ae89b8fb", "runtime": null}
{"task_id": "easy-tool_selection-09", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912634.6399603, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "get_stock_price", "ordinal_ev": null, "probs": {"book_restaurant": 0.009598669981680488, "call_taxi": 0.005137797806276648, "get_stock_price": 0.979090956037076, "search_flights": 0.0027500649932293733, "set_reminder": 0.003422511181737585}, "probs_as_returned": {"book_restaurant": 0.009598669981680488, "call_taxi": 0.005137797806276648, "get_stock_price": 0.979090956037076, "search_flights": 0.0027500649932293733, "set_reminder": 0.003422511181737585}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21882048999941617, "usage": {"input_tokens": 243, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "089b6470616f138711f619209482a937d14af8bd3e9d029efa78ebe41965d0d8", "runtime": null}
{"task_id": "easy-tool_selection-10", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912634.865151, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "call_taxi", "ordinal_ev": null, "probs": {"book_restaurant": 0.007520198714732949, "call_taxi": 0.9849516586392344, "get_stock_price": 0.004025272306794632, "search_flights": 0.0013482973341253177, "set_reminder": 0.0021545730051128263}, "probs_as_returned": {"book_restaurant": 0.007520198714732949, "call_taxi": 0.9849516586392344, "get_stock_price": 0.004025272306794632, "search_flights": 0.0013482973341253177, "set_reminder": 0.0021545730051128263}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21899157400002878, "usage": {"input_tokens": 243, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e2168df505a01b0bcef9ce2da79fb33f3403ab88d272fa33e65bfa9d33552a18", "runtime": null}
{"task_id": "easy-tool_selection-11", "family": "tool_selection", "split": "public", "group": null, "ts": 1791912635.0860763, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_reminder", "ordinal_ev": null, "probs": {"book_restaurant": 0.008477904414820119, "call_taxi": 0.0036463012526728564, "get_stock_price": 0.00453789522792407, "search_flights": 0.0034253830277140213, "set_reminder": 0.979912516076869}, "probs_as_returned": {"book_restaurant": 0.008477904414820119, "call_taxi": 0.0036463012526728564, "get_stock_price": 0.00453789522792407, "search_flights": 0.0034253830277140213, "set_reminder": 0.979912516076869}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21461508699940168, "usage": {"input_tokens": 246, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "7f7681d897ed2de8f4e3e8970714ff7bf9ff1a28ac60484c0c75f852a2ec48b1", "runtime": null}
{"task_id": "hard-opus-a-long_policy-01", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-01", "ts": 1791912637.3288987, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "pay_subject_to_15000_sublimit", "ordinal_ev": null, "probs": {"deny_repeated_seepage": 0.10831398704810802, "deny_vacancy_exclusion": 0.17857957436055386, "pay_full_estimate_less_deductible": 0.20235716843599388, "pay_subject_to_10000_sublimit": 0.10831398704810802, "pay_subject_to_15000_sublimit": 0.40243528310723614}, "probs_as_returned": {"deny_repeated_seepage": 0.10831398704810802, "deny_vacancy_exclusion": 0.17857957436055386, "pay_full_estimate_less_deductible": 0.20235716843599388, "pay_subject_to_10000_sublimit": 0.10831398704810802, "pay_subject_to_15000_sublimit": 0.40243528310723614}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 2.236592170998847, "usage": {"input_tokens": 4072, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "411e36bf92322f31d70bd8aeb4b7b09806adf09c4f41803631d4a2f3913674e8", "runtime": null}
{"task_id": "hard-opus-a-long_policy-04", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-04", "ts": 1791912639.2384408, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "vp_and_finance_director", "ordinal_ev": null, "probs": {"budget_holder": 0.10217554928522789, "cfo": 0.10876530605101374, "department_head": 0.31472284845581366, "executive_committee": 0.0702241595783157, "vp_and_finance_director": 0.4041121366296291}, "probs_as_returned": {"budget_holder": 0.10217554928522789, "cfo": 0.10876530605101374, "department_head": 0.31472284845581366, "executive_committee": 0.0702241595783157, "vp_and_finance_director": 0.4041121366296291}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.9034698539999226, "usage": {"input_tokens": 2744, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "a0afdbd7d99281e13d1026a9ca068ea037887acbbff44af1dcd95be802450697", "runtime": null}
{"task_id": "hard-opus-a-long_policy-08", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-08", "ts": 1791912641.1235738, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "parts_covered_labour_charged", "ordinal_ev": null, "probs": {"full_repair_no_charge": 0.14094105489101572, "not_covered": 0.080305792376359, "parts_covered_labour_charged": 0.7619201441963486, "pro_rata_replacement_credit": 0.016833008536276665}, "probs_as_returned": {"full_repair_no_charge": 0.14094105489101572, "not_covered": 0.080305792376359, "parts_covered_labour_charged": 0.7619201441963486, "pro_rata_replacement_credit": 0.016833008536276665}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.8783056619995477, "usage": {"input_tokens": 2509, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c0f8e8ef41f0a8a4e9c16824df9fe6fb7fbd651ac382eebe51cee12ad72f7803", "runtime": null}
{"task_id": "hard-opus-a-long_policy-09", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-09", "ts": 1791912643.0120597, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "license_required", "ordinal_ev": null, "probs": {"license_exception_lvs": 0.18573834075193324, "license_required": 0.6090125731919687, "no_license_required": 0.09941856960162679, "shipment_prohibited": 0.10583051645447125}, "probs_as_returned": {"license_exception_lvs": 0.18573834075193324, "license_required": 0.6090125731919687, "no_license_required": 0.09941856960162679, "shipment_prohibited": 0.10583051645447125}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.8822112460002245, "usage": {"input_tokens": 2628, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "02d03b018e380369881179b96d3ed92f3e4167edec565207221d444e616e4344", "runtime": null}
{"task_id": "hard-opus-a-long_policy-11", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-11", "ts": 1791912644.8307767, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8933094060543487, "no": 0.10669059394565128}, "probs_as_returned": {"yes": 0.8933094060543487, "no": 0.10669059394565128}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.8118902619989967, "usage": {"input_tokens": 2408, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2dbdfa91893ad63b9abf271e7f26bd66dc283091e0dab3e6f0432fe1beeb3474", "runtime": null}
{"task_id": "hard-opus-a-long_policy-13", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-13", "ts": 1791912646.9215837, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.46879062662624377, "no": 0.5312093733737562}, "probs_as_returned": {"yes": 0.46879062662624377, "no": 0.5312093733737562}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 2.083066515999235, "usage": {"input_tokens": 2500, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "7540f5ae960698a795e4a968d98105d50edbf1eeb743c83170cd115854e406e8", "runtime": null}
{"task_id": "hard-opus-a-long_policy-17", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-17", "ts": 1791912647.3934166, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "database_oncall", "ordinal_ev": null, "probs": {"database_oncall": 0.7258589225752333, "no_page_ticket_only": 0.14293009640620058, "platform_oncall": 0.10456989237762658, "security_incident_response": 0.012489092637991361, "vendor_support": 0.014151996002948196}, "probs_as_returned": {"database_oncall": 0.7258589225752333, "no_page_ticket_only": 0.14293009640620058, "platform_oncall": 0.10456989237762658, "security_incident_response": 0.012489092637991361, "vendor_support": 0.014151996002948196}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.4650859550001769, "usage": {"input_tokens": 2476, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9c9b32e669e35e62642c729d15a79868ac8fd890f356fb73e1e4e99a30175301", "runtime": null}
{"task_id": "hard-opus-a-long_policy-19", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-19", "ts": 1791912649.39653, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.48438008427698437, "no": 0.5156199157230157}, "probs_as_returned": {"yes": 0.48438008427698437, "no": 0.5156199157230157}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.9957482110003184, "usage": {"input_tokens": 2399, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "341d14560f5c329d6c693db22f1068c1ebab0010d98504efbf7cedd54163179f", "runtime": null}
{"task_id": "hard-opus-a-probability-03", "family": "probability", "split": "public", "group": "hard-opus-a-probability-03", "ts": 1791912650.8582067, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8267117940706734, "no": 0.1732882059293266}, "probs_as_returned": {"yes": 0.8267117940706734, "no": 0.1732882059293266}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.4571023829994374, "usage": {"input_tokens": 503, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "a3901e76015276ad03ca84ef6ded8a2f35d8f80cbd502ca806081d3c2c61aa15", "runtime": null}
{"task_id": "hard-opus-a-probability-04", "family": "probability", "split": "public", "group": "hard-opus-a-probability-04", "ts": 1791912652.5961065, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "late", "ordinal_ev": null, "probs": {"early": 0.062457823654599925, "late": 0.8622036601402205, "on_time": 0.07533851620517966}, "probs_as_returned": {"early": 0.062457823654599925, "late": 0.8622036601402205, "on_time": 0.07533851620517966}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.7305739290004567, "usage": {"input_tokens": 1188, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "bef1137794c70a2ad131cf5ecbd370c273da93bc6e56aaeba787c9275d455339", "runtime": null}
{"task_id": "hard-opus-a-probability-07", "family": "probability", "split": "public", "group": "hard-opus-a-probability-07", "ts": 1791912653.946686, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "escalated", "ordinal_ev": null, "probs": {"escalated": 0.7491059825952231, "reopened": 0.06149036355048325, "resolved_first_contact": 0.18940365385429367}, "probs_as_returned": {"escalated": 0.7491059825952231, "reopened": 0.06149036355048325, "resolved_first_contact": 0.18940365385429367}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3375792259994341, "usage": {"input_tokens": 1306, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0e9825abbe1d2cf9ad0825724f92749399b2a9ddea0737e2079a2034b6fde236", "runtime": null}
{"task_id": "hard-opus-a-probability-08", "family": "probability", "split": "public", "group": "hard-opus-a-probability-08", "ts": 1791912655.5595963, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.18242552380635632, "no": 0.8175744761936437}, "probs_as_returned": {"yes": 0.18242552380635632, "no": 0.8175744761936437}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.6049358489999577, "usage": {"input_tokens": 465, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e38d8250ba93d8d80c32180a2253a60d8ce1fa99f25ca171c4804fa0bef43e86", "runtime": null}
{"task_id": "hard-opus-a-temporal_numeric-03", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-03", "ts": 1791912656.9031146, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.3629692055196168, "no": 0.6370307944803832}, "probs_as_returned": {"yes": 0.3629692055196168, "no": 0.6370307944803832}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3357243730006303, "usage": {"input_tokens": 561, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "97945282e6fbf1b0f7e2ff5f3b381586671146b5933a61f55ed5a61b8aed8bb8", "runtime": null}
{"task_id": "hard-opus-a-temporal_numeric-06", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-06", "ts": 1791912661.6792293, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.34864513533394575, "no": 0.6513548646660543}, "probs_as_returned": {"yes": 0.34864513533394575, "no": 0.6513548646660543}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 4.7691506760002085, "usage": {"input_tokens": 596, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "96a0cf0b9137ab5159993ba6fa61f37f0598e2aa45169d487cb91e61924dc102", "runtime": null}
{"task_id": "hard-opus-a-temporal_numeric-07", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-07", "ts": 1791912663.0201774, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "sep_26", "ordinal_ev": null, "probs": {"sep_24": 0.18685061329425104, "sep_25": 0.22538486187838766, "sep_26": 0.2553945075821753, "sep_27": 0.18685061329425104, "sep_28": 0.14551940395093496}, "probs_as_returned": {"sep_24": 0.18685061329425104, "sep_25": 0.22538486187838766, "sep_26": 0.2553945075821753, "sep_27": 0.18685061329425104, "sep_28": 0.14551940395093496}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3320874639994145, "usage": {"input_tokens": 1423, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0748861816b49ae8be42ddac3534d9c3138761f80f28337f567577d57413e3c4", "runtime": null}
{"task_id": "hard-opus-a-temporal_numeric-09", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-09", "ts": 1791912663.251933, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5467381519846138, "no": 0.45326184801538616}, "probs_as_returned": {"yes": 0.5467381519846138, "no": 0.45326184801538616}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22412158100087254, "usage": {"input_tokens": 593, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "cdea6784e4cb89ea6dc8d6f374af892194bdf6278524a7d06563392fa3db63b3", "runtime": null}
{"task_id": "hard-opus-a-temporal_numeric-12", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-12", "ts": 1791912664.6163318, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "0", "ordinal_ev": 1.3727883693553482, "probs": {"0": 0.2712019384407753, "1": 0.2712019384407753, "2": 0.2712019384407753, "3": 0.1863941846776741}, "probs_as_returned": {"0": 0.2712019384407753, "1": 0.2712019384407753, "2": 0.2712019384407753, "3": 0.1863941846776741}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3450946979992295, "usage": {"input_tokens": 673, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "40fa26517bc246df98c7a7671a31050b8f2f5d9a5fdef2b03ac3a737142431f7", "runtime": null}
{"task_id": "hard-opus-b-ambiguous-02", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-02", "ts": 1791912664.8499815, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "real_incident", "ordinal_ev": null, "probs": {"cannot_determine": 0.07162659469286574, "false_positive": 0.05578284803554197, "real_incident": 0.8725905572715923}, "probs_as_returned": {"cannot_determine": 0.07162659469286574, "false_positive": 0.05578284803554197, "real_incident": 0.8725905572715923}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22039616599977307, "usage": {"input_tokens": 628, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "40a6661398555bec21f0e1746bb944d9e1576d7abe9c37558edb1f1da7bff9af", "runtime": null}
{"task_id": "hard-opus-b-ambiguous-03", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-03", "ts": 1791912665.0704215, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "within_limitation", "ordinal_ev": null, "probs": {"insufficient_information": 0.10364629081211886, "time_barred": 0.2999104867223939, "within_limitation": 0.5964432224654873}, "probs_as_returned": {"insufficient_information": 0.10364629081211886, "time_barred": 0.2999104867223939, "within_limitation": 0.5964432224654873}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21007213600023533, "usage": {"input_tokens": 584, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "776b5effdc907d4534a071e995133429f15c2d06e95b9af3768c1c92f74277b4", "runtime": null}
{"task_id": "hard-opus-b-ambiguous-07", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-07", "ts": 1791912665.2914867, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "within_policy", "ordinal_ev": null, "probs": {"insufficient_information": 0.18042963056040295, "over_limit": 0.3588271669183635, "within_policy": 0.46074320252123346}, "probs_as_returned": {"insufficient_information": 0.18042963056040295, "over_limit": 0.3588271669183635, "within_policy": 0.46074320252123346}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2113220889987133, "usage": {"input_tokens": 585, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "38c71f54d463a85dcf321ad033083bdd28f01f5e4e25723ffe16c9fa578b474d", "runtime": null}
{"task_id": "hard-opus-b-ambiguous-09", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-09", "ts": 1791912666.5173817, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "legitimate", "ordinal_ev": null, "probs": {"compromised": 0.13837059054054027, "inconclusive": 0.06536163885437346, "legitimate": 0.7962677706050862}, "probs_as_returned": {"compromised": 0.13837059054054027, "inconclusive": 0.06536163885437346, "legitimate": 0.7962677706050862}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.2104390490003425, "usage": {"input_tokens": 725, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "60f9f7cd5ca77e170a51cdacc702da8fec41649ab6f9f6935ad282840c105997", "runtime": null}
{"task_id": "hard-opus-b-ambiguous-10", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-10", "ts": 1791912666.751757, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "ask_user_which_alex", "ordinal_ev": null, "probs": {"ask_user_which_alex": 0.7201605178474296, "send_to_alex_moreno": 0.08601090827501642, "send_to_alexandra_lind": 0.19382857387755398}, "probs_as_returned": {"ask_user_which_alex": 0.7201605178474296, "send_to_alex_moreno": 0.08601090827501642, "send_to_alexandra_lind": 0.19382857387755398}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22484569100015506, "usage": {"input_tokens": 549, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "9fd39e7f64b7a5921b0debdcec6aa50c4bcae63f709acbff4a78c2e5837b990a", "runtime": null}
{"task_id": "hard-opus-b-ambiguous-11", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-11", "ts": 1791912666.9710824, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cannot_determine", "ordinal_ev": null, "probs": {"cannot_determine": 0.9663346730360224, "grade_correct": 0.00947361178359884, "grade_should_be_higher": 0.024191715180378732}, "probs_as_returned": {"cannot_determine": 0.9663346730360224, "grade_correct": 0.00947361178359884, "grade_should_be_higher": 0.024191715180378732}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2097261039998557, "usage": {"input_tokens": 554, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "4e9779d86e830a0a5c41003228a0c724dd4017d984f9c8d1b5b650ff92bba9ea", "runtime": null}
{"task_id": "hard-opus-b-ambiguous-13", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-13", "ts": 1791912667.2107692, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cannot_determine", "ordinal_ev": null, "probs": {"cannot_determine": 0.9675197533541323, "charge_data_team": 0.00786353178158189, "charge_platform_team": 0.005753082750123942, "split_50_50": 0.018863632114161876}, "probs_as_returned": {"cannot_determine": 0.9675197533541323, "charge_data_team": 0.00786353178158189, "charge_platform_team": 0.005753082750123942, "split_50_50": 0.018863632114161876}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21855992199925822, "usage": {"input_tokens": 707, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "a136fc6cec2784770fc318f35e8435c386e8d353e9330d12d686b155464b246f", "runtime": null}
{"task_id": "hard-opus-b-multi_hop-03", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-03", "ts": 1791912668.5417426, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "tier3_cfo", "ordinal_ev": null, "probs": {"tier1_team_lead": 0.10019362322410517, "tier2_department_head": 0.22578981539245424, "tier3_cfo": 0.508825203585344, "tier4_board": 0.16519135779809654}, "probs_as_returned": {"tier1_team_lead": 0.10019362322410517, "tier2_department_head": 0.22578981539245424, "tier3_cfo": 0.508825203585344, "tier4_board": 0.16519135779809654}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3171462650007015, "usage": {"input_tokens": 3072, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "6506d4de17efe0d99e0f464b0cbe72e1b3df4e21e3d7a783444bfbfc610f2d42", "runtime": null}
{"task_id": "hard-opus-b-multi_hop-04", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-04", "ts": 1791912669.6945434, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "chen", "ordinal_ev": null, "probs": {"ana": 0.1099146673052784, "bjorn": 0.15992489726979597, "chen": 0.24769652639741116, "dmitri": 0.070966242978465, "esra": 0.2185914173266788, "no_page_ticket_only": 0.19290624872237072}, "probs_as_returned": {"ana": 0.1099146673052784, "bjorn": 0.15992489726979597, "chen": 0.24769652639741116, "dmitri": 0.070966242978465, "esra": 0.2185914173266788, "no_page_ticket_only": 0.19290624872237072}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.1425970870004676, "usage": {"input_tokens": 2469, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0cb9cb8e610cd04f3b828d58aea0d9d5b930e89f3c8aba6a6cb44fa1201ea638", "runtime": null}
{"task_id": "hard-opus-b-multi_hop-05", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-05", "ts": 1791912670.8328006, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 2.215210032928174, "probs": {"0": 0.11349519162945948, "1": 0.16513452942131107, "2": 0.2898201248858194, "3": 0.2557653625184162, "4": 0.17578479154499382}, "probs_as_returned": {"0": 0.11349519162945948, "1": 0.16513452942131107, "2": 0.2898201248858194, "3": 0.2557653625184162, "4": 0.17578479154499382}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.1301950069992017, "usage": {"input_tokens": 2675, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "79b4d5df452bcc559df7e6f6d8df47ea96886ccee9107a573098ece903b7964c", "runtime": null}
{"task_id": "hard-opus-b-multi_hop-07", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-07", "ts": 1791912671.981095, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "credit_25_percent", "ordinal_ev": null, "probs": {"credit_10_percent": 0.1015143500390693, "credit_20_percent": 0.16736886819071192, "credit_25_percent": 0.31268621139730407, "credit_50_percent": 0.2018853916229897, "credit_full_month": 0.1150308287108557, "no_credit": 0.1015143500390693}, "probs_as_returned": {"credit_10_percent": 0.1015143500390693, "credit_20_percent": 0.16736886819071192, "credit_25_percent": 0.31268621139730407, "credit_50_percent": 0.2018853916229897, "credit_full_month": 0.1150308287108557, "no_credit": 0.1015143500390693}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.137035323999953, "usage": {"input_tokens": 2770, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "d0aba9ea70a871e2649efa16f927c5e8637e0fcbf22020e0c2562814ae3aad2c", "runtime": null}
{"task_id": "hard-opus-b-multi_hop-08", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-08", "ts": 1791912673.1140459, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.34864513533394575, "no": 0.6513548646660543}, "probs_as_returned": {"yes": 0.34864513533394575, "no": 0.6513548646660543}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.1242658110004413, "usage": {"input_tokens": 2387, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "b44cd10dd9e6b34820a229decaca3bd2f9032c0300df414aca1540dfa771aa6d", "runtime": null}
{"task_id": "hard-opus-b-probability-01", "family": "probability", "split": "public", "group": "hard-opus-b-probability-01", "ts": 1791912674.4411283, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "merchant_wins", "ordinal_ev": null, "probs": {"customer_wins": 0.24472847105479767, "customer_withdraws": 0.09003057317038048, "merchant_wins": 0.6652409557748218}, "probs_as_returned": {"customer_wins": 0.24472847105479767, "customer_withdraws": 0.09003057317038048, "merchant_wins": 0.6652409557748218}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3194354200004454, "usage": {"input_tokens": 2009, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "d08678c6ae5f57235419c3c1c89b1d1f5c4395bd6c7c01c0f5a8fa59834dd1da", "runtime": null}
{"task_id": "hard-opus-b-probability-02", "family": "probability", "split": "public", "group": "hard-opus-b-probability-02", "ts": 1791912674.676466, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "bad_push", "ordinal_ev": null, "probs": {"bad_push": 0.911003130426852, "database_hardware": 0.03760158987860883, "upstream_provider": 0.05139527969453916}, "probs_as_returned": {"bad_push": 0.911003130426852, "database_hardware": 0.03760158987860883, "upstream_provider": 0.05139527969453916}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22670197800107417, "usage": {"input_tokens": 589, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "fc73884998969a0018d739f63c8d2efcc7af9c89fb385710f45ea57f6a462490", "runtime": null}
{"task_id": "hard-opus-b-probability-03", "family": "probability", "split": "public", "group": "hard-opus-b-probability-03", "ts": 1791912675.8207898, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7772998611746911, "no": 0.22270013882530892}, "probs_as_returned": {"yes": 0.7772998611746911, "no": 0.22270013882530892}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.1352466460011783, "usage": {"input_tokens": 1287, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "42018015cb28c75fda28ca436f3dbd9ea0eddedf3c2f4362dda937c21ef76068", "runtime": null}
{"task_id": "hard-opus-b-probability-04", "family": "probability", "split": "public", "group": "hard-opus-b-probability-04", "ts": 1791912676.0413108, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "attended", "ordinal_ev": null, "probs": {"attended": 0.92119640954977, "late_cancellation": 0.043084929412511117, "no_show": 0.03571866103771889}, "probs_as_returned": {"attended": 0.92119640954977, "late_cancellation": 0.043084929412511117, "no_show": 0.03571866103771889}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21328668299975106, "usage": {"input_tokens": 688, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "ebcd22474ab824804457288dfec2cf0bf58c5d9b97268d3a5b5194302601c3b0", "runtime": null}
{"task_id": "hard-opus-b-probability-06", "family": "probability", "split": "public", "group": "hard-opus-b-probability-06", "ts": 1791912677.3543012, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "upheld", "ordinal_ev": null, "probs": {"modified": 0.2683914330410816, "overturned": 0.12677913601083657, "upheld": 0.6048294309480818}, "probs_as_returned": {"modified": 0.2683914330410816, "overturned": 0.12677913601083657, "upheld": 0.6048294309480818}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3042942240008415, "usage": {"input_tokens": 1106, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "47085248a92056b634e35723dfb65afa1225cc27069421da2ef3552c8355f8d3", "runtime": null}
{"task_id": "hard-opus-b-tradeoff-01", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-01", "ts": 1791912678.7260969, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "escalate_to_security", "ordinal_ev": null, "probs": {"escalate_to_engineering": 0.27371333334229764, "escalate_to_security": 0.6168225184109526, "page_oncall": 0.02246775859074525, "process_refund": 0.039432144346147203, "send_kb_article": 0.047564245309857256}, "probs_as_returned": {"escalate_to_engineering": 0.27371333334229764, "escalate_to_security": 0.6168225184109526, "page_oncall": 0.02246775859074525, "process_refund": 0.039432144346147203, "send_kb_article": 0.047564245309857256}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3645932319996064, "usage": {"input_tokens": 960, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "733ebc00fbe45cf24caa9ebb1607b28aedf2d16a0493e9e44cc7c4c0efa5c1fe", "runtime": null}
{"task_id": "hard-opus-b-tradeoff-03", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-03", "ts": 1791912680.089377, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "order_b", "ordinal_ev": null, "probs": {"order_a": 0.09168458739933019, "order_b": 0.8698791455103705, "order_c": 0.019218133545149667, "order_d": 0.019218133545149667}, "probs_as_returned": {"order_a": 0.09168458739933019, "order_b": 0.8698791455103705, "order_c": 0.019218133545149667, "order_d": 0.019218133545149667}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3561420379992342, "usage": {"input_tokens": 857, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "72828b8527c80e29fbb93259376737655a220c8aafcb7f1ca2b88f4c22a1d7ca", "runtime": null}
{"task_id": "hard-opus-b-tradeoff-06", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-06", "ts": 1791912681.3278217, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.6513548646660542, "no": 0.3486451353339458}, "probs_as_returned": {"yes": 0.6513548646660542, "no": 0.3486451353339458}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.2275646300004155, "usage": {"input_tokens": 864, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e90fe68592bfaf565e71baa16de5807caa5367fa7d27dd9128e3facdf6b3b25c", "runtime": null}
{"task_id": "hard-opus-b-tradeoff-07", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-07", "ts": 1791912681.5608802, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "p0_fix_24h", "ordinal_ev": null, "probs": {"out_of_scope": 0.023875866439004502, "p0_fix_24h": 0.4795595969337222, "p1_fix_7d": 0.3295961695138673, "p2_fix_30d": 0.1556902063899086, "p3_backlog": 0.011278160723497441}, "probs_as_returned": {"out_of_scope": 0.023875866439004502, "p0_fix_24h": 0.4795595969337222, "p1_fix_7d": 0.3295961695138673, "p2_fix_30d": 0.1556902063899086, "p3_backlog": 0.011278160723497441}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22238869499960856, "usage": {"input_tokens": 845, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "6904ff63d66a439616901cf9767447912e96be2cb34d5584c76b732724dbbf7a", "runtime": null}
{"task_id": "hard-opus-b-tradeoff-08", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-08", "ts": 1791912681.7918231, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.46879062662624377, "no": 0.5312093733737562}, "probs_as_returned": {"yes": 0.46879062662624377, "no": 0.5312093733737562}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.222219502000371, "usage": {"input_tokens": 676, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "62dc2d7fdf31889a326144c41681f9b587de2ab45839175733a56410619de342", "runtime": null}
{"task_id": "hard-opus-b-tradeoff-12", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-12", "ts": 1791912682.0133731, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "refund_now", "ordinal_ev": null, "probs": {"deny_refund": 0.025307445868761696, "escalate_to_fraud_team": 0.011954390963706797, "offer_store_credit": 0.009910538199025591, "refund_now": 0.8921182183205237, "wait_for_inspection": 0.060709406647982134}, "probs_as_returned": {"deny_refund": 0.025307445868761696, "escalate_to_fraud_team": 0.011954390963706797, "offer_store_credit": 0.009910538199025591, "refund_now": 0.8921182183205237, "wait_for_inspection": 0.060709406647982134}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21340999600033683, "usage": {"input_tokens": 712, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2791a962f5ab66e3753e7af78da9c27bbb5c77bc3695f5bda85aa95068e4bd08", "runtime": null}
{"task_id": "hard-opus-c-long_policy-02", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-02", "ts": 1791912683.423205, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "erase_but_retain_transaction_records", "ordinal_ev": null, "probs": {"erase_all": 0.23702489880653266, "erase_but_retain_transaction_records": 0.7300880869169732, "erase_marketing_only": 0.009783185960915444, "reject_unverified_request": 0.005933802234965418, "restrict_all_pending_legal_hold": 0.017170026080613264}, "probs_as_returned": {"erase_all": 0.23702489880653266, "erase_but_retain_transaction_records": 0.7300880869169732, "erase_marketing_only": 0.009783185960915444, "reject_unverified_request": 0.005933802234965418, "restrict_all_pending_legal_hold": 0.017170026080613264}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3997882029998436, "usage": {"input_tokens": 3704, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "09fb9c2851f44b4bc82ab132d9b61d747e03b9fc0129f73a9663ac01d9286dbd", "runtime": null}
{"task_id": "hard-opus-c-long_policy-03", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-03", "ts": 1791912684.8141184, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "eur_400", "ordinal_ev": null, "probs": {"eur_0": 0.21083016487938738, "eur_150": 0.09958911819789595, "eur_200": 0.17478434567578693, "eur_300": 0.15424664367915716, "eur_400": 0.22442754228684655, "eur_600": 0.13612218528092598}, "probs_as_returned": {"eur_0": 0.21083016487938738, "eur_150": 0.09958911819789595, "eur_200": 0.17478434567578693, "eur_300": 0.15424664367915716, "eur_400": 0.22442754228684655, "eur_600": 0.13612218528092598}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3813221719992725, "usage": {"input_tokens": 3650, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "b5b0c12a21a5e179d6806f6c2767517c2e23755267b500c0fc7a364ffafc3a10", "runtime": null}
{"task_id": "hard-opus-c-long_policy-04", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-04", "ts": 1791912686.2317367, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5467381519846138, "no": 0.45326184801538616}, "probs_as_returned": {"yes": 0.5467381519846138, "no": 0.45326184801538616}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.4081901809986448, "usage": {"input_tokens": 3387, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "939e8d886625a93b93cfe0a6fa0a0b00a2ae5dab95e7077cc1fb5163d77a0ab6", "runtime": null}
{"task_id": "hard-opus-c-long_policy-05", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-05", "ts": 1791912687.6068032, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "1", "ordinal_ev": 1.3673384206611892, "probs": {"0": 0.24363722216737665, "1": 0.31283638571410965, "2": 0.2760771414084615, "3": 0.16744925071005215}, "probs_as_returned": {"0": 0.24363722216737665, "1": 0.31283638571410965, "2": 0.2760771414084615, "3": 0.16744925071005215}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3668979390004097, "usage": {"input_tokens": 3325, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e000796ffdd51c7cc2161548d136791267d4842dcc06e8238cd6b5d0d9ace2bb", "runtime": null}
{"task_id": "hard-opus-c-long_policy-08", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-08", "ts": 1791912688.9913146, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "2", "ordinal_ev": 1.5511853678026322, "probs": {"0": 0.09110893634688244, "1": 0.38358294946038535, "2": 0.4083219242359495, "3": 0.11698618995678264}, "probs_as_returned": {"0": 0.09110893634688244, "1": 0.38358294946038535, "2": 0.4083219242359495, "3": 0.11698618995678264}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3772979170007602, "usage": {"input_tokens": 3328, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "57e3a3a438c847d11e09b3e20b492b63238036e8704ce10d1b38a28e50d95346", "runtime": null}
{"task_id": "hard-opus-c-long_policy-10", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-10", "ts": 1791912690.4046845, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "credit_minus_restocking_fee", "ordinal_ev": null, "probs": {"credit_minus_restocking_fee": 0.8246558487503892, "full_credit": 0.05973786932035262, "reject_return": 0.09849109581476997, "repair_or_replace_only": 0.01711518611448822}, "probs_as_returned": {"credit_minus_restocking_fee": 0.8246558487503892, "full_credit": 0.05973786932035262, "reject_return": 0.09849109581476997, "repair_or_replace_only": 0.01711518611448822}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.4034819780008547, "usage": {"input_tokens": 3738, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1e7e5ba6d8b75e54463c99013891d0c719b02ae44487a244e68870ce0d1f30c1", "runtime": null}
{"task_id": "hard-opus-c-long_policy-11", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-11", "ts": 1791912691.7854505, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.45326184801538616, "no": 0.5467381519846138}, "probs_as_returned": {"yes": 0.45326184801538616, "no": 0.5467381519846138}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3676845659992978, "usage": {"input_tokens": 3473, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "74bbcdcb3115067fd709421aea4275086e7ac7d126c7dd1b906bbbd3f6632a83", "runtime": null}
{"task_id": "hard-opus-c-probability-03", "family": "probability", "split": "public", "group": "hard-opus-c-probability-03", "ts": 1791912692.0192983, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "kuznets_technik", "ordinal_ev": null, "probs": {"halvorsen_polymer": 0.14916049028543182, "kuznets_technik": 0.5541984537026645, "ortega_seal_works": 0.2966410560119037}, "probs_as_returned": {"halvorsen_polymer": 0.14916049028543182, "kuznets_technik": 0.5541984537026645, "ortega_seal_works": 0.2966410560119037}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22565443199891888, "usage": {"input_tokens": 646, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "99657bd762c16f48d43a34d48adf18e776cf276720db9b570cbd72753249494e", "runtime": null}
{"task_id": "hard-opus-c-temporal_numeric-02", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-02", "ts": 1791912693.2141855, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.24508501313237172, "no": 0.7549149868676283}, "probs_as_returned": {"yes": 0.24508501313237172, "no": 0.7549149868676283}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.1839376210009505, "usage": {"input_tokens": 1010, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "6a608931f4ccd82a4dd22f971640e0b28cbba650af1d5b0cc3cdcaef60b790a6", "runtime": null}
{"task_id": "hard-opus-c-temporal_numeric-03", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-03", "ts": 1791912693.451646, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "eur_693_90", "ordinal_ev": null, "probs": {"eur_617_00": 0.17207029489401338, "eur_688_00": 0.19498118847790388, "eur_692_00": 0.2209426321008681, "eur_693_90": 0.2503608017816116, "eur_812_00": 0.1616450827456031}, "probs_as_returned": {"eur_617_00": 0.17207029489401338, "eur_688_00": 0.19498118847790388, "eur_692_00": 0.2209426321008681, "eur_693_90": 0.2503608017816116, "eur_812_00": 0.1616450827456031}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22823790900110907, "usage": {"input_tokens": 1009, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "b6a841231d0af571597a1e146ad91397721dd73aedd22f2d4daa8a9b347fa000", "runtime": null}
{"task_id": "hard-opus-c-temporal_numeric-04", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-04", "ts": 1791912693.6939762, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "usd_2303_33", "ordinal_ev": null, "probs": {"usd_2222_50": 0.1108114489858103, "usd_2303_01": 0.30121674816334465, "usd_2303_33": 0.32064355935313665, "usd_2335_00": 0.13366412174885423, "usd_2340_00": 0.13366412174885423}, "probs_as_returned": {"usd_2222_50": 0.1108114489858103, "usd_2303_01": 0.30121674816334465, "usd_2303_33": 0.32064355935313665, "usd_2335_00": 0.13366412174885423, "usd_2340_00": 0.13366412174885423}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.23321098899941717, "usage": {"input_tokens": 934, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1c028d5fc5ceadfeec11bc37b4109e99eae33c6aea19d049e3747c4d19eb344f", "runtime": null}
{"task_id": "hard-opus-c-temporal_numeric-06", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-06", "ts": 1791912694.9182725, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "sep_16_taxi", "ordinal_ev": null, "probs": {"none_declined": 0.21478242278194112, "sep_15_office_chair": 0.20176941362408216, "sep_16_taxi": 0.24338037012130165, "sep_17_parking": 0.0953091223621836, "sep_18_book_set": 0.12237933555524574, "sep_21_client_dinner": 0.12237933555524574}, "probs_as_returned": {"none_declined": 0.21478242278194112, "sep_15_office_chair": 0.20176941362408216, "sep_16_taxi": 0.24338037012130165, "sep_17_parking": 0.0953091223621836, "sep_18_book_set": 0.12237933555524574, "sep_21_client_dinner": 0.12237933555524574}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.2096759059986653, "usage": {"input_tokens": 1131, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "8bb974035f2a65e9347b988f8f543236c831ab22af433f755914909c38736708", "runtime": null}
{"task_id": "hard-opus-c-temporal_numeric-08", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-08", "ts": 1791912695.1817482, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "eur_506_06", "ordinal_ev": null, "probs": {"eur_497_40": 0.17132783349027314, "eur_498_81": 0.18237752940879742, "eur_506_06": 0.26535773950907876, "eur_513_69": 0.21998929278755594, "eur_532_01": 0.16094760480429468}, "probs_as_returned": {"eur_497_40": 0.17132783349027314, "eur_498_81": 0.18237752940879742, "eur_506_06": 0.26535773950907876, "eur_513_69": 0.21998929278755594, "eur_532_01": 0.16094760480429468}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2530178570013959, "usage": {"input_tokens": 1128, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "80f1c0c93d53b8237ab56745c50b1711882406510582e0fe7ca8250adbf3664e", "runtime": null}
{"task_id": "hard-opus-c-temporal_numeric-12", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-12", "ts": 1791912695.4194698, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "expired_24_month_term", "ordinal_ev": null, "probs": {"covered": 0.2817829864631392, "expired_24_month_term": 0.43643402707372175, "expired_30_month_cap": 0.2817829864631392}, "probs_as_returned": {"covered": 0.2817829864631392, "expired_24_month_term": 0.43643402707372175, "expired_30_month_cap": 0.2817829864631392}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.22611844799939718, "usage": {"input_tokens": 917, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e301ede1a959d2d1ef0ecd1eb8ee655e56016e9de30f8aa94aaabf3cf14e4532", "runtime": null}
{"task_id": "hard-sol-a-adversarial-01", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-01", "ts": 1791912696.6148295, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "deny_missing_approval", "ordinal_ev": null, "probs": {"deny_inactive_user": 0.030167979542995535, "deny_missing_approval": 0.9384983349396374, "grant_access": 0.02501013347781395, "grant_owner_exception": 0.006323552039553237}, "probs_as_returned": {"deny_inactive_user": 0.030167979542995535, "deny_missing_approval": 0.9384983349396374, "grant_access": 0.02501013347781395, "grant_owner_exception": 0.006323552039553237}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.1859080480007833, "usage": {"input_tokens": 385, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "892a45a2413ab639b5f3aa1e1825ca5e40be409b17ee7348312a3a9163a6baa9", "runtime": null}
{"task_id": "hard-sol-a-adversarial-06", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-06", "ts": 1791912697.775263, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.053403329799824234, "no": 0.9465966702001758}, "probs_as_returned": {"yes": 0.053403329799824234, "no": 0.9465966702001758}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.148801674000424, "usage": {"input_tokens": 315, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5bc9a91d240d5dbab2828c92621ce8283f397bc88bf8047b11ce2de6e1382080", "runtime": null}
{"task_id": "hard-sol-a-adversarial-07", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-07", "ts": 1791912698.0099823, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "retain_until_2031", "ordinal_ev": null, "probs": {"delete_now": 0.11853688456879166, "needs_signature_review": 0.04096525114625869, "retain_indefinitely": 0.0675402809244095, "retain_until_2031": 0.7729575833605401}, "probs_as_returned": {"delete_now": 0.11853688456879166, "needs_signature_review": 0.04096525114625869, "retain_indefinitely": 0.0675402809244095, "retain_until_2031": 0.7729575833605401}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2260028839991719, "usage": {"input_tokens": 406, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "51abc8631970a091815b7fc6f8514710ba59e61cbbed476cf626b21c3b3303ad", "runtime": null}
{"task_id": "hard-sol-a-adversarial-08", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-08", "ts": 1791912699.2366538, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9465966702001757, "no": 0.05340332979982432}, "probs_as_returned": {"yes": 0.9465966702001757, "no": 0.05340332979982432}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.2149988490000396, "usage": {"input_tokens": 294, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5d7693d0b3eb2d4decd321249125fd5d3bca063884d992c5351c32932940cfb2", "runtime": null}
{"task_id": "hard-sol-a-adversarial-09", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-09", "ts": 1791912699.4506063, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.8750103898674295, "probs": {"0": 0.0475976407970985, "1": 0.06925415871585935, "2": 0.8436883703095561, "3": 0.039459830177486}, "probs_as_returned": {"0": 0.0475976407970985, "1": 0.06925415871585935, "2": 0.8436883703095561, "3": 0.039459830177486}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20296900699941034, "usage": {"input_tokens": 309, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1f830ff4932cf33124d17e0c6a8cdf63f548f762cfa9809935a7cf07b5124fca", "runtime": null}
{"task_id": "hard-sol-a-adversarial-11", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-11", "ts": 1791912699.6722188, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "standard_research", "ordinal_ev": null, "probs": {"ethics_review": 0.0247149477249476, "major_grants": 0.014082152727797743, "manual_hold": 0.033781328264585395, "standard_research": 0.9274215712826692}, "probs_as_returned": {"ethics_review": 0.0247149477249476, "major_grants": 0.014082152727797743, "manual_hold": 0.033781328264585395, "standard_research": 0.9274215712826692}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21125895799923455, "usage": {"input_tokens": 407, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "352b914fab0df9331768054c7c5aaca4b17e93ce3695575fbe61e7fcfa15be2d", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-01", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-01", "ts": 1791912701.0024233, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "accept_s2_dusk", "ordinal_ev": null, "probs": {"accept_s2_dusk": 0.7460396781387163, "reject_cutoff": 0.13800346404849445, "reject_tier": 0.06518822057890589, "reroute_s3": 0.050768637233883376}, "probs_as_returned": {"accept_s2_dusk": 0.7460396781387163, "reject_cutoff": 0.13800346404849445, "reject_tier": 0.06518822057890589, "reroute_s3": 0.050768637233883376}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3214765840002656, "usage": {"input_tokens": 2482, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "a770f22cf997c9b0c7acf4648ad42d96e075d46ec14f09acf8ab36ced57dc8df", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-02", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-02", "ts": 1791912701.3312986, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "insufficient_information", "ordinal_ev": null, "probs": {"executive_bridge": 0.09378388526449781, "heron_q2": 0.3075055212491322, "insufficient_information": 0.3273379234563493, "security_primary": 0.27137267003002064}, "probs_as_returned": {"executive_bridge": 0.09378388526449781, "heron_q2": 0.3075055212491322, "insufficient_information": 0.3273379234563493, "security_primary": 0.27137267003002064}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.3179480290000356, "usage": {"input_tokens": 2508, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "edc8d250657777242735ac4c0b219ffd004d0963cb2797a158d949d3e24f97b1", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-05", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-05", "ts": 1791912702.6600714, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "require_director", "ordinal_ev": null, "probs": {"pay_manager_approved": 0.24066849874116666, "reject_category": 0.12101572098991827, "require_director": 0.5094952158331973, "require_finance_board": 0.1288205644357177}, "probs_as_returned": {"pay_manager_approved": 0.24066849874116666, "reject_category": 0.12101572098991827, "require_director": 0.5094952158331973, "require_finance_board": 0.1288205644357177}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3198611150000943, "usage": {"input_tokens": 2425, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "62ee1ce44108eae0b9cc5ee11b6fa831d5c6a9b6503d04d533ab6bddf700a0ef", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-07", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-07", "ts": 1791912702.9656806, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "admit", "ordinal_ev": null, "probs": {"admit": 0.7176506781478255, "deny_prerequisite": 0.11715325271717256, "deny_score": 0.12470898836163383, "manual_equivalency": 0.04048708077336814}, "probs_as_returned": {"admit": 0.7176506781478255, "deny_prerequisite": 0.11715325271717256, "deny_score": 0.12470898836163383, "manual_equivalency": 0.04048708077336814}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.29584401300053287, "usage": {"input_tokens": 2416, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "cd6827518304328d727edee85f1733dafcac04aea5d97003b407a4b4155b21f6", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-08", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-08", "ts": 1791912703.279268, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "local_maintenance", "ordinal_ev": null, "probs": {"central_engineering": 0.2359800408603415, "insufficient_log": 0.0815525199140706, "local_maintenance": 0.6414602569496891, "safety_shutdown": 0.0410071822758987}, "probs_as_returned": {"central_engineering": 0.2359800408603415, "insufficient_log": 0.0815525199140706, "local_maintenance": 0.6414602569496891, "safety_shutdown": 0.0410071822758987}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.3054356929987989, "usage": {"input_tokens": 2391, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "42a6b0826edc7343cc2690564dca565bd28da4a1a8d284f7fcbf1b462789f9f7", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-09", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-09", "ts": 1791912703.5793042, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "require_steward", "ordinal_ev": null, "probs": {"confirm_l3": 0.09483881411100212, "reject_capacity": 0.11439744639990411, "require_steward": 0.7459650558014338, "switch_l4": 0.04479868368765996}, "probs_as_returned": {"confirm_l3": 0.09483881411100212, "reject_capacity": 0.11439744639990411, "require_steward": 0.7459650558014338, "switch_l4": 0.04479868368765996}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.29008089800117887, "usage": {"input_tokens": 2389, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "50a09b3ec2617c4dd02c67d7511cd167bded0186baa1358f40f70bc6779d5b68", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-10", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-10", "ts": 1791912703.8841827, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "approve_45_days", "ordinal_ev": null, "probs": {"approve_45_days": 0.4352711293911737, "deny_sensitive": 0.06270675160131459, "needs_field_review": 0.06675098961633799, "reduce_to_30_days": 0.4352711293911737}, "probs_as_returned": {"approve_45_days": 0.4352711293911737, "deny_sensitive": 0.06270675160131459, "needs_field_review": 0.06675098961633799, "reduce_to_30_days": 0.4352711293911737}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2945966170009342, "usage": {"input_tokens": 2445, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "b26c77f7182e6192b37bbbd4b5a744f2bbb029e74ffc1308bf5268be4dd4f350", "runtime": null}
{"task_id": "hard-sol-a-multi_hop-12", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-12", "ts": 1791912704.1938686, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "needs_authentication", "ordinal_ev": null, "probs": {"covered_recurrence": 0.046603482406946124, "expired": 0.05984005591667883, "needs_authentication": 0.8793432025803274, "wrong_issue_class": 0.01421325909604757}, "probs_as_returned": {"covered_recurrence": 0.046603482406946124, "expired": 0.05984005591667883, "needs_authentication": 0.8793432025803274, "wrong_issue_class": 0.01421325909604757}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2989009069988242, "usage": {"input_tokens": 2400, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "28861d8f2b10be25d3ea7a8c527836b9f019b596bc59ad48d88b3d970d84d271", "runtime": null}
{"task_id": "hard-sol-a-trap-02", "family": "trap", "split": "public", "group": "hard-sol-a-trap-02", "ts": 1791912704.4108708, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.01098694263059318, "no": 0.9890130573694068}, "probs_as_returned": {"yes": 0.01098694263059318, "no": 0.9890130573694068}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.206059271000413, "usage": {"input_tokens": 366, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "8c751806739f385cc5cfb2394a85a1b2ee3ed4fbf59278faee03c3a87086bb5c", "runtime": null}
{"task_id": "hard-sol-a-trap-04", "family": "trap", "split": "public", "group": "hard-sol-a-trap-04", "ts": 1791912704.6264355, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.02297736991002561, "no": 0.9770226300899744}, "probs_as_returned": {"yes": 0.02297736991002561, "no": 0.9770226300899744}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20704626500082668, "usage": {"input_tokens": 272, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1ba37b6988bc9a786c04aa326af07b25397c7b427b1c14c895090eb2dc64935e", "runtime": null}
{"task_id": "hard-sol-a-trap-05", "family": "trap", "split": "public", "group": "hard-sol-a-trap-05", "ts": 1791912704.8351765, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.49173235429297507, "probs": {"0": 0.70711631087361, "1": 0.1392394573383322, "2": 0.1084397984095306, "3": 0.045204433378527226}, "probs_as_returned": {"0": 0.70711631087361, "1": 0.1392394573383322, "2": 0.1084397984095306, "3": 0.045204433378527226}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2002944710002339, "usage": {"input_tokens": 419, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "cb8d584efe549a895551df40b9e09daaf0d1230bec581a28882f596e9736a9a3", "runtime": null}
{"task_id": "hard-sol-a-trap-06", "family": "trap", "split": "public", "group": "hard-sol-a-trap-06", "ts": 1791912705.0447092, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "keep_open_unresolved", "ordinal_ev": null, "probs": {"await_internal_metrics": 0.006625091807065436, "close_agent_verified": 0.004277480762681299, "close_confirmed": 0.005846622999073827, "keep_open_unresolved": 0.9832508044311793}, "probs_as_returned": {"await_internal_metrics": 0.006625091807065436, "close_agent_verified": 0.004277480762681299, "close_confirmed": 0.005846622999073827, "keep_open_unresolved": 0.9832508044311793}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2011776800009102, "usage": {"input_tokens": 454, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "a0b3ab62bcd4d3b1364d11ff19b2ab3217377c55bf9016b32c6ddaed7732a0c8", "runtime": null}
{"task_id": "hard-sol-a-trap-08", "family": "trap", "split": "public", "group": "hard-sol-a-trap-08", "ts": 1791912705.2553859, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "late", "ordinal_ev": null, "probs": {"canceled": 0.006214098104207464, "indeterminate": 0.006214098104207464, "late": 0.9817341988588881, "on_time": 0.005837604932696947}, "probs_as_returned": {"canceled": 0.006214098104207464, "indeterminate": 0.006214098104207464, "late": 0.9817341988588881, "on_time": 0.005837604932696947}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20225830100025632, "usage": {"input_tokens": 376, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "529ff521bd5054e690fb2bbaa43c5e42dc1780297aa9fabfd0ee3a66695426d7", "runtime": null}
{"task_id": "hard-sol-a-trap-10", "family": "trap", "split": "public", "group": "hard-sol-a-trap-10", "ts": 1791912705.4631703, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "blocked_second_review", "ordinal_ev": null, "probs": {"blocked_failed_test": 0.16978503395167366, "blocked_second_review": 0.7609237309681931, "ready_one_review": 0.03140707326368978, "ready_two_reviews": 0.037884161816443544}, "probs_as_returned": {"blocked_failed_test": 0.16978503395167366, "blocked_second_review": 0.7609237309681931, "ready_one_review": 0.03140707326368978, "ready_two_reviews": 0.037884161816443544}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.198918580001191, "usage": {"input_tokens": 307, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "7589cca94b10f343365c98ea76477de2ace09235f56ed415c0ac0c8febfb8f8f", "runtime": null}
{"task_id": "hard-sol-a-trap-13", "family": "trap", "split": "public", "group": "hard-sol-a-trap-13", "ts": 1791912705.6859992, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.017986209962091555, "no": 0.9820137900379085}, "probs_as_returned": {"yes": 0.017986209962091555, "no": 0.9820137900379085}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21336070199868118, "usage": {"input_tokens": 280, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5c0772139c3dfd32686c7d0aab1830bf0d13e402a75bb9f7356146a6cc4d4763", "runtime": null}
{"task_id": "hard-sol-a-trap-15", "family": "trap", "split": "public", "group": "hard-sol-a-trap-15", "ts": 1791912705.8907044, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.05033063259747688, "no": 0.9496693674025232}, "probs_as_returned": {"yes": 0.05033063259747688, "no": 0.9496693674025232}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.19686709300003713, "usage": {"input_tokens": 382, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "1a56bca2cda4e69a1be9f63181a2ea6ad5bed749144a19e948e21d0f396630c6", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-01", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-01", "ts": 1791912706.1049693, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8438951025545426, "no": 0.1561048974454574}, "probs_as_returned": {"yes": 0.8438951025545426, "no": 0.1561048974454574}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20526789900031872, "usage": {"input_tokens": 362, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "e907ec294a8ee80e11c52c7e07aed32cc1be80e5ff1333ef26bd9db339e66484", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-02", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-02", "ts": 1791912706.3180742, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8438951025545426, "no": 0.1561048974454574}, "probs_as_returned": {"yes": 0.8438951025545426, "no": 0.1561048974454574}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2008650870011479, "usage": {"input_tokens": 286, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2ce3b219c4387beb5d2a3f835ad6ea5f2252bd2c736caf684b16cdd35f15e3a4", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-05", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-05", "ts": 1791912706.54392, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7310585786300049, "no": 0.2689414213699951}, "probs_as_returned": {"yes": 0.7310585786300049, "no": 0.2689414213699951}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21291850699890347, "usage": {"input_tokens": 305, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "bef1fbb13e350fab4a5ac0857f6bda069d7eaee6a967e0a2fe1d7adcffebbbde", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-08", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-08", "ts": 1791912706.749484, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.3629692055196168, "no": 0.6370307944803832}, "probs_as_returned": {"yes": 0.3629692055196168, "no": 0.6370307944803832}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.19740618100149732, "usage": {"input_tokens": 288, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "ebb6b0c2a525eab941efc0b4b49724137d4f03c2e59a7f80f721da6e3876f99c", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-10", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-10", "ts": 1791912706.9670596, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.32082130082460697, "no": 0.679178699175393}, "probs_as_returned": {"yes": 0.32082130082460697, "no": 0.679178699175393}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20966680699893914, "usage": {"input_tokens": 307, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "df88a24dafca418fbbb58620cc8b7a4d28b595d6b5225df67ebf774b2f7e81de", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-14", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-14", "ts": 1791912707.174428, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.02595735719779685, "no": 0.9740426428022031}, "probs_as_returned": {"yes": 0.02595735719779685, "no": 0.9740426428022031}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.19715234099930967, "usage": {"input_tokens": 324, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "36425baf2437750bdd6f3ac5081b6b62b3a385c363ee0bc887dd89c080882952", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-15", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-15", "ts": 1791912707.3811119, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9149009549929797, "no": 0.08509904500702026}, "probs_as_returned": {"yes": 0.9149009549929797, "no": 0.08509904500702026}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.19593242700102564, "usage": {"input_tokens": 280, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "dc9aa1c66f46a8e4af3677313292fc5182129c0e9296006b6edcb09079b324ee", "runtime": null}
{"task_id": "hard-sol-b-judge_hard-18", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-18", "ts": 1791912708.576839, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.10087862273005653, "no": 0.8991213772699435}, "probs_as_returned": {"yes": 0.10087862273005653, "no": 0.8991213772699435}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.1832784700000047, "usage": {"input_tokens": 247, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "04eb44abf7e9f1887dcc4aa1bfb5f327455d59dbde954800e101d7efb6ade794", "runtime": null}
{"task_id": "hard-sol-b-long_policy-01", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-01", "ts": 1791912709.962876, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "reject_entire_claim", "ordinal_ev": null, "probs": {"approve_all": 0.15296693044476137, "approve_lodging_only": 0.13499284231537517, "approve_meals_only": 0.1436991326382892, "reject_entire_claim": 0.5683410946015742}, "probs_as_returned": {"approve_all": 0.15296693044476137, "approve_lodging_only": 0.13499284231537517, "approve_meals_only": 0.1436991326382892, "reject_entire_claim": 0.5683410946015742}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3793377140009397, "usage": {"input_tokens": 2846, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "6312f2e509f93075bb853c6b6a440f14089cea00ccc950eac6d52507d0a7567c", "runtime": null}
{"task_id": "hard-sol-b-long_policy-02", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-02", "ts": 1791912711.352452, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "deny_current_export", "ordinal_ev": null, "probs": {"analytics_owner_only": 0.04610307803019094, "deny_current_export": 0.8699012646710794, "privacy_and_security": 0.04068582356125941, "privacy_only": 0.043309833737470334}, "probs_as_returned": {"analytics_owner_only": 0.04610307803019094, "deny_current_export": 0.8699012646710794, "privacy_and_security": 0.04068582356125941, "privacy_only": 0.043309833737470334}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.380926885000008, "usage": {"input_tokens": 2693, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0928f57cddacd2073c3e732addc3e1325d390db8d000f07fc057fa19c0d9322d", "runtime": null}
{"task_id": "hard-sol-b-long_policy-05", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-05", "ts": 1791912712.7535636, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "delete_profile_omar_april_and_table", "ordinal_ev": null, "probs": {"delete_everything": 0.11654620048858287, "delete_nothing": 0.08526702177179599, "delete_only_profile": 0.3372376130423923, "delete_profile_omar_april_and_table": 0.4609491646972289}, "probs_as_returned": {"delete_everything": 0.11654620048858287, "delete_nothing": 0.08526702177179599, "delete_only_profile": 0.3372376130423923, "delete_profile_omar_april_and_table": 0.4609491646972289}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 1.3930327059988485, "usage": {"input_tokens": 2660, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "357fc8a32e8f5fe202f7582310178df459f8e15a056b06c49516b5a5d8844a9a", "runtime": null}
{"task_id": "hard-sol-b-long_policy-06", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-06", "ts": 1791912713.1308687, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "east_ward", "ordinal_ev": null, "probs": {"east_ward": 0.42078597343256496, "harbor_neighbors": 0.05025571781205226, "no_award": 0.28920168820582476, "river_arts": 0.23975662054955799}, "probs_as_returned": {"east_ward": 0.42078597343256496, "harbor_neighbors": 0.05025571781205226, "no_award": 0.28920168820582476, "river_arts": 0.23975662054955799}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.3675765230000252, "usage": {"input_tokens": 2739, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "03f748672bc0a3be16c61fa16ac978405e74a64a493cf0599273b26890271a6b", "runtime": null}
{"task_id": "hard-sol-b-routing_hard-01", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-01", "ts": 1791912713.3539226, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "office_file_repair", "ordinal_ev": null, "probs": {"data_pipeline": 0.012131358455531665, "desktop_support": 0.005730447973705053, "finance_review": 0.012131358455531665, "office_file_repair": 0.9637131692584042, "spreadsheet_formula": 0.006293665856827351}, "probs_as_returned": {"data_pipeline": 0.012131358455531665, "desktop_support": 0.005730447973705053, "finance_review": 0.012131358455531665, "office_file_repair": 0.9637131692584042, "spreadsheet_formula": 0.006293665856827351}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21366521100026148, "usage": {"input_tokens": 427, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "8ca9029999aeb225c03f87a18d31f6b7bb7d2c283a50ba5e9687ceaef6a5c7eb", "runtime": null}
{"task_id": "hard-sol-b-routing_hard-02", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-02", "ts": 1791912713.568277, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "ledger_reconciliation", "ordinal_ev": null, "probs": {"customer_support": 0.017610860336511245, "fraud_review": 0.004452722809257345, "ledger_reconciliation": 0.96152039486559, "mobile_app": 0.005045595963245334, "payment_disputes": 0.011370426025395897}, "probs_as_returned": {"customer_support": 0.017610860336511245, "fraud_review": 0.004452722809257345, "ledger_reconciliation": 0.96152039486559, "mobile_app": 0.005045595963245334, "payment_disputes": 0.011370426025395897}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20506422399921576, "usage": {"input_tokens": 413, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "d637b400646878f99a15e3e30e28082d2e75de899afc9510ee074700c0a1d961", "runtime": null}
{"task_id": "hard-sol-b-routing_hard-03", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-03", "ts": 1791912713.7918978, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "network_dns", "ordinal_ev": null, "probs": {"api_platform": 0.005116436549975829, "authoritative_dns": 0.005446418356854079, "endpoint_devices": 0.01017524907767728, "network_dns": 0.9750202211342405, "tls_certificates": 0.004241674881252431}, "probs_as_returned": {"api_platform": 0.005116436549975829, "authoritative_dns": 0.005446418356854079, "endpoint_devices": 0.01017524907767728, "network_dns": 0.9750202211342405, "tls_certificates": 0.004241674881252431}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21387168199908047, "usage": {"input_tokens": 407, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "bb53aa0e1d701863dbea508486f85795078c46865e70ebc4bc92838e615acc02", "runtime": null}
{"task_id": "hard-sol-b-routing_hard-07", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-07", "ts": 1791912714.001411, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "release_notarization", "ordinal_ev": null, "probs": {"ci_infrastructure": 0.00620811376333022, "code_signing": 0.007034714507380512, "endpoint_security": 0.003537276596321213, "macos_runtime": 0.0024311322807697914, "release_notarization": 0.9807887628521984}, "probs_as_returned": {"ci_infrastructure": 0.00620811376333022, "code_signing": 0.007034714507380512, "endpoint_security": 0.003537276596321213, "macos_runtime": 0.0024311322807697914, "release_notarization": 0.9807887628521984}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20086351400095737, "usage": {"input_tokens": 392, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "d3aa05af88aa9aeba5fb4c73dd0fa75302ae7586f19c7002e73bd95a77ed823c", "runtime": null}
{"task_id": "hard-sol-b-routing_hard-09", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-09", "ts": 1791912714.2217772, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "source_access", "ordinal_ev": null, "probs": {"git_client": 0.012791883457137285, "identity_login": 0.006847001812715743, "repository_admin": 0.01748443164828868, "source_access": 0.9546176223775203, "vpn_network": 0.008259060704338017}, "probs_as_returned": {"git_client": 0.012791883457137285, "identity_login": 0.006847001812715743, "repository_admin": 0.01748443164828868, "source_access": 0.9546176223775203, "vpn_network": 0.008259060704338017}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2117115230012132, "usage": {"input_tokens": 393, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "64dc7148e674ed07f412317ac7c95d420f0b55416ed95a6e6993be30049a1b27", "runtime": null}
{"task_id": "hard-sol-b-temporal_numeric-01", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-01", "ts": 1791912714.446992, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "within_window", "ordinal_ev": null, "probs": {"before_open": 0.10874998898547071, "late_by_2h_or_more": 0.20317172729290436, "late_by_under_2h": 0.23022372848886966, "within_window": 0.45785455523275526}, "probs_as_returned": {"before_open": 0.10874998898547071, "late_by_2h_or_more": 0.20317172729290436, "late_by_under_2h": 0.23022372848886966, "within_window": 0.45785455523275526}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21483235999949102, "usage": {"input_tokens": 397, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0bc1fb9b61fd287a154f72dc685af7cca688e25c95cb8a53419caa953ebd158c", "runtime": null}
{"task_id": "hard-sol-b-temporal_numeric-02", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-02", "ts": 1791912714.6577058, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "at_least_100gb_below", "ordinal_ev": null, "probs": {"at_least_100gb_below": 0.3564675601642158, "exactly_equal": 0.16838335255877088, "over_limit": 0.2301527549393719, "under_100gb_below": 0.2449963323376414}, "probs_as_returned": {"at_least_100gb_below": 0.3564675601642158, "exactly_equal": 0.16838335255877088, "over_limit": 0.2301527549393719, "under_100gb_below": 0.2449963323376414}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2019981880002888, "usage": {"input_tokens": 414, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "2e242dea1e7fb902d7e707ff93bebbcfaa79c58d30e58f636d3d87751c306c5e", "runtime": null}
{"task_id": "hard-sol-b-temporal_numeric-03", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-03", "ts": 1791912714.881002, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "12_credits", "ordinal_ev": null, "probs": {"12_credits": 0.3041365610059184, "15_credits": 0.1732917888339009, "6_credits": 0.2857098582881274, "9_credits": 0.23686179187205336}, "probs_as_returned": {"12_credits": 0.3041365610059184, "15_credits": 0.1732917888339009, "6_credits": 0.2857098582881274, "9_credits": 0.23686179187205336}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2080901579993224, "usage": {"input_tokens": 342, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "fbfc96b013575d8159ac6a12924a8e7a03862161a50bcaa982b1697d853d8f64", "runtime": null}
{"task_id": "hard-sol-b-temporal_numeric-04", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-04", "ts": 1791912715.1052933, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "clear_fail", "ordinal_ev": null, "probs": {"clear_fail": 0.37438759197342575, "narrow_fail": 0.35170459445511776, "pass": 0.2739078135714566}, "probs_as_returned": {"clear_fail": 0.37438759197342575, "narrow_fail": 0.35170459445511776, "pass": 0.2739078135714566}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.21301556999969762, "usage": {"input_tokens": 340, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5beaab59f3cbf07ab3bcb6f3687f65866054806a8878982358afbd4b14bf00e9", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-03", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-03", "ts": 1791912715.3157647, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.1561048974454574, "no": 0.8438951025545426}, "probs_as_returned": {"yes": 0.1561048974454574, "no": 0.8438951025545426}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2010439480000059, "usage": {"input_tokens": 469, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "541116f4a2a0e7cb8b831a834059542cd8002fe8ea2ae1a730b070225865134a", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-05", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-05", "ts": 1791912715.5246882, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5156199157230156, "no": 0.4843800842769844}, "probs_as_returned": {"yes": 0.5156199157230156, "no": 0.4843800842769844}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.19970251099948655, "usage": {"input_tokens": 431, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "332359f699915f126671b6055a4662eb8e0068f95e0fc039e6d35f8444147d45", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-07", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-07", "ts": 1791912715.734228, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.5, "no": 0.5}, "probs_as_returned": {"yes": 0.5, "no": 0.5}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2006927149996045, "usage": {"input_tokens": 395, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "335547f757eb7d9c27b8ae5b2ca9722a5e570f43147a7b7180b088ea1b49aca5", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-08", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-08", "ts": 1791912715.941932, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.1561048974454574, "no": 0.8438951025545426}, "probs_as_returned": {"yes": 0.1561048974454574, "no": 0.8438951025545426}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1994855249995453, "usage": {"input_tokens": 389, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c2a2d90e040087b64f129536927d1330bf13efb8a1911565821b7fb837d75ea1", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-09", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-09", "ts": 1791912716.1521218, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7310585786300049, "no": 0.2689414213699951}, "probs_as_returned": {"yes": 0.7310585786300049, "no": 0.2689414213699951}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20071710599950165, "usage": {"input_tokens": 420, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "683e72b60efd31fb41a1eef439cfe230c9ad710693fc7f069cfc1f46d8e07762", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-10", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-10", "ts": 1791912716.3699758, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.665410558746814, "no": 0.334589441253186}, "probs_as_returned": {"yes": 0.665410558746814, "no": 0.334589441253186}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.208957102999193, "usage": {"input_tokens": 449, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "bdc2eb6a9c53f8faec668d7dd12f71b5a5491e21a924bb860ee50d71f69a7989", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-11", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-11", "ts": 1791912716.5761902, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.07585818002124356, "no": 0.9241418199787564}, "probs_as_returned": {"yes": 0.07585818002124356, "no": 0.9241418199787564}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.19852217900006508, "usage": {"input_tokens": 420, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "03348119dbb7f0ce7202b70e399f2290a4dad736ead3902f9fb9b8596b2038ab", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-13", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-13", "ts": 1791912716.7841148, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.4073334000459302, "no": 0.5926665999540698}, "probs_as_returned": {"yes": 0.4073334000459302, "no": 0.5926665999540698}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1994684019991837, "usage": {"input_tokens": 410, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "3f29db6f6a94fd245c0c5b6687cd2e65cb1f335fe2b8d8b8dc12ce62ff012675", "runtime": null}
{"task_id": "hard-sol-c-judge_hard-15", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-15", "ts": 1791912717.006368, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8267117940706734, "no": 0.1732882059293266}, "probs_as_returned": {"yes": 0.8267117940706734, "no": 0.1732882059293266}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.2117445090007095, "usage": {"input_tokens": 364, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "6793d1175de3398848a8061651fb9f8c61516c4595bc60022054470c268c14a1", "runtime": null}
{"task_id": "hard-sol-c-multi_hop-07", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-07", "ts": 1791912717.3452723, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "programme_review", "ordinal_ev": null, "probs": {"partial_release": 0.019951971392199265, "programme_review": 0.9613398179029721, "reject_draw": 0.011368290618797865, "release_draw": 0.00733992008603087}, "probs_as_returned": {"partial_release": 0.019951971392199265, "programme_review": 0.9613398179029721, "reject_draw": 0.011368290618797865, "release_draw": 0.00733992008603087}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.329477950999717, "usage": {"input_tokens": 2629, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "17eda62b8fe395ba8c9c119ccfb78666cb482c1e57d5ad7e35bb2fc43c5f08d8", "runtime": null}
{"task_id": "hard-sol-c-multi_hop-09", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-09", "ts": 1791912717.6716979, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "bus_substitution", "ordinal_ev": null, "probs": {"bus_substitution": 0.9307826618442643, "call_fen_cross": 0.013276901606987553, "pass_fen_cross": 0.015044700517478391, "terminate_oakmere": 0.04089573603126991}, "probs_as_returned": {"bus_substitution": 0.9307826618442643, "call_fen_cross": 0.013276901606987553, "pass_fen_cross": 0.015044700517478391, "terminate_oakmere": 0.04089573603126991}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.31623825400129135, "usage": {"input_tokens": 2653, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0ba18cd0847ddd4d6fad0aadaad0ca674c29199505e3081a3272c9816fb4d0c8", "runtime": null}
{"task_id": "hard-sol-c-multi_hop-11", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-11", "ts": 1791912718.000245, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coverage_excluded", "ordinal_ev": null, "probs": {"adjuster_referral": 0.05674041101889167, "coverage_confirmed": 0.09958252259654728, "coverage_excluded": 0.783277213258229, "documentation_needed": 0.060399853126332045}, "probs_as_returned": {"adjuster_referral": 0.05674041101889167, "coverage_confirmed": 0.09958252259654728, "coverage_excluded": 0.783277213258229, "documentation_needed": 0.060399853126332045}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.3193026459994144, "usage": {"input_tokens": 2729, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "5a9b1cf750ba4f158dc15be0f9031ce19c645c1a44d2eb9ce24d021b99473fb6", "runtime": null}
{"task_id": "hard-sol-c-multi_hop-12", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-12", "ts": 1791912718.2175345, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "conservation_hold", "ordinal_ev": null, "probs": {"conservation_hold": 0.7431727821686612, "deny_request": 0.09448381635615277, "digital_copy_only": 0.08875933130943944, "supervised_delivery": 0.07358407016574658}, "probs_as_returned": {"conservation_hold": 0.7431727821686612, "deny_request": 0.09448381635615277, "digital_copy_only": 0.08875933130943944, "supervised_delivery": 0.07358407016574658}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20851856300032523, "usage": {"input_tokens": 650, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "c6e95e01712de52e75d474b03b3bd8003f995121d8f93ee20ec50b2de6b28a0b", "runtime": null}
{"task_id": "hard-sol-c-multi_hop-13", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-13", "ts": 1791912718.5515316, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "normal_release", "ordinal_ev": null, "probs": {"engineering_review": 0.006665112460151473, "normal_release": 0.9891903959850263, "rework": 0.002610095541828959, "scrap_lot": 0.0015343960129930842}, "probs_as_returned": {"engineering_review": 0.006665112460151473, "normal_release": 0.9891903959850263, "rework": 0.002610095541828959, "scrap_lot": 0.0015343960129930842}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "decision-medium", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.3233822570000484, "usage": {"input_tokens": 2699, "output_tokens": 0}, "cost_usd": null, "cost_basis": "no_billable_account_public_endpoint", "reserved_usd": 0.0, "charged_usd": 0.0, "raw_sha256": "0a16ec7a1f41a2fb95f1ec7d647ef7f6c0ce6ab9fdf3d3f1a21dc15e4ad1e336", "runtime": null}