[{"record_id":"16d849d843b034f3","ts_utc":"2026-09-20T00:40:41Z","kind":"compare","gate_version":"0.1.0.dev1","spec_name":"drift-blocks","spec_hash":"1b6d78c7491ba911d9e248109ca47e41cfc07c22c88be59c552502a5e3b88451","graders_hash":"ef0cdb4508edb15b","baseline":{"kind":"drift_block","month":"2026-09","arm":"anthropic-alias","repeats":"all","graded_by":"ef0cdb4508edb15b","run_id":"drift-2026-09","suite_hash":"72f780dfb525d84da3c20b31012e30f495d79fd972aee35fdbd745088048f8a1","suite_version":"v1"},"candidate":{"kind":"drift_block","month":"2026-09-run2","arm":"anthropic-alias","repeats":"all","graded_by":"ef0cdb4508edb15b","run_id":"drift-2026-09-run2","suite_hash":"72f780dfb525d84da3c20b31012e30f495d79fd972aee35fdbd745088048f8a1","suite_version":"v1"},"suites":[{"suite":"closed_form_reasoning","paired_items":120,"baseline_accuracy":1.0,"candidate_accuracy":1.0,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":0.0,"items_needed":118,"under_powered":false,"effective_items":10.471112817514681,"verdict":"pass","reasons":["non-inferior: +0.0% (+0.0% to +0.0%) over 120 items stays above -3% (p 0.000, Holm 0.000)","local dependence (gsm8k): 120 items are worth about 10; corrected interval +0.0% to +0.0% shown, not used"]},{"suite":"multiple_choice","paired_items":100,"baseline_accuracy":0.93,"candidate_accuracy":0.92,"difference":-0.01,"lo":-0.05,"hi":0.019999999999999997,"delta":0.03,"p_inferior":0.1975,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":46.54626698938746,"verdict":"warn","reasons":["100 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only","local dependence (mmlu): 100 items are worth about 47; corrected interval -6.9% to +3.4% shown, not used"]},{"suite":"instruction_following","paired_items":60,"baseline_accuracy":0.95,"candidate_accuracy":0.95,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["60 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"structured_extraction","paired_items":20,"baseline_accuracy":0.9,"candidate_accuracy":0.9,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["20 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"structured_extraction_heldout","paired_items":20,"baseline_accuracy":0.85,"candidate_accuracy":0.85,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["20 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"refusal_calibration","paired_items":40,"baseline_accuracy":0.85,"candidate_accuracy":0.85,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["40 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"long_context_recall","paired_items":20,"baseline_accuracy":1.0,"candidate_accuracy":1.0,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["20 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"paraphrase_robustness","paired_items":40,"baseline_accuracy":1.0,"candidate_accuracy":1.0,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":9.03321966531921,"verdict":"warn","reasons":["40 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only","local dependence (gsm8k): 40 items are worth about 9; corrected interval +0.0% to +0.0% shown, not used"]}],"lines":["latency p50: 1,387 ms against 1,648 ms (-16%)","cost per call: US$0.00133 against US$0.00133 (+0%)"],"passed":true,"reasons":["pass: 1 of 8 suites decided, none inferior","under-powered, not decided: multiple_choice, instruction_following, structured_extraction, structured_extraction_heldout, refusal_calibration, long_context_recall, paraphrase_robustness"],"supersedes":null},{"record_id":"e3fdc06fa056e927","ts_utc":"2026-09-20T00:40:46Z","kind":"compare","gate_version":"0.1.0.dev1","spec_name":"drift-blocks","spec_hash":"1b6d78c7491ba911d9e248109ca47e41cfc07c22c88be59c552502a5e3b88451","graders_hash":"ef0cdb4508edb15b","baseline":{"kind":"drift_block","month":"2026-09","arm":"openweights-control","repeats":"all","graded_by":"ef0cdb4508edb15b","run_id":"drift-2026-09","suite_hash":"72f780dfb525d84da3c20b31012e30f495d79fd972aee35fdbd745088048f8a1","suite_version":"v1"},"candidate":{"kind":"drift_block","month":"2026-09-run2","arm":"openweights-control","repeats":"all","graded_by":"ef0cdb4508edb15b","run_id":"drift-2026-09-run2","suite_hash":"72f780dfb525d84da3c20b31012e30f495d79fd972aee35fdbd745088048f8a1","suite_version":"v1"},"suites":[{"suite":"closed_form_reasoning","paired_items":120,"baseline_accuracy":0.9666666666666667,"candidate_accuracy":0.95,"difference":-0.016666666666666666,"lo":-0.05,"hi":0.016666666666666666,"delta":0.03,"p_inferior":0.222,"p_adjusted":0.222,"items_needed":118,"under_powered":false,"effective_items":10.471112817514681,"verdict":"block","reasons":["cannot rule out a 3% drop: -1.7% (-5.0% to +1.7%) over 120 items, 3 worse and 1 better (p 0.222, Holm 0.222)","local dependence (gsm8k): 120 items are worth about 10; corrected interval -13.0% to +9.6% shown, not used"]},{"suite":"multiple_choice","paired_items":100,"baseline_accuracy":0.82,"candidate_accuracy":0.83,"difference":0.01,"lo":0.0,"hi":0.03,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":46.54626698938746,"verdict":"warn","reasons":["100 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only","local dependence (mmlu): 100 items are worth about 47; corrected interval -0.5% to +3.9% shown, not used"]},{"suite":"instruction_following","paired_items":60,"baseline_accuracy":0.95,"candidate_accuracy":0.9666666666666667,"difference":0.016666666666666666,"lo":0.0,"hi":0.05,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["60 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"structured_extraction","paired_items":20,"baseline_accuracy":0.95,"candidate_accuracy":0.95,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["20 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"structured_extraction_heldout","paired_items":20,"baseline_accuracy":0.9,"candidate_accuracy":0.9,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["20 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"refusal_calibration","paired_items":40,"baseline_accuracy":0.925,"candidate_accuracy":0.95,"difference":0.025,"lo":0.0,"hi":0.075,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["40 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"long_context_recall","paired_items":20,"baseline_accuracy":1.0,"candidate_accuracy":1.0,"difference":0.0,"lo":0.0,"hi":0.0,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":null,"verdict":"warn","reasons":["20 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only"]},{"suite":"paraphrase_robustness","paired_items":40,"baseline_accuracy":0.95,"candidate_accuracy":0.975,"difference":0.025,"lo":0.0,"hi":0.075,"delta":0.03,"p_inferior":0.0,"p_adjusted":null,"items_needed":118,"under_powered":true,"effective_items":9.03321966531921,"verdict":"warn","reasons":["40 paired items, fewer than the 118 needed to see a 3% drop at 80% power: under-powered, warns only","local dependence (gsm8k): 40 items are worth about 9; corrected interval -2.8% to +13.0% shown, not used"]}],"lines":["latency p50: 977 ms against 993 ms (-2%)","cost per call: US$0.00060 against US$0.00060 (+0%)"],"passed":false,"reasons":["closed_form_reasoning: cannot rule out a 3% drop: -1.7% (-5.0% to +1.7%) over 120 items, 3 worse and 1 better (p 0.222, Holm 0.222)","under-powered, not decided: multiple_choice, instruction_following, structured_extraction, structured_extraction_heldout, refusal_calibration, long_context_recall, paraphrase_robustness"],"supersedes":null}]