{
 "claim": "openjev publishes raw per-row option logits and its own gold labels but no calibration result \u2014 it lists \"calibrated probabilities suitable for operational thresholds\" under what it has NOT established. Joining the two files gives one anyway. A completely untrained Qwen3.5-4B read through direct option logits lands at ECE 0.071 over 252 rows, and one fitted scalar temperature takes it to 0.080 on held-out perturbed rows and 0.060 on held-out clean ones. Accuracy never moves, because dividing every logit by one positive number cannot reorder them.",
 "method": "results/raw/predictions/direct-{authored144,perturbations108}.jsonl joined by row id to benchmarks/data/{authored144,perturbations108}.jsonl; each row's option_logits reordered from its option_ids into the gold file's options order; softmax at temperature T; ECE over 10 equal-width confidence bins; Brier is the multi-class sum of squared errors; NLL is -log p(gold). Temperatures fitted by grid search over 0.20 to 12.00 in steps of 0.01, minimising the stated objective on the stated split.",
 "source": "https://github.com/TheoLeeCJ/openjev",
 "captured": "2026-09-18",
 "note": "The 252 rows come from 36 authored cases (144 = 36 groups \u00d7 4 variants; 108 = the same 36 groups \u00d7 3 perturbations), so rows inside a group are correlated and the effective sample is far below 252. No confidence intervals are computed here; openjev's own evaluation code does bootstrap by source group and mine does not. The two rows that matter most are the last two: minimising NLL and minimising ECE on the very same data pick different temperatures (1.32 against 1.19) and neither is best on the other's metric.",
 "columns": [
  {
   "key": "split",
   "label": "evaluated on"
  },
  {
   "key": "fit",
   "label": "temperature fitted on"
  },
  {
   "key": "T",
   "label": "T",
   "align": "right"
  },
  {
   "key": "acc",
   "label": "accuracy",
   "align": "right"
  },
  {
   "key": "conf",
   "label": "mean conf",
   "align": "right"
  },
  {
   "key": "ece",
   "label": "ECE",
   "align": "right"
  },
  {
   "key": "brier",
   "label": "Brier",
   "align": "right"
  },
  {
   "key": "nll",
   "label": "NLL",
   "align": "right"
  }
 ],
 "rows": [
  {
   "split": "authored144",
   "fit": "\u2014",
   "T": "1.00",
   "acc": "0.8056",
   "conf": "0.8628",
   "ece": "0.0678",
   "brier": "0.2449",
   "nll": "0.4221"
  },
  {
   "split": "authored144",
   "fit": "perturbations108 (held out)",
   "T": "1.42",
   "acc": "0.8056",
   "conf": "0.8106",
   "ece": "0.0599",
   "brier": "0.2401",
   "nll": "0.4173"
  },
  {
   "split": "perturbations108",
   "fit": "\u2014",
   "T": "1.00",
   "acc": "0.7778",
   "conf": "0.8454",
   "ece": "0.1209",
   "brier": "0.3240",
   "nll": "0.5581"
  },
  {
   "split": "perturbations108",
   "fit": "authored144 (held out)",
   "T": "1.23",
   "acc": "0.7778",
   "conf": "0.8113",
   "ece": "0.0801",
   "brier": "0.3119",
   "nll": "0.5339"
  },
  {
   "split": "both, 252",
   "fit": "\u2014",
   "T": "1.00",
   "acc": "0.7937",
   "conf": "0.8553",
   "ece": "0.0709",
   "brier": "0.2788",
   "nll": "0.4804"
  },
  {
   "split": "both, 252",
   "fit": "same rows, min NLL (in-sample)",
   "T": "1.32",
   "acc": "0.7937",
   "conf": "0.8124",
   "ece": "0.0565",
   "brier": "0.2696",
   "nll": "0.4642"
  },
  {
   "split": "both, 252",
   "fit": "same rows, min ECE (in-sample)",
   "T": "1.19",
   "acc": "0.7937",
   "conf": "0.8295",
   "ece": "0.0480",
   "brier": "0.2720",
   "nll": "0.4664"
  }
 ]
}