{
  "claim": "Reversing the order of a question's options, changing nothing else, changes openjev's answer on 10 of 36 cases — 27.8%. This is the cost of the vocabulary-readout design, where options live in the prompt as lettered text and the model only ever picks a letter. It is also exactly the failure the per-option scalar scorer removes by construction, and the reason TypeSafe can say Jev 'doesn't see a level's number or its neighbours'.",
  "method": "openjev commits both its benchmark fixtures and its raw predictions. benchmarks/data/perturbations108.jsonl carries provenance.variant and provenance.base_id, so every option_reversal row can be joined back to the authored144 row it was derived from. For each pair I realigned both predictions from their own option_ids into the row's option order, took the argmax, and compared the chosen option id. Model: a frozen Qwen3.5-4B read through direct option logits, no training.",
  "source": "https://github.com/TheoLeeCJ/openjev",
  "captured": "2026-09-18",
  "note": "36 pairs from 36 source cases, so these are not independent of the authored set they came from, and 10/36 carries a wide interval. The direction is not in doubt though: the flips are confident in both directions, not coin-flips near 0.5. Accuracy happens to be similar either way (80.6% reversed against 80.6% on irrelevant context) — order sensitivity shows up as churn between the two runs, not as a drop in the aggregate, which is exactly why an accuracy number alone would hide it.",
  "columns": [
    {
      "key": "orig",
      "label": "original chose"
    },
    {
      "key": "op",
      "label": "p",
      "align": "right"
    },
    {
      "key": "rev",
      "label": "reversed chose"
    },
    {
      "key": "rp",
      "label": "p",
      "align": "right"
    },
    {
      "key": "gold",
      "label": "gold"
    },
    {
      "key": "verdict",
      "label": ""
    }
  ],
  "rows": [
    {
      "orig": "insufficient",
      "op": "0.871",
      "rev": "contradicted",
      "rp": "0.987",
      "gold": "contradicted",
      "verdict": "reversal fixed it"
    },
    {
      "orig": "insufficient",
      "op": "0.506",
      "rev": "prohibited",
      "rp": "0.657",
      "gold": "prohibited",
      "verdict": "reversal fixed it"
    },
    {
      "orig": "A",
      "op": "0.904",
      "rev": "B",
      "rp": "0.500",
      "gold": "A",
      "verdict": "reversal broke it"
    },
    {
      "orig": "A",
      "op": "0.832",
      "rev": "B",
      "rp": "0.898",
      "gold": "B",
      "verdict": "reversal fixed it"
    },
    {
      "orig": "A",
      "op": "0.817",
      "rev": "B",
      "rp": "0.970",
      "gold": "B",
      "verdict": "reversal fixed it"
    },
    {
      "orig": "prohibited",
      "op": "0.877",
      "rev": "permitted",
      "rp": "0.621",
      "gold": "prohibited",
      "verdict": "reversal broke it"
    }
  ]
}