{
  "claim": "On the one public benchmark that ran all of them on the same items, two untrained decoders read through their own logits beat every trained small encoder in the field — and a zero-shot encoder, the control, lands with the trained encoders rather than with the decoders. The thing that wins is the decoder, not the absence of training.",
  "method": "JevBench v1.3.0 hard tier: 220 items (129 choice, 77 noul, 14 score), 111 public and 109 held out, frozen 2026-09-19. Accuracy is raw correct-out-of-220 on identical items. Calibration is the suite's own 0-100 axis, higher is better. The training column is the benchmark's own `underlying` field, quoted, not my inference.",
  "source": "https://benchmarkheaven.com/api/jevbench/v1.2",
  "captured": "2026-09-22",
  "note": "Hard-tier items do not all have the same option count, so raw accuracy is not chance-corrected; the suite's separate Intelligence axis is. The ordering here is the same under both. Encoder entries were served on CPU and decoder entries on GPU, which moves the speed axis and nothing in these two columns.",
  "columns": [
    { "key": "system", "label": "system" },
    { "key": "family", "label": "readout family" },
    { "key": "trained", "label": "trained for the decision?" },
    { "key": "hard", "label": "hard tier", "align": "right", "mono": true },
    { "key": "calib", "label": "calibration axis", "align": "right", "mono": true }
  ],
  "rows": [
    {
      "system": "Jev 1.13.0",
      "family": "closed",
      "trained": "yes, and nobody outside TypeSafe knows how",
      "hard": "163/220",
      "calib": "82.7"
    },
    {
      "system": "jqv (Qwen3-32B)",
      "family": "decoder, direct-logit readout",
      "trained": "no — \"read as a direct-logit classifier (no fine-tuning)\"",
      "hard": "142/220",
      "calib": "79.0"
    },
    {
      "system": "Bespoke Nimble 9B",
      "family": "decoder, allowed-token readout",
      "trained": "yes — LoRA on 2,676 contrastive rows",
      "hard": "144/220",
      "calib": "65.3"
    },
    {
      "system": "SemIf / openjev (Qwen3.5-4B)",
      "family": "decoder, option-letter readout",
      "trained": "no — \"Qwen/Qwen3.5-4B (frozen, BF16)\"",
      "hard": "131/220",
      "calib": "72.6"
    },
    {
      "system": "kev 8B",
      "family": "decoder, LoRA + pointer head",
      "trained": "yes",
      "hard": "104/220",
      "calib": "44.2"
    },
    {
      "system": "kev 4B",
      "family": "decoder, LoRA + pointer head",
      "trained": "yes",
      "hard": "93/220",
      "calib": "42.0"
    },
    {
      "system": "kev 0.6B",
      "family": "decoder, LoRA + pointer head",
      "trained": "yes",
      "hard": "88/220",
      "calib": "51.1"
    },
    {
      "system": "openJev Verdict (151M)",
      "family": "encoder, GLiClass head",
      "trained": "yes — \"gliclass-modern-base-v2.0 fine-tuned\"",
      "hard": "84/220",
      "calib": "51.3"
    },
    {
      "system": "jeff (GLiFormer 400M)",
      "family": "encoder, scoring head",
      "trained": "yes, head only; backbone frozen",
      "hard": "83/220",
      "calib": "64.6"
    },
    {
      "system": "open-jev-deberta-v3-large",
      "family": "encoder, per-option scalar head",
      "trained": "yes — CE + Brier on 42,000 questions",
      "hard": "80/220",
      "calib": "66.4"
    },
    {
      "system": "Laya (ModernBERT-large 421M)",
      "family": "encoder, option-marker head",
      "trained": "yes — \"RLCD-trained\"",
      "hard": "75/220",
      "calib": "62.5"
    },
    {
      "system": "OpenDecision (ModernBERT-large)",
      "family": "encoder, zero-shot NLI",
      "trained": "no — the control for \"untrained\"",
      "hard": "73/220",
      "calib": "56.1"
    }
  ]
}