{
  "claim": "The place ORPO wins is the one its own paper never measured. On the Open LLM Leaderboard tasks in SimPO's Mistral-Base setting, ORPO posts the highest GSM8K and the highest MMLU of any row — 42.15 against the SFT checkpoint's 28.13, and 63.20 against 60.10 — while DPO takes GSM8K down to 21.76 and loses 1.6 points of MMLU.",
  "method": "Transcribed from Table 9 of SimPO (arXiv 2405.14734v3), Mistral-Base block, in full — every objective they ran, not a selection. Same checkpoints and protocol as the AlpacaEval table above, so the SFT row is the shared starting point for every other row. SimPO's own text attributes ORPO's math retention to the supervised fine-tuning term in its objective.",
  "source": "https://arxiv.org/abs/2405.14734",
  "captured": "2026-09-19",
  "note": "This is one setting. ORPO is not uniformly best across the four settings in that table — see the GSM8K deltas below. Few-shot counts are the leaderboard's: MMLU 5-shot, ARC 25-shot, HellaSwag 10-shot, TruthfulQA 0-shot, Winograd 5-shot, GSM8K 5-shot. These are multiple-choice and exact-match tasks rather than LLM-judged preference, which is why they are worth more than the win rates above, and why it is strange that the ORPO paper reports none of them.",
  "columns": [
    { "key": "method", "label": "objective", "align": "left" },
    { "key": "mmlu", "label": "MMLU", "align": "right" },
    { "key": "arc", "label": "ARC", "align": "right" },
    { "key": "hs", "label": "HellaSwag", "align": "right" },
    { "key": "tqa", "label": "TruthfulQA", "align": "right" },
    { "key": "wg", "label": "Winograd", "align": "right" },
    { "key": "gsm", "label": "GSM8K", "align": "right" },
    { "key": "avg", "label": "avg", "align": "right" }
  ],
  "rows": [
    { "method": "SFT (start point)", "mmlu": "60.10", "arc": "58.28", "hs": "80.76", "tqa": "40.35", "wg": "76.40", "gsm": "28.13", "avg": "57.34" },
    { "method": "RRHF", "mmlu": "57.41", "arc": "52.13", "hs": "80.16", "tqa": "43.73", "wg": "76.64", "gsm": "4.78", "avg": "52.48" },
    { "method": "SLiC-HF", "mmlu": "59.24", "arc": "55.38", "hs": "81.15", "tqa": "48.36", "wg": "77.35", "gsm": "33.74", "avg": "59.20" },
    { "method": "DPO", "mmlu": "58.48", "arc": "61.26", "hs": "83.59", "tqa": "53.06", "wg": "76.80", "gsm": "21.76", "avg": "59.16" },
    { "method": "IPO", "mmlu": "60.23", "arc": "60.84", "hs": "83.30", "tqa": "45.44", "wg": "77.58", "gsm": "27.14", "avg": "59.09" },
    { "method": "CPO", "mmlu": "59.39", "arc": "57.00", "hs": "80.75", "tqa": "47.07", "wg": "76.48", "gsm": "33.06", "avg": "58.96" },
    { "method": "KTO", "mmlu": "60.90", "arc": "62.37", "hs": "84.88", "tqa": "56.60", "wg": "77.27", "gsm": "38.51", "avg": "63.42" },
    { "method": "ORPO", "mmlu": "63.20", "arc": "61.01", "hs": "84.09", "tqa": "47.91", "wg": "78.61", "gsm": "42.15", "avg": "62.83" },
    { "method": "R-DPO", "mmlu": "59.58", "arc": "61.35", "hs": "84.29", "tqa": "46.12", "wg": "76.56", "gsm": "18.12", "avg": "57.67" },
    { "method": "SimPO", "mmlu": "59.21", "arc": "62.63", "hs": "83.60", "tqa": "50.68", "wg": "77.27", "gsm": "22.21", "avg": "59.27" }
  ]
}