{
  "claim": "The only matched head-to-head that includes ORPO is the SimPO paper's, which trained every objective on the same data from the same checkpoints and tuned each one's hyperparameters. AlpacaEval 2.0 length-controlled win rate, four settings. ORPO loses to DPO in three of four and ties in the fourth.",
  "method": "Transcribed from Table 4 of SimPO (arXiv 2405.14734v3). Base settings SFT on UltraChat then preference-train on UltraFeedback; Instruct settings use the off-the-shelf instruct model as the SFT checkpoint. SimPO's footnote 7 states that they ran ORPO from the same SFT checkpoints as every other baseline because that \"yields better results than starting from base checkpoints\" — so these ORPO numbers are SFT → ORPO, not the single-stage recipe the ORPO paper argues for.",
  "source": "https://arxiv.org/abs/2405.14734",
  "captured": "2026-09-19",
  "note": "Not my measurement, and SimPO is an interested party: it is the paper proposing the winning row. The value here is that every column was produced by one group under one protocol, which is not true of the ORPO paper's own 7B comparisons.",
  "columns": [
    { "key": "method", "label": "objective", "align": "left" },
    { "key": "ref", "label": "ref model", "align": "left" },
    { "key": "mb", "label": "Mistral-Base", "align": "right" },
    { "key": "mi", "label": "Mistral-Inst", "align": "right" },
    { "key": "lb", "label": "Llama-3-Base", "align": "right" },
    { "key": "li", "label": "Llama-3-Inst", "align": "right" }
  ],
  "rows": [
    { "method": "SFT only", "ref": "—", "mb": "8.4", "mi": "17.1", "lb": "6.2", "li": "26.0" },
    { "method": "DPO", "ref": "yes", "mb": "15.1", "mi": "26.8", "lb": "18.2", "li": "40.3" },
    { "method": "IPO", "ref": "yes", "mb": "11.8", "mi": "20.3", "lb": "14.4", "li": "35.6" },
    { "method": "KTO", "ref": "yes", "mb": "13.1", "mi": "24.5", "lb": "14.2", "li": "33.1" },
    { "method": "R-DPO", "ref": "yes", "mb": "17.4", "mi": "27.3", "lb": "17.6", "li": "41.1" },
    { "method": "CPO", "ref": "no", "mb": "9.8", "mi": "23.8", "lb": "10.8", "li": "28.9" },
    { "method": "ORPO", "ref": "no", "mb": "14.7", "mi": "24.5", "lb": "12.2", "li": "28.5" },
    { "method": "SimPO", "ref": "no", "mb": "21.5", "mi": "32.1", "lb": "22.0", "li": "44.7" }
  ]
}