{
  "claim": "On the official AlpacaEval 2.0 leaderboard, Mistral-ORPO-β beats Zephyr-β by 1.57 raw points and 1.51 length-controlled points. The denominator is 805 instructions, and the margin is 17 extra head-to-head wins. Both models' standard errors are about ±1.0, so the gap is roughly one standard error — real, and not separable from noise on this benchmark alone.",
  "method": "Read from the leaderboard CSV committed in the alpaca_eval repository (src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv) rather than from the ORPO paper, so the ORPO and Zephyr rows come from the same evaluator run. The paper reports 12.20% and 10.99%; the leaderboard's weighted win rate reads 12.57% and 10.99%. Length-controlled win rate was introduced after the paper and is not in it. Mistral-7B-Instruct-v0.2 is included as the off-the-shelf baseline nobody trained for this comparison.",
  "source": "https://github.com/tatsu-lab/alpaca_eval",
  "captured": "2026-09-19",
  "note": "Judge is GPT-4-turbo; the opponent whose outputs you must beat is also GPT-4-turbo. n_total is 805 for every row. avg_length is characters of the model's response, and it is the reason the length-controlled column exists.",
  "columns": [
    { "key": "model", "label": "model", "align": "left" },
    { "key": "align", "label": "alignment", "align": "left" },
    { "key": "wr", "label": "raw WR %", "align": "right" },
    { "key": "se", "label": "± SE", "align": "right" },
    { "key": "lc", "label": "LC WR %", "align": "right" },
    { "key": "wins", "label": "wins / 805", "align": "right" },
    { "key": "len", "label": "avg len", "align": "right" }
  ],
  "rows": [
    { "model": "mistral-orpo-beta (7B)", "align": "ORPO, one stage", "wr": "12.57", "se": "0.99", "lc": "14.72", "wins": "95", "len": "1636" },
    { "model": "zephyr-7b-beta (7B)", "align": "SFT → DPO", "wr": "10.99", "se": "0.96", "lc": "13.20", "wins": "78", "len": "1444" },
    { "model": "zephyr-7b-alpha (7B)", "align": "SFT → DPO", "wr": "8.35", "se": "0.87", "lc": "10.29", "wins": "59", "len": "1302" },
    { "model": "Mistral-7B-Instruct-v0.2", "align": "Mistral's own post-training", "wr": "14.72", "se": "1.08", "lc": "17.11", "wins": "113", "len": "1676" }
  ]
}