{
  "claim": "Zero-shot TabFM beats tuned-and-ensembled CatBoost on 90.5% of TabArena folds. It loses to LimiX-2 on 67.5% of them. Same weights, same 51 datasets, same day — the number you quote is a choice of opponent.",
  "method": "Head-to-head rates are TabArena's own winrate_matrix.csv for all 51 tasks, every entrant, imputation off. Elo, mean rank and the two costs come from the matching website_leaderboard.csv. Costs are median seconds per 1,000 samples; 'fit' is context encoding for a foundation model and hyperparameter search plus training for a tuned baseline.",
  "source": "https://huggingface.co/spaces/TabArena/leaderboard",
  "captured": "2026-09-19",
  "note": "\"Tuned + ensembled\" on TabArena means 200 randomly sampled hyperparameter configurations per model, chosen on an inner cross-validation and then post-hoc ensembled, with the search spaces taken from the original papers or developed with the models' authors. These are not straw-man trees. Taking the same board's per-dataset mean errors and comparing pairwise rather than fold-by-fold, TabFM beats CatBoost (T+E) on 48 of the 51 datasets; it loses on hiva_agnostic, Amazon_employee_access and Diabetes130US. What the cost columns show is that the comparison is not like-for-like in the other direction either: CatBoost's 1,346 s/1K is search, paid once; TabFM's 16.1 s/1K is a forward pass over the context, paid on every call, and its 6.65 s/1K prediction is 19x CatBoost's.",
  "columns": [
    { "key": "opponent", "label": "opponent" },
    { "key": "kind", "label": "class" },
    { "key": "elo", "label": "Elo", "align": "right" },
    { "key": "wr", "label": "TabFM wins", "align": "right" },
    { "key": "fit", "label": "fit s/1K", "align": "right" },
    { "key": "pred", "label": "predict s/1K", "align": "right" }
  ],
  "rows": [
    { "opponent": "LimiX-2 (default)", "kind": "foundation model", "elo": "1973", "wr": "32.5%", "fit": "30.94", "pred": "9.028" },
    { "opponent": "TabPFN-3.5 (default)", "kind": "foundation model", "elo": "1890", "wr": "48.5%", "fit": "1.84", "pred": "0.473" },
    { "opponent": "TabFM-Ensemble (TabFM+)", "kind": "system", "elo": "1847", "wr": "40.3%", "fit": "20.04", "pred": "7.629" },
    { "opponent": "AutoGluon 1.6 (noncommercial, 4h)", "kind": "system", "elo": "1810", "wr": "57.7%", "fit": "14.48", "pred": "0.818" },
    { "opponent": "TabFM (default) — the subject", "kind": "foundation model", "elo": "1803", "wr": "—", "fit": "16.12", "pred": "6.653" },
    { "opponent": "AutoGluon 1.5 (extreme, 4h)", "kind": "system", "elo": "1673", "wr": "73.7%", "fit": "289.07", "pred": "4.031" },
    { "opponent": "TabPFN-3 (default)", "kind": "foundation model", "elo": "1649", "wr": "78.1%", "fit": "1.29", "pred": "0.382" },
    { "opponent": "TabPFN-2.6 (default)", "kind": "foundation model", "elo": "1599", "wr": "82.9%", "fit": "5.48", "pred": "0.555" },
    { "opponent": "TabICLv2 (default)", "kind": "foundation model", "elo": "1583", "wr": "84.8%", "fit": "0.75", "pred": "0.139" },
    { "opponent": "TabM (tuned + ensembled)", "kind": "neural net", "elo": "1442", "wr": "91.2%", "fit": "2450.13", "pred": "2.247" },
    { "opponent": "LightGBM (tuned + ensembled)", "kind": "tree", "elo": "1426", "wr": "90.7%", "fit": "417.05", "pred": "2.639" },
    { "opponent": "CatBoost (tuned + ensembled)", "kind": "tree", "elo": "1416", "wr": "90.5%", "fit": "1346.21", "pred": "0.344" },
    { "opponent": "XGBoost (tuned + ensembled)", "kind": "tree", "elo": "1369", "wr": "92.4%", "fit": "693.49", "pred": "1.689" },
    { "opponent": "CatBoost (default)", "kind": "tree", "elo": "1376", "wr": "not published", "fit": "5.88", "pred": "0.025" },
    { "opponent": "RandomForest (tuned + ensembled)", "kind": "tree", "elo": "1168", "wr": "96.0%", "fit": "373.24", "pred": "0.771" }
  ]
}