{
  "claim": "The 44-Elo gap between TabFM and TabFM-Ensemble is worth a median of half a percent of error on the 30 binary tasks, and concentrates almost entirely in imbalanced and very wide tables. Computed from Google's own committed per-fold results.",
  "method": "Read the four parquet files under results/ in google-research/tabfm — 816 (dataset, fold) rows per configuration, JAX on TPU — merged them on (dataset, fold, metric, problem_type) and compared metric_error pairwise. The aggregate rows report the median across datasets of the per-dataset mean relative error change; the single-dataset rows report that dataset's own mean relative change. Nothing here is re-run: these are the numbers Google shipped beside the weights.",
  "source": "https://github.com/google-research/tabfm/tree/main/results",
  "captured": "2026-09-19",
  "note": "TabArena scores binary tasks with ROC AUC. Platt scaling, which the ensemble preset adds for binary problems, is a monotone map and so cannot change ROC AUC at all — on 30 of the 51 datasets the calibration half of the preset is, by construction, invisible to the score. What is left is the feature crosses, the SVD features and the non-negative-least-squares blend, and their median return is half a percent. TabArena runs 10 repeats of 3-fold CV on datasets up to 2,400 rows and 3 repeats above 2,584, which is why the fold counts differ.",
  "columns": [
    { "key": "slice", "label": "slice" },
    { "key": "n", "label": "datasets", "align": "right" },
    { "key": "folds", "label": "folds", "align": "right" },
    { "key": "metric", "label": "metric" },
    { "key": "foldwin", "label": "ensemble wins folds", "align": "right" },
    { "key": "dswin", "label": "wins datasets", "align": "right" },
    { "key": "cut", "label": "error cut", "align": "right" }
  ],
  "rows": [
    { "slice": "binary", "n": "30", "folds": "438", "metric": "ROC AUC", "foldwin": "59.4%", "dswin": "21 / 30", "cut": "+0.50%" },
    { "slice": "multiclass", "n": "8", "folds": "156", "metric": "log loss", "foldwin": "75.0%", "dswin": "8 / 8", "cut": "+1.59%" },
    { "slice": "regression", "n": "13", "folds": "222", "metric": "RMSE", "foldwin": "76.6%", "dswin": "10 / 13", "cut": "+0.35%" },
    { "slice": "all tasks", "n": "51", "folds": "816", "metric": "mixed", "foldwin": "67.0%", "dswin": "39 / 51", "cut": "—" },
    { "slice": "anneal — imbalance 85:1, 898 rows", "n": "1", "folds": "30", "metric": "log loss", "foldwin": "—", "dswin": "—", "cut": "+25.2%" },
    { "slice": "APSFailure — 54:1, 170 features", "n": "1", "folds": "9", "metric": "ROC AUC", "foldwin": "—", "dswin": "—", "cut": "+25.1%" },
    { "slice": "polish_companies_bankruptcy — 13:1", "n": "1", "folds": "9", "metric": "ROC AUC", "foldwin": "—", "dswin": "—", "cut": "+17.3%" },
    { "slice": "hiva_agnostic — 88:1, 1,617 features", "n": "1", "folds": "9", "metric": "log loss", "foldwin": "—", "dswin": "—", "cut": "+14.5%" },
    { "slice": "churn — the worst single loss", "n": "1", "folds": "9", "metric": "ROC AUC", "foldwin": "—", "dswin": "—", "cut": "-2.23%" }
  ]
}