{
  "claim": "DIFF costs 5 to 10 percent of throughput at matched parameters on H100, measured by the authors with the same customised FlashAttention-2 kernel for both models. The paper's printed percentages are the Transformer's lead divided by DIFF's throughput, which makes the cost look slightly larger than the drop from the Transformer's number.",
  "method": "Tokens per second copied from Table 7 of arXiv 2410.05258v2 (H100-80GB; 3B models have 24 heads for the Transformer and 12 for DIFF, 13B have 40 and 20; head dimension 128). My arithmetic: 'drop' is DIFF / Transformer - 1; 'as printed' is the paper's column, which matches 1 - Transformer / DIFF to rounding.",
  "source": "https://arxiv.org/abs/2410.05258",
  "captured": "2026-09-26",
  "note": "Forward + backward is training throughput; forward only is the paper's prefill number. Decoding is not measured. A kernel without separate Q/K and V head widths (stock flash-attention) needs the four-call FlashDiffAttn_2 path and does twice the attention-core arithmetic; the customised kernel used here needs two calls.",
  "columns": [
    { "key": "cfg", "label": "model, length", "align": "left" },
    { "key": "pass", "label": "pass", "align": "left" },
    { "key": "t", "label": "Transformer tok/s", "align": "right" },
    { "key": "d", "label": "DIFF tok/s", "align": "right" },
    { "key": "printed", "label": "as printed", "align": "right" },
    { "key": "drop", "label": "drop", "align": "right" }
  ],
  "rows": [
    { "cfg": "3B, 2K", "pass": "fwd + bwd", "t": 7247, "d": 6635, "printed": "−9%", "drop": "−8.4%" },
    { "cfg": "3B, 2K", "pass": "fwd", "t": 51228, "d": 46811, "printed": "−9%", "drop": "−8.6%" },
    { "cfg": "3B, 4K", "pass": "fwd + bwd", "t": 7491, "d": 6718, "printed": "−12%", "drop": "−10.3%" },
    { "cfg": "3B, 4K", "pass": "fwd", "t": 48762, "d": 44521, "printed": "−10%", "drop": "−8.7%" },
    { "cfg": "13B, 2K", "pass": "fwd + bwd", "t": 998, "d": 942, "printed": "−6%", "drop": "−5.6%" },
    { "cfg": "13B, 2K", "pass": "fwd", "t": 14346, "d": 13653, "printed": "−5%", "drop": "−4.8%" }
  ]
}