{
  "claim": "The project's headline 'precision 0.94, recall 0.98' rests on 90 true positives, and 62 of them — 69% — come from the two queries that match most of the corpus. The eight ordinary queries produce 28 true positives between them, and one of the ten cases scores zero on both metrics.",
  "method": "Transcribed from tests/report.md, which the committed runner tests/judge.mts generates. The judge column is the number of lines Claude Sonnet marked as matching, per meaning, with the boolean expression evaluated afterwards by the runner rather than by the judge; the semgrep column is the number of lines the tool returned. TP / FP / FN are the differences, and they reconcile to the report's own totals of TP 90 / FP 7 / FN 0. The per-case table is reported at the sweep's best thresholds (-t 0.45 -T 0.65), not at the defaults; at the defaults the same 10 cases give TP 88 / FP 6 / FN 2, which is the 0.94 / 0.98 the README quotes.",
  "source": "github.com/uehaj/jev-semgrep tests/report.md, tests/cases.json, tests/verdicts.json and tests/judge.mts at v0.2.1",
  "captured": "2026-09-19",
  "note": "The denominators are the point. Ten queries over one 51-line file is 510 line-level decisions, of which ~414 are true negatives; a query like 'not a timestamped server log line' matches 39 of 51 lines by construction. The sweep also fits 361 threshold pairs on exactly the cases it reports, so the 'best' row is tuned on its own test set — the default row is the honest one, and it is the one the README quotes.",
  "columns": [
    { "key": "case", "label": "case", "align": "left" },
    { "key": "judge", "label": "judge", "align": "right", "mono": true },
    { "key": "tool", "label": "semgrep", "align": "right", "mono": true },
    { "key": "tp", "label": "TP", "align": "right", "mono": true },
    { "key": "fp", "label": "FP", "align": "right", "mono": true },
    { "key": "p", "label": "P", "align": "right", "mono": true },
    { "key": "r", "label": "R", "align": "right", "mono": true }
  ],
  "rows": [
    { "case": "not/only-non-log", "judge": "39", "tool": "39", "tp": "39", "fp": "0", "p": "1.00", "r": "1.00" },
    { "case": "ornot/code-or-not-english", "judge": "23", "tool": "25", "tp": "23", "fp": "2", "p": "0.92", "r": "1.00" },
    { "case": "single/ja-meaning", "judge": "6", "tool": "8", "tp": "6", "fp": "2", "p": "0.75", "r": "1.00" },
    { "case": "single/angry-customer", "judge": "5", "tool": "5", "tp": "5", "fp": "0", "p": "1.00", "r": "1.00" },
    { "case": "or/refund-or-address", "judge": "4", "tool": "4", "tp": "4", "fp": "0", "p": "1.00", "r": "1.00" },
    { "case": "andnot/error-not-network", "judge": "4", "tool": "5", "tp": "4", "fp": "1", "p": "0.80", "r": "1.00" },
    { "case": "single/abstract", "judge": "3", "tool": "4", "tp": "3", "fp": "1", "p": "0.75", "r": "1.00" },
    { "case": "mixed/(finance and negative) or weather", "judge": "3", "tool": "3", "tp": "3", "fp": "0", "p": "1.00", "r": "1.00" },
    { "case": "single/security-risk", "judge": "3", "tool": "3", "tp": "3", "fp": "0", "p": "1.00", "r": "1.00" },
    { "case": "and/net-and-retry", "judge": "0", "tool": "1", "tp": "0", "fp": "1", "p": "0.00", "r": "0.00" },
    { "case": "— total, best thresholds (-t 0.45 -T 0.65)", "judge": "90", "tool": "97", "tp": "90", "fp": "7", "p": "0.93", "r": "1.00" },
    { "case": "— total, shipped defaults (-t 0.5 -T 0.5)", "judge": "90", "tool": "94", "tp": "88", "fp": "6", "p": "0.94", "r": "0.98" },
    { "case": "— the two broad queries alone", "judge": "62", "tool": "64", "tp": "62", "fp": "2", "p": "0.97", "r": "1.00" },
    { "case": "— the other eight queries", "judge": "28", "tool": "33", "tp": "28", "fp": "5", "p": "0.85", "r": "1.00" }
  ]
}