{
  "claim": "BrowserSkill's eval README says \"the direct smoke lane covers 25 of 28 operations.\" Running its own harness reproduces that exactly. The number the README does not give is the denominator behind the denominator: those 28 operations map onto 28 of the 41 tool.* methods the protocol declares, so 13 browser methods have no case in the corpus at all — including all seven that 0.3.0 added, none of which is present in the protocol at tag cli-v0.2.1.",
  "method": "node evals/browser/cli.mjs validate | coverage | list, and node --test evals/browser/tests/*.test.mjs, run at commit d1356fd with no local changes. The protocol count is the number of #[serde(rename = \"tool.…\")] variants in crates/bsk-protocol/src/method.rs; the dispatcher count is the number of distinct case \"tool.…\" labels in apps/extension/src/tools/dispatcher.ts.",
  "source": "https://github.com/Tencent/BrowserSkill/blob/main/evals/browser/README.md",
  "captured": "2026-09-18",
  "note": "The 13 unevaluated methods are wheel, scroll_to, focus, blur, upload, download, screenshot_full_page, screenshot_read, screenshot_release, evaluate, record_start, record_stop and record_await. tool.wait_ms is the one protocol method the extension never sees — the daemon answers it (crates/bsk-cli/src/daemon/ipc.rs:301), so the sleep never reaches Chrome.",
  "columns": [
    { "key": "measure", "label": "measure", "align": "left" },
    { "key": "value", "label": "count", "align": "right" },
    { "key": "how", "label": "how it was produced", "align": "left" }
  ],
  "rows": [
    { "measure": "cases in the corpus", "value": 9, "how": "cli.mjs validate — core 6, matrix 1, regression 2" },
    { "measure": "fixture routes", "value": 18, "how": "cli.mjs validate" },
    { "measure": "operations in the inventory", "value": 28, "how": "OPERATION_CATALOG in evals/browser/lib/operations.mjs" },
    { "measure": "operations the direct smoke lane reaches", "value": 25, "how": "cli.mjs coverage — direct_smoke = yes" },
    { "measure": "operations left to the manual lane", "value": 3, "how": "tabs.borrow, tabs.return, assist.request-help" },
    { "measure": "operations any agent prompt declares", "value": 21, "how": "cli.mjs coverage — rows with a non-empty agent_cases column" },
    { "measure": "tool.* methods the protocol declares", "value": 41, "how": "crates/bsk-protocol/src/method.rs" },
    { "measure": "tool.* methods the extension dispatches", "value": 40, "how": "apps/extension/src/tools/dispatcher.ts" },
    { "measure": "tool.* methods with no eval operation", "value": 13, "how": "the 41 protocol methods minus the 28 the catalog names" },
    { "measure": "harness unit tests, all passing", "value": 15, "how": "node --test evals/browser/tests/*.test.mjs" }
  ]
}