{
 "meta": {
  "id": "reproduce",
  "title": "Build gates, scripts, provenance and changelog",
  "sources": [
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/site-build/pinned/scorer-provenance.json",
    "sha256": "e4e6e9b14f61627101ebe3af9a03ab6f57a1783445ffb269e97dd4403f4a7665",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/artifacts/cohort-scores.json",
    "sha256": "775c0e8fcfd24c601e03c2e03f4299381f410d82c76b7da2f943b4a4f66f852c",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/artifacts/cohort-length-controlled-ranking.json",
    "sha256": "6ddf42c8b3aafd42122fee3e3e8e81d6c473f233cad57558e0aa1d82546a2015",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "hf-space:Vineethsain/defenseclaw-slm-toolcall",
    "path": "_build-figures.json",
    "sha256": "ac634d586f16d12dc325ec6672317870fd689608d410de0ba77dd87764c90da8",
    "commit": "3c4ba84da3f1884a8e9119cb21e5402d7421258a"
   },
   {
    "repo": "hf-space:Vineethsain/defenseclaw-slm-toolcall",
    "path": "methodology.html",
    "sha256": "4ab65335294e3da1f319d08434f7c02c0e807cc2fab0397f03f750fd0f3c93f3",
    "commit": "3c4ba84da3f1884a8e9119cb21e5402d7421258a"
   }
  ],
  "extracted_at": "2026-10-16T14:45:21+00:00",
  "notes": "assertions_checked, the artifact list and the chart ids are the previous Space's own build record; every artifact digest it recorded is re-hashed here (artifacts_read[].matches)."
 },
 "data": {
  "build_gates": [
   "site-build/build.py recomputes 2,251 figures from their primitives and checks each against its artifact before any page is written, over 20 input files and 22 charts. It aborts on any mismatch.",
   "Each ROC curve is carried as integer (false blocks, true blocks) pairs, 6,515 points across 22 models. The area under each drawn curve must equal the published raw AUC exactly, on all 28 AUCs, and the budget point must lie on the curve.",
   "site-build/guard.py , byte-identical to the System One Space's copy, fails on any corpus case id, sensitive-text shingle, CJK codepoint or credential shape, and on any file that is not .html .css .md .json .txt or is over 1 MiB.",
   "site-build/verify.py checks structure, links, SVG bounds, table width, the reading precision of every figure, the required disclosures and the list of retired figures.",
   "site-build/publish.py re-runs both, records the repository's visibility, uploads, and aborts if the visibility changed. It never moves a repository between public and private."
  ],
  "gate_figures": {
   "assertions_checked": 2251,
   "artifacts_read": 20,
   "charts": 22,
   "chart_ids": [
    "accuracy_delta",
    "cap_fpr",
    "control_dist",
    "corpus",
    "cost",
    "cues",
    "envelope",
    "gating",
    "grade_slope",
    "grades",
    "jaccard",
    "prf",
    "quintiles",
    "ranking",
    "reliability",
    "roc_zoom",
    "size_scatter",
    "srcheat",
    "throughput",
    "trunc",
    "union",
    "zerofp"
   ],
   "roc_points": 6515,
   "aucs_reconciled": 28,
   "models": 22
  },
  "scripts": [
   {
    "path": "benchmarks/EXPERIMENTS.md",
    "role": "the experiment ledger"
   },
   {
    "path": "benchmarks/scripts/benchmark_prepare_system_one.py",
    "role": "corpus assembly"
   },
   {
    "path": "benchmarks/scripts/benchmark_run_system_one.py",
    "role": "the reference driver (build_state, build_questions)"
   },
   {
    "path": "benchmarks/scripts/benchmark_score_system_one.py",
    "role": "the reference scorer"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/arms.py",
    "role": "the model registry"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/score_arm.py",
    "role": "the runner"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/settle.py",
    "role": "retrofits complete and prediction_sha256, re-verifies"
   },
   {
    "path": "benchmarks/slm_toolcall/scoring/score_cohort.py",
    "role": "scorecards behind the parity check"
   },
   {
    "path": "benchmarks/slm_toolcall/scoring/leakage.py",
    "role": "the counting-variable diagnostic"
   },
   {
    "path": "benchmarks/slm_toolcall/scoring/final.py",
    "role": "the baselines and chance band"
   },
   {
    "path": "benchmarks/slm_toolcall/ranking/rank_cohort.py",
    "role": "the length-controlled ranking"
   },
   {
    "path": "benchmarks/slm_toolcall/s3/score_s3.py",
    "role": "the second-corpus scoring"
   },
   {
    "path": "benchmarks/slm_toolcall/s3/stats_s3.py",
    "role": "DeLong intervals on the second corpus"
   },
   {
    "path": "benchmarks/system_one/reproduce/07-analysis/rescoring/remine.py",
    "role": "the shared arithmetic and the two AUC definitions"
   },
   {
    "path": "benchmarks/slm_toolcall/site-build/build.py",
    "role": "the previous Space's build"
   },
   {
    "path": "benchmarks/slm_toolcall/site-build/verify.py",
    "role": "the previous Space's checks"
   },
   {
    "path": "benchmarks/slm_toolcall/site-build/guard.py",
    "role": "the leak guard"
   },
   {
    "path": "benchmarks/slm_toolcall/site-build/publish.py",
    "role": "the upload step"
   },
   {
    "path": "tools/extract_slm.py",
    "role": "this extraction (monorepo); reads the files above and writes data/"
   }
  ],
  "artifacts_read": [
   {
    "path": "benchmarks/archive/records/archive-2026-09-24-dev-host.json",
    "sha256_published": "ccec457183e89e2a36a6109a669a52d9e1e91f04c65c20be5167afbac362b0bd",
    "sha256_fetched": "ccec457183e89e2a36a6109a669a52d9e1e91f04c65c20be5167afbac362b0bd",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/archive/records/archive-2026-09-24-dev-host.json"
   },
   {
    "path": "benchmarks/datasets.lock.json",
    "sha256_published": "48a6639433402efcb7069439cb90941d953df89bf4a1a342bdd215ba61298679",
    "sha256_fetched": "48a6639433402efcb7069439cb90941d953df89bf4a1a342bdd215ba61298679",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/datasets.lock.json"
   },
   {
    "path": "benchmarks/scripts/benchmark_inventory_system_one_sources.py",
    "sha256_published": "b1c66462dfd8a6cbb8a3332e5e548774e10e4ba426facdfb43cf5fbacf194995",
    "sha256_fetched": "b1c66462dfd8a6cbb8a3332e5e548774e10e4ba426facdfb43cf5fbacf194995",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/scripts/benchmark_inventory_system_one_sources.py"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/cohort-curves.json",
    "sha256_published": "0142a9c1a36e53694b642015428ef8ca3e9d93c030ee5393b16a68394f7dd17c",
    "sha256_fetched": "0142a9c1a36e53694b642015428ef8ca3e9d93c030ee5393b16a68394f7dd17c",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/cohort-curves.json"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/cohort-length-controlled-ranking.json",
    "sha256_published": "6ddf42c8b3aafd42122fee3e3e8e81d6c473f233cad57558e0aa1d82546a2015",
    "sha256_fetched": "6ddf42c8b3aafd42122fee3e3e8e81d6c473f233cad57558e0aa1d82546a2015",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/cohort-length-controlled-ranking.json"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/cohort-rank.json",
    "sha256_published": "ce33902eef5166a0b67e8c9efbe611cf0a0cfe9b3f54ccbe469914d90208d433",
    "sha256_fetched": "ce33902eef5166a0b67e8c9efbe611cf0a0cfe9b3f54ccbe469914d90208d433",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/cohort-rank.json"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/cohort-scores.json",
    "sha256_published": "775c0e8fcfd24c601e03c2e03f4299381f410d82c76b7da2f943b4a4f66f852c",
    "sha256_fetched": "775c0e8fcfd24c601e03c2e03f4299381f410d82c76b7da2f943b4a4f66f852c",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/cohort-scores.json"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/final-comparisons.json",
    "sha256_published": "8a9731eb740ccfa3425d448d64fdb32afb0ad93738f70c63e52c98e3387a2d71",
    "sha256_fetched": "8a9731eb740ccfa3425d448d64fdb32afb0ad93738f70c63e52c98e3387a2d71",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/final-comparisons.json"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/leakage-diagnostic.json",
    "sha256_published": "5f1fbe70117d1a0b6e9d3e47aaa68453ab4eb3a9af59cce02a2a6127d20a8d2d",
    "sha256_fetched": "5f1fbe70117d1a0b6e9d3e47aaa68453ab4eb3a9af59cce02a2a6127d20a8d2d",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/leakage-diagnostic.json"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/s3-scores-in-scope.json",
    "sha256_published": "2882c283f010b344b227569936bf206abdab3373a68d60930777dfcfeb01063c",
    "sha256_fetched": "2882c283f010b344b227569936bf206abdab3373a68d60930777dfcfeb01063c",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/s3-scores-in-scope.json"
   },
   {
    "path": "benchmarks/slm_toolcall/artifacts/s3-stats.json",
    "sha256_published": "77d76ba3f1aafe0510f1948233c4756e1536ed9f71abc33b9bb6bae09889442a",
    "sha256_fetched": "77d76ba3f1aafe0510f1948233c4756e1536ed9f71abc33b9bb6bae09889442a",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/artifacts/s3-stats.json"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/arms.py",
    "sha256_published": "de32f8da40c61e6781015d0be496a4c1768436aad0f5b776d7772caf65b09bf9",
    "sha256_fetched": "de32f8da40c61e6781015d0be496a4c1768436aad0f5b776d7772caf65b09bf9",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/harness/arms.py"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/weights_manifest.json",
    "sha256_published": "875d692e322b8714c21b01a9373c2b30190e9ac4a9d87bf0ee5b65e2037c92e9",
    "sha256_fetched": "875d692e322b8714c21b01a9373c2b30190e9ac4a9d87bf0ee5b65e2037c92e9",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/harness/weights_manifest.json"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/weights_manifest2.json",
    "sha256_published": "5f620dc168e1ec151fc13da1a57dd406493a095fbbe8793657cabdb72d551d38",
    "sha256_fetched": "5f620dc168e1ec151fc13da1a57dd406493a095fbbe8793657cabdb72d551d38",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/harness/weights_manifest2.json"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/weights_manifest3.json",
    "sha256_published": "027753b1ef346c1949643d4542809756d9c687cd7a36fafbc3d8897a44dba0fe",
    "sha256_fetched": "027753b1ef346c1949643d4542809756d9c687cd7a36fafbc3d8897a44dba0fe",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/harness/weights_manifest3.json"
   },
   {
    "path": "benchmarks/slm_toolcall/harness/weights_manifest4.json",
    "sha256_published": "667f65d3bfb4454ed1bd3942a74fc7820b0ea314e6c26bfa68cec68397b6e2b8",
    "sha256_fetched": "667f65d3bfb4454ed1bd3942a74fc7820b0ea314e6c26bfa68cec68397b6e2b8",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/harness/weights_manifest4.json"
   },
   {
    "path": "benchmarks/slm_toolcall/site-build/pinned/laptop-feasibility.json",
    "sha256_published": "bcff5768ef5fa1839e83cb2446a0ed5982049754f60a1c284b49ad82d42b0abc",
    "sha256_fetched": "bcff5768ef5fa1839e83cb2446a0ed5982049754f60a1c284b49ad82d42b0abc",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/site-build/pinned/laptop-feasibility.json"
   },
   {
    "path": "benchmarks/slm_toolcall/site-build/pinned/roster.json",
    "sha256_published": "64d140cb6c44441d6f2b0fdc5ce7ee08f14fa949bf8f49349a696fe804db279e",
    "sha256_fetched": "64d140cb6c44441d6f2b0fdc5ce7ee08f14fa949bf8f49349a696fe804db279e",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/site-build/pinned/roster.json"
   },
   {
    "path": "benchmarks/slm_toolcall/site-build/pinned/scorer-provenance.json",
    "sha256_published": "e4e6e9b14f61627101ebe3af9a03ab6f57a1783445ffb269e97dd4403f4a7665",
    "sha256_fetched": "e4e6e9b14f61627101ebe3af9a03ab6f57a1783445ffb269e97dd4403f4a7665",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/slm_toolcall/site-build/pinned/scorer-provenance.json"
   },
   {
    "path": "benchmarks/system_one/reproduce/07-analysis/rescoring/remine.py",
    "sha256_published": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
    "sha256_fetched": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
    "matches": true,
    "url": "https://github.com/cisco-ai-defense/defenseclaw/blob/c7e264e63e6a1609a69992ab768ce72c488950c5/benchmarks/system_one/reproduce/07-analysis/rescoring/remine.py"
   }
  ],
  "shared_arithmetic": {
   "repo_path": "benchmarks/system_one/reproduce/07-analysis/rescoring/remine.py",
   "sha256": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
   "bytes": 22586,
   "imported_verbatim_by": [
    "benchmarks/slm_toolcall/scoring/score_cohort.py",
    "benchmarks/slm_toolcall/scoring/leakage.py",
    "benchmarks/slm_toolcall/scoring/final.py"
   ],
   "how_it_is_imported": "the three scoring scripts put a dev-host working directory on sys.path and import the module from there, so the path in the source is not the repository path",
   "dev_host_copy_sha256": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
   "dev_host_copy_is_byte_identical": true,
   "dev_host_copy_verified_on": "2026-09-23",
   "note": "An earlier review of this build recorded the module as not vendored, on the strength of the dev-host path in the scoring scripts. That was wrong: the module is in the repository and git-tracked, and the copy the scoring run imported hashes to the same digest.",
   "fetched_sha256": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
   "fetched_matches_pinned": true
  },
  "request_construction": {
   "what": "The cohort runner imports request construction from the System One reference driver, which is why each row's context digest, context byte count and truncation flag are byte-identical to the settled board rows.",
   "reference_driver": "benchmarks/scripts/benchmark_run_system_one.py",
   "functions": [
    "build_state",
    "build_questions"
   ],
   "imported_by": "benchmarks/slm_toolcall/harness/score_arm.py"
  },
  "parity_gate": {
   "references": 3,
   "max_abs_delta_f1": 1.2977099395072855e-09,
   "confusion_matches": 2,
   "auc_abs_delta": 0.0,
   "raw": {
    "reference_1": {
     "abs_delta_f1": 1.2716763198028502e-09
    },
    "reference_2": {
     "abs_delta_f1": 1.2977099395072855e-09,
     "auc_abs_delta": 0.0,
     "auc_agrees_within_5e-12": true,
     "confusion_matches_published": true
    },
    "reference_3": {
     "abs_delta_f1": 0.0,
     "confusion_matches_published": true
    }
   }
  },
  "settlement": {
   "settled": 22,
   "unsettled": 0,
   "digest_mismatch": 0,
   "rows": 30310,
   "summary": {
    "all_22_settled": true,
    "arms": 22,
    "finding": "22 of 22 s2 bodies carry complete: true with a prediction_sha256 that matches the body on disk, read from the archived settled tree.  5 of the WORKING tree's .meta.json files still lack those keys, so an audit that reads the working tree would wrongly call those arms unsettled.  The bodies themselves are byte-identical across both trees, so no score in this file is affected -- but the working tree's metadata should be refreshed.",
    "settled": 22,
    "stale_arms": [
     "gemma-3-1b-it",
     "granite-guardian-3.1-2b",
     "llama-3.2-3b-instruct",
     "phi-4-mini-instruct",
     "smollm3-3b"
    ],
    "working_tree_metas_stale": 5
   },
   "per_model": {
    "control-modernbert-base": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "9d62074edd4ca545839b6f4246e4e7638804b3dc1c96cd2d940c8f1eae74a984"
    },
    "control-modernbert-large": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "46cbba9bab176aa69afcdd383025999ae112b20e201ea13d126b672ee282f33e"
    },
    "deberta-v3-prompt-injection-v2": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "927baa5ccc8bbf9b7885381ed50781264f848ab1c666e053166f9aab1721eeb4"
    },
    "falcon3-1b-instruct": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "958b7c62105ee24995c405bce1a9cf6f33d56ba7558789e12991e4651ca97474"
    },
    "falcon3-3b-instruct": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "6a3791986e18ede8492206e633386cfbba832d18e86aaefb558a7d7d920bde54"
    },
    "gemma-3-1b-it": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "f4719aab1ba8b9497c09d63850bafe07598aa5fd652cd8fadd3031939273e38d"
    },
    "gemma-3-4b-it": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "b945f36e33a4e0ea4f41b76b400b1e22901d8b0b6d7a034cf85faf587b112d5e"
    },
    "granite-4.0-1b": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "44dae504260a11544d1daf5b3bf01046454c8f020c0ccf944794fb1faf988914"
    },
    "granite-4.0-micro": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "dfa2f3f847e47d122091d1244c0ef19b7fabb47650e1f24facd10c16de1ec660"
    },
    "granite-guardian-3.1-2b": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "10abb332428f9da6ce3c200395a421ae5cecd1c2e26a776fd3eb9c970dcd780a"
    },
    "granite-guardian-3.2-3b-a800m": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "d76cfce2377f9be9a3e05926ccf93a4798f11dd312d3ccead34121c18743138e"
    },
    "llama-3.2-1b-instruct": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "236286439a7624a23e3610309aa3a31de036d66291176f19f628453557f3c13d"
    },
    "llama-3.2-3b-instruct": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "2501574f34cb99d8271c64a0d94a624762c35260f5d5dc35fd18f5498e2c97ca"
    },
    "llama-guard-3-1b": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "68795ca2ee59c16a70f6b67b752ae751dcbc4e563f0ccfe7c30cd61e556c1e36"
    },
    "olmo-2-1b-instruct": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "f7a32cfd6eb06a790c22de827891123deb9f5c455043f27cb4c437ecdbda0db1"
    },
    "phi-4-mini-instruct": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "7388e2b13551ae7d04f808f5212248f2cc444b093ec86e11d88e7d5ee0e6e51b"
    },
    "prompt-guard-2-22m": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "b446c1297e78490c40697800cbd656adf8e3587ae6e1b7c930a25501e27ed1fa"
    },
    "prompt-guard-2-86m": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "6b34ce108bb324043d5094cd32d2c282882312acc2bccbfbdcef5d460cdd25e5"
    },
    "shieldgemma-2b": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "d016410646fe40fed8e8ee9b6dc4977b63991f4c94646e3329258c84eed8177f"
    },
    "shieldstral-1.0-3b": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "d370b60df1178d4658329a286464dfdc2095da7cd9040d8952a773b5da4d1537"
    },
    "smollm2-1.7b-instruct": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "4fe2c3af3206fe2ee67727b7c89c1c2feb15047facb84bc7fbb41a1b3a093f19"
    },
    "smollm3-3b": {
     "settled": true,
     "rows": 30310,
     "errors": 0,
     "digest_matches_disk": true,
     "digest": "0a84032a29f5d510fbee50cdb33afd97e45f34c5670fbb8f337c5375db7cba1f"
    }
   }
  },
  "changelog": [
   "This revision. Eleven pages became five. The two controls' length-controlled AUCs, the DeBERTa tile and the length counter had still been printed with the retracted unweighted estimator; they now use the published one, and the leakage-gate verdict was re-read from it. The laptop figures are keyed to model names, and the overlap, grade-B and memory statements were corrected against the artifacts.",
   "Earlier. The unweighted mean of per-quintile AUCs was retracted as the length-control estimator in favour of the pair-weighted pooled figure. The claim that Prompt Guard 2 supplied an independent encoder backbone was withdrawn.",
   "Monorepo redesign (2026-10): five pages became one chaptered page plus an appendix; every chart reads data/*.json written by tools/extract_slm.py from the artifacts at commit c7e264e63e6a; parity against the previous Space's 577 published figure values is recorded in data/_parity.json."
  ],
  "publication": [
   "Aggregates only. No per-case row detail appears on this Space; every figure is a count, rate or distribution over the corpus or a stratum of it. mcptox is local-evaluation-only and one further source is aggregate-only; a census of the source.dataset field found 0 rows from either in either corpus. Licences and redistribution markers for every source are in benchmarks/datasets.lock.json , 92 entries at sha256 48a6639433402efcb7069439cb90941d953df89bf4a1a342bdd215ba61298679 .",
   "The four data repositories that hold the corpora and the prediction files stay private. This Space is public and carries no corpus rows, no provider rationales, no credentials and no unredacted identifiers. Corpus cases_sha256 39f2c1df2369952a0525cc4c5575f4bdb590fb3ca8c1bc6805cf4f376c1adbf7 ."
  ],
  "cases_sha256": {
   "s2": "39f2c1df2369952a0525cc4c5575f4bdb590fb3ca8c1bc6805cf4f376c1adbf7",
   "s3": "0ccbc08fb408ffefc89e051c585cfe22433b1ed4c9714f60cc0f7e242969fa03"
  },
  "provenance": {
   "ranking": {
    "arithmetic": "sweep/best_point/mann_whitney_auc/f1_of/wilson/truth_grade/ACTION_RANK imported from remine, not reimplemented",
    "gpu_used": false,
    "remine_path": "/home/ubuntu/rescoring-remine/remine.py",
    "remine_sha256": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
    "scope_rule": "no Jev-family or System One board model in the cohort ranking"
   },
   "curves": {
    "arithmetic": "sweep / mann_whitney_auc / at_fpr_cap / wilson / truth_grade imported from remine.py",
    "bootstrap": {
     "note": "threshold held fixed at the arm's common-budget threshold",
     "resamples": 2000,
     "seed": 20260924
    },
    "fpr_cap": 0.00384502,
    "gpu_used": false,
    "half_cap": 0.00192251,
    "published_here": "aggregates over the corpus or a stratum of it; no case id and no corpus text",
    "remine_path": "/home/ubuntu/rescoring-remine/remine.py",
    "remine_sha256": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
    "s2_bodies": "/home/ubuntu/cohort-rank/preds-s2",
    "s2_cases_sha256": "39f2c1df2369952a0525cc4c5575f4bdb590fb3ca8c1bc6805cf4f376c1adbf7",
    "s3_bodies": "/home/ubuntu/hf-stage/toolcall/predictions-s3",
    "s3_cases_sha256": "0ccbc08fb408ffefc89e051c585cfe22433b1ed4c9714f60cc0f7e242969fa03",
    "wilson_z": 1.959963984540054
   },
   "authoritative": {
    "arithmetic": "sweep / best_point / mann_whitney_auc / f1_of / wilson / at_fpr_cap / zero_fp_point / truth_grade imported from remine.py; the AUC kernel here returns the wins+0.5*ties numerator so bins pool exactly, and is the same kernel",
    "bodies_read_from": "/home/ubuntu/cohort-rank/preds-s2",
    "delong_path": "/home/ubuntu/s3-escalation-2026-09-24/delong.py",
    "delong_sha256": "d54ae280eb2241ae8cbe8dc57f9b698c171e05d2c03f1760271486fff1a32f12",
    "gpu_used": false,
    "metadata_read_from": "/home/ubuntu/archive-stage-2026-09-24/laptopguard/preds",
    "remine_path": "/home/ubuntu/rescoring-remine/remine.py",
    "remine_sha256": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
    "scope_rule": "recorded in the parent artifact; the families it names are out of scope on this Space and the string is replaced here rather than reproduced",
    "tree_note": "all 22 bodies are byte-identical between the working tree and the archived settled tree (verified by sha256), but 5 of the working tree's .meta.json files are STALE and lack the settle.py keys.  Settlement is therefore read from the archived tree, and the discrepancy is recorded per arm below."
   },
   "s3": {
    "arithmetic": "sweep / best_point / mann_whitney_auc / f1_of / wilson / at_fpr_cap / zero_fp_point / truth_grade / ACTION_RANK imported from remine.py; read_arm / projections_for / length_controlled imported from rank_cohort.py",
    "gpu_used": false,
    "rank_cohort_sha256": "79df39e26c7e631e59be46d84bc02c5e69e567d88ca2dc945c442c6b8b7c2ac1",
    "remine_sha256": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f",
    "s3_bodies": "/home/ubuntu/archive-stage-2026-09-24/laptopguard/preds-s3 (settled)",
    "scope_rule": "recorded in the parent artifact; the families it names are out of scope on this Space and the string is replaced here rather than reproduced"
   },
   "laptop": {
    "source": "benchmarks/EXPERIMENTS.md, laptop-feasibility section",
    "raw_artifacts_location": "the Lightning studio",
    "raw_artifacts_read_for_this_build": false,
    "why": "a GPU sweep and a reshard were running on the Lightning studio and its drive was at about 80% I/O stall, so nothing was run there for this build. Every value below is a primitive transcribed from the ledger; every derived figure on the site is recomputed from these primitives at build time.",
    "measurement_conditions": {
     "device": "CPU only, CUDA_VISIBLE_DEVICES empty",
     "threads": 8,
     "thread_placement": "pinned",
     "decoder_quantization": "GGUF Q4_K_M under llama.cpp",
     "encoder_quantization": "dynamic int8 under transformers",
     "latency_model": "row_latency = 1200/pp1200 + 1/tg32",
     "bench_command": "llama-bench -t 8 -p 1200 -n 32 -r 3",
     "bench_deviation": "under 4%",
     "generated_token_share_of_latency": "under 1% in every case",
     "conversions_attempted": {
      "decoders": 11,
      "encoders": 3
     },
     "conversions_succeeded": {
      "decoders": 11,
      "encoders": 3
     }
    },
    "reproducibility": "harness/build_llamacpp.sh builds the CPU benchmarking toolchain, and no script in the harness runs it or writes the CPU results. The measurements were ad-hoc invocations that were never saved to a file, so the logs on the studio survive and the measurement step does not. These figures are archived rather than reproducible from vendored code. Recorded in benchmarks/slm_toolcall/harness/README.md."
   }
  }
 }
}