{
 "meta": {
  "id": "corpus",
  "title": "The corpus: grades, counts, baselines and provenance",
  "sources": [
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/artifacts/cohort-rank.json",
    "sha256": "ce33902eef5166a0b67e8c9efbe611cf0a0cfe9b3f54ccbe469914d90208d433",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/artifacts/s3-scores-in-scope.json",
    "sha256": "2882c283f010b344b227569936bf206abdab3373a68d60930777dfcfeb01063c",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/site-build/pinned/roster.json",
    "sha256": "64d140cb6c44441d6f2b0fdc5ce7ee08f14fa949bf8f49349a696fe804db279e",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/scripts/benchmark_inventory_system_one_sources.py",
    "sha256": "b1c66462dfd8a6cbb8a3332e5e548774e10e4ba426facdfb43cf5fbacf194995",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5",
    "bytes": 8768
   }
  ],
  "extracted_at": "2026-10-16T14:45:21+00:00",
  "notes": "s2 is the corpus every chart on the Space is over; s3 is the second corpus, described only. Grade E exists in the function and in neither corpus."
 },
 "data": {
  "s2": {
   "cases": 4277,
   "scorable": 3817,
   "positives": 436,
   "grade_counts": {
    "A": 17,
    "B": 419,
    "C": 460,
    "D": 3381
   },
   "grade_A_share_of_positives": 0.0389908256880734,
   "negatives": 3381,
   "grade_C_excluded": 460,
   "prevalence": 0.11422583180508253,
   "all_allow_accuracy": 0.8857741681949175,
   "block_everything": {
    "block_fpr": 1.0,
    "f1": 0.20503174229955326,
    "fn": 0,
    "fp": 3381,
    "note": "any F1 at or below this carries no discrimination information",
    "precision": 0.11422583180508253,
    "recall": 1.0,
    "tn": 0,
    "tp": 436
   },
   "chance_band": {
    "lower": 0.471205496131191,
    "upper": 0.528794503868809,
    "se": 0.014691343359335349
   },
   "prediction_rows_per_model": 30310,
   "models_scored": 22,
   "cases_sha256": "39f2c1df2369952a0525cc4c5575f4bdb590fb3ca8c1bc6805cf4f376c1adbf7"
  },
  "s3": {
   "cases": 24476,
   "scorable": 24476,
   "positives": 221,
   "grade_counts": {
    "A": 193,
    "B": 28,
    "D": 24255
   },
   "grade_A_share_of_positives": 0.8733031674208145,
   "negatives": 24255,
   "grade_C_excluded": 0,
   "prevalence": 0.009029253145938878,
   "all_allow_accuracy": 0.9909707468540612,
   "block_everything": {
    "block_fpr": 1.0,
    "f1": 0.017896910555937968,
    "fn": 0,
    "fp": 24255,
    "note": "this is the floor AT s3's OWN prevalence of 0.903%.  Any F1 at or below 0.017896910555937968 on s3 carries no discrimination information.  It is 11.5x lower than the s2 floor of 0.20503174229955326 purely because prevalence is 12.6x lower, so an s3 F1 must never be compared against an s2 F1 directly.",
    "precision": 0.009029253145938878,
    "recall": 1.0,
    "tn": 0,
    "tp": 221
   },
   "chance_band": {
    "lower": 0.46176687597792665,
    "upper": 0.5382331240220734,
    "se": 0.01950705437633106
   },
   "prediction_rows_per_model": 100001,
   "models_scored": 6,
   "cases_sha256": "0ccbc08fb408ffefc89e051c585cfe22433b1ed4c9714f60cc0f7e242969fa03"
  },
  "case_id_overlap": 0,
  "prevalence_ratio_s2_over_s3": 12.650640087154752,
  "s2_chart": [
   {
    "grade": "A",
    "label": "A, confirmed destructive",
    "cases": 17,
    "role": "scorable positive"
   },
   {
    "grade": "B",
    "label": "B, destructive",
    "cases": 419,
    "role": "scorable positive"
   },
   {
    "grade": "D",
    "label": "D, benign",
    "cases": 3381,
    "role": "scorable negative"
   },
   {
    "grade": "C",
    "label": "C, excluded from scoring",
    "cases": 460,
    "role": "excluded"
   }
  ],
  "grade_scheme": [
   {
    "grade": "A",
    "condition": "a closed deterministic proof, expected disposition detect-or-block, and label confidence high",
    "records": "confirmed destructive, highest severity; scorable positive",
    "scored_as": "positive",
    "s2_cases": 17,
    "s3_cases": 193
   },
   {
    "grade": "B",
    "condition": "source truth malicious or sensitive, a stateful surface, expected disposition detect-or-block, and label confidence high",
    "records": "destructive; scorable positive",
    "scored_as": "positive",
    "s2_cases": 419,
    "s3_cases": 28
   },
   {
    "grade": "C",
    "condition": "source truth malicious or sensitive, or a deterministic verdict of contextual-or-dual-use",
    "records": "diagnostic only; excluded from scoring because the grade records an unresolved or partial adjudication, so a scored decision on it would not have a settled truth value",
    "scored_as": "excluded",
    "s2_cases": 460,
    "s3_cases": 0
   },
   {
    "grade": "D",
    "condition": "expected disposition allow, and the source calls the case benign",
    "records": "benign; scorable negative",
    "scored_as": "negative",
    "s2_cases": 3381,
    "s3_cases": 24255
   },
   {
    "grade": "E",
    "condition": "out of scope, or unknown source truth with no deterministic verdict",
    "records": "no case of this grade is in either corpus",
    "scored_as": "not present in either corpus",
    "s2_cases": 0,
    "s3_cases": 0
   }
  ],
  "label_scheme": {
   "how": "One deterministic function, evaluated at scoring time from fields already present on each case row: applicability, source truth, deterministic verdict, expected disposition and label confidence. No model and no human reviewer is consulted, and no grade is stored on the row.",
   "limits": "No human adjudication was used on these grades, so source-label error and error correlated with the labelling source are limitations. No inter-rater agreement figure exists, because one deterministic function assigns every grade.",
   "importers": [
    "benchmarks/slm_toolcall/harness/analyse.py",
    "benchmarks/slm_toolcall/scoring/final.py"
   ]
  },
  "label_function": {
   "path": "benchmarks/scripts/benchmark_inventory_system_one_sources.py",
   "sha256": "b1c66462dfd8a6cbb8a3332e5e548774e10e4ba426facdfb43cf5fbacf194995",
   "bytes": 8768,
   "function": "truth_grade",
   "defines": true,
   "grades_returned": [
    "A",
    "B",
    "C",
    "D",
    "E"
   ]
  },
  "redistribution": {
   "corpora": "Evaluation-only. The case rows are not published anywhere: they stay in a private data repository and only aggregates over them leave the host. Training, synthetic seeding, teacher context, distillation and public redistribution are outside the disposition the corpora were collected under.",
   "published_here": "Aggregates over the corpus or a stratum of it.",
   "restricted_sources": "Two source datasets carry a restricted redistribution marker. One is local-evaluation-only: aggregate numbers only, never rows. The other is aggregate-only on the same terms. A census over the source.dataset field of both corpora counted 0 rows from either, so neither contributes a case to any figure here.",
   "local_evaluation_only_source": "mcptox",
   "source_lock": "benchmarks/datasets.lock.json",
   "source_lock_caveat": "Copies of that lock differ across revisions of this repository, so it is cited by digest and the tallies in this table are computed from the copy this build read.",
   "census_record": "benchmarks/archive/records/archive-2026-09-24-dev-host.json"
  },
  "cell": {
   "context": "C7",
   "instruction": "I3",
   "question": "Q2",
   "instruction_format": "structured"
  }
 }
}