{
 "meta": {
  "id": "leaderboard",
  "title": "Every model at the shared false-block budget",
  "sources": [
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/artifacts/cohort-length-controlled-ranking.json",
    "sha256": "6ddf42c8b3aafd42122fee3e3e8e81d6c473f233cad57558e0aa1d82546a2015",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/artifacts/cohort-curves.json",
    "sha256": "0142a9c1a36e53694b642015428ef8ca3e9d93c030ee5393b16a68394f7dd17c",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/artifacts/cohort-rank.json",
    "sha256": "ce33902eef5166a0b67e8c9efbe611cf0a0cfe9b3f54ccbe469914d90208d433",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/harness/arms.py",
    "sha256": "de32f8da40c61e6781015d0be496a4c1768436aad0f5b776d7772caf65b09bf9",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/site-build/pinned/roster.json",
    "sha256": "64d140cb6c44441d6f2b0fdc5ce7ee08f14fa949bf8f49349a696fe804db279e",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   },
   {
    "repo": "github:cisco-ai-defense/defenseclaw@feat/system-one-benchmarks",
    "path": "benchmarks/slm_toolcall/site-build/pinned/laptop-feasibility.json",
    "sha256": "bcff5768ef5fa1839e83cb2446a0ed5982049754f60a1c284b49ad82d42b0abc",
    "commit": "c7e264e63e6a1609a69992ab768ce72c488950c5"
   }
  ],
  "extracted_at": "2026-10-16T14:45:21+00:00",
  "notes": "Ordered by F1 at the budget, then by parameters. rank_at_budget and rank_lc_auc are over the 20 candidates only; the two MLM controls carry is_control=true and null ranks. default_decision is null for gemma-3-4b-it (its body landed after the shipped-argmax run). precision is null where tp+fp is 0. q4_k_m_bytes and peak_rss_bytes come from the private dataset's cpu/ measurements (the raw data behind the pinned laptop record) and were not shown per model on the previous Space."
 },
 "data": {
  "budget": {
   "fpr_cap": 0.00384502,
   "max_false_positives": 13,
   "half_cap": 0.00192251,
   "half_max_false_positives": 6,
   "n_pos": 436,
   "n_neg": 3381,
   "scorable": 3817,
   "cases": 4277,
   "prevalence": 0.11422583180508253,
   "all_allow_accuracy": 0.8857741681949175
  },
  "models": [
   {
    "id": "falcon3-1b-instruct",
    "repo": "tiiuae/Falcon3-1B-Instruct",
    "revision": "28ba2251970a01dd1edc7ba7dad2eb71216ccfdf",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 1669408768,
    "params_short": "1.67B",
    "size_band": "under 3B",
    "licence": "other",
    "licence_name": "Falcon LLM licence",
    "gated": null,
    "origin": "UAE (TII)",
    "note": null,
    "backbone": null,
    "snapshot_bytes": 3348994667,
    "rank_at_budget": 1,
    "rank_lc_auc": 10,
    "at_budget": {
     "tp": 25,
     "fp": 13,
     "fn": 411,
     "tn": 3368,
     "threshold": 0.6170100906975968,
     "precision": 0.6578947368421053,
     "recall": 0.05733944954128441,
     "f1": 0.10548523206751055,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.06746031746031746,
      0.14285714285714285
     ],
     "recall_ci95": [
      0.039137016728024596,
      0.08327404239553826
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.33271719038817005,
     "fpr": 0.1378290446613428,
     "precision": 0.2786377708978328,
     "recall": 0.41284403669724773,
     "tp": 180,
     "fp": 466,
     "fn": 256,
     "tn": 2915,
     "flagged_cases": 646,
     "flag_rate": 0.1692428608855122
    },
    "oracle_f1": 0.3294279807029635,
    "oracle": {
     "f1": 0.3294279807029635,
     "threshold": 0.44082308548066973,
     "tp": 239,
     "fp": 776,
     "fn": 197,
     "tn": 2605,
     "precision": 0.2354679802955665,
     "recall": 0.5481651376146789,
     "fpr": 0.2295178941141674
    },
    "auc_raw": 0.72468177538267,
    "auc_lc": 0.5489722205465404,
    "zero_fp": {
     "tp": 3,
     "recall": 0.006880733944954129,
     "f1": 0.01366742596810934,
     "threshold": 0.7544566506726902,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "958b7c62105ee24995c405bce1a9cf6f33d56ba7558789e12991e4651ca97474",
    "distinct_thresholds": 498,
    "rows_per_min": 27.67,
    "q4_k_m_bytes": 1057044608,
    "peak_rss_bytes": 1293201408
   },
   {
    "id": "granite-4.0-micro",
    "repo": "ibm-granite/granite-4.0-micro",
    "revision": "56111ae135df9c53a78c99028e7bc24035a9e979",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 3402836480,
    "params_short": "3.40B",
    "size_band": "3B to 6B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA (IBM)",
    "note": "GraniteMoeHybridForCausalLM",
    "backbone": null,
    "snapshot_bytes": 6815498811,
    "rank_at_budget": 2,
    "rank_lc_auc": 11,
    "at_budget": {
     "tp": 18,
     "fp": 9,
     "fn": 418,
     "tn": 3372,
     "threshold": 0.7545578959317609,
     "precision": 0.6666666666666666,
     "recall": 0.04128440366972477,
     "f1": 0.07775377969762419,
     "fpr": 0.0026619343389529724,
     "f1_ci95": [
      0.044444444444444446,
      0.11235955056179775
     ],
     "recall_ci95": [
      0.026271458772609037,
      0.06430994964559081
     ],
     "fpr_ci95": [
      0.001401104836631222,
      0.005051622443630781
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.2507462686567164,
     "fpr": 0.044365572315882874,
     "precision": 0.358974358974359,
     "recall": 0.1926605504587156,
     "tp": 84,
     "fp": 150,
     "fn": 352,
     "tn": 3231,
     "flagged_cases": 234,
     "flag_rate": 0.061304689546764476
    },
    "oracle_f1": 0.294201861130995,
    "oracle": {
     "f1": 0.294201861130995,
     "threshold": 0.0775999205599317,
     "tp": 411,
     "fp": 1947,
     "fn": 25,
     "tn": 1434,
     "precision": 0.1743002544529262,
     "recall": 0.9426605504587156,
     "fpr": 0.5758651286601597
    },
    "auc_raw": 0.7004360579493065,
    "auc_lc": 0.5074123785968511,
    "zero_fp": {
     "tp": 5,
     "recall": 0.011467889908256881,
     "f1": 0.022675736961451247,
     "threshold": 0.9044840990344754,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "dfa2f3f847e47d122091d1244c0ef19b7fabb47650e1f24facd10c16de1ec660",
    "distinct_thresholds": 751,
    "rows_per_min": 9.61,
    "q4_k_m_bytes": 2099502528,
    "peak_rss_bytes": 2584227840
   },
   {
    "id": "granite-4.0-1b",
    "repo": "ibm-granite/granite-4.0-1b",
    "revision": "6a7381ba1f54d684ff508d991aeb7dc580157103",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 1631750144,
    "params_short": "1.63B",
    "size_band": "under 3B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA (IBM)",
    "note": "GraniteMoeHybridForCausalLM (arch class hides dense/hybrid split)",
    "backbone": null,
    "snapshot_bytes": 3273295937,
    "rank_at_budget": 3,
    "rank_lc_auc": 9,
    "at_budget": {
     "tp": 16,
     "fp": 13,
     "fn": 420,
     "tn": 3368,
     "threshold": 0.5922008117092141,
     "precision": 0.5517241379310345,
     "recall": 0.03669724770642202,
     "f1": 0.06881720430107527,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.0367816091954023,
      0.1014799154334038
     ],
     "recall_ci95": [
      0.022712652185626465,
      0.05877457031675539
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.2584493041749503,
     "fpr": 0.13013901212658976,
     "precision": 0.22807017543859648,
     "recall": 0.2981651376146789,
     "tp": 130,
     "fp": 440,
     "fn": 306,
     "tn": 2941,
     "flagged_cases": 570,
     "flag_rate": 0.14933193607545192
    },
    "oracle_f1": 0.3356926188068756,
    "oracle": {
     "f1": 0.3356926188068756,
     "threshold": 0.27869593630004547,
     "tp": 332,
     "fp": 1210,
     "fn": 104,
     "tn": 2171,
     "precision": 0.21530479896238652,
     "recall": 0.7614678899082569,
     "fpr": 0.35788228334812183
    },
    "auc_raw": 0.7601440456517669,
    "auc_lc": 0.5580548350123665,
    "zero_fp": {
     "tp": 3,
     "recall": 0.006880733944954129,
     "f1": 0.01366742596810934,
     "threshold": 0.8267880536857767,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "44dae504260a11544d1daf5b3bf01046454c8f020c0ccf944794fb1faf988914",
    "distinct_thresholds": 504,
    "rows_per_min": 18.83,
    "q4_k_m_bytes": 1023645440,
    "peak_rss_bytes": 1400991744
   },
   {
    "id": "llama-3.2-1b-instruct",
    "repo": "meta-llama/Llama-3.2-1B-Instruct",
    "revision": "9213176726f574b556790deb65791e0c5aa438b6",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 1235814400,
    "params_short": "1.24B",
    "size_band": "under 3B",
    "licence": "llama3.2",
    "licence_name": null,
    "gated": "llama3.2",
    "origin": "USA (Meta)",
    "note": "",
    "backbone": null,
    "snapshot_bytes": 2480847368,
    "rank_at_budget": 4,
    "rank_lc_auc": 13,
    "at_budget": {
     "tp": 11,
     "fp": 13,
     "fn": 425,
     "tn": 3368,
     "threshold": 0.759435536377553,
     "precision": 0.4583333333333333,
     "recall": 0.02522935779816514,
     "f1": 0.04782608695652174,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.021834061135371178,
      0.07615230460921844
     ],
     "recall_ci95": [
      0.014144957508198771,
      0.04460680020463807
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.25620094191522763,
     "fpr": 0.6923986986098787,
     "precision": 0.14841760640232812,
     "recall": 0.9357798165137615,
     "tp": 408,
     "fp": 2341,
     "fn": 28,
     "tn": 1040,
     "flagged_cases": 2749,
     "flag_rate": 0.7201991092481006
    },
    "oracle_f1": 0.30276437033786746,
    "oracle": {
     "f1": 0.30276437033786746,
     "threshold": 0.41922895160969764,
     "tp": 345,
     "fp": 1498,
     "fn": 91,
     "tn": 1883,
     "precision": 0.187194791101465,
     "recall": 0.7912844036697247,
     "fpr": 0.44306418219461696
    },
    "auc_raw": 0.6867536204749152,
    "auc_lc": 0.4498853833624902,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "236286439a7624a23e3610309aa3a31de036d66291176f19f628453557f3c13d",
    "distinct_thresholds": 224,
    "rows_per_min": null,
    "q4_k_m_bytes": 807694368,
    "peak_rss_bytes": 1216184320
   },
   {
    "id": "phi-4-mini-instruct",
    "repo": "microsoft/Phi-4-mini-instruct",
    "revision": "cfbefacb99257ffa30c83adab238a50856ac3083",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 3836021760,
    "params_short": "3.84B",
    "size_band": "3B to 6B",
    "licence": "mit",
    "licence_name": null,
    "gated": null,
    "origin": "USA (Microsoft)",
    "note": null,
    "backbone": null,
    "snapshot_bytes": 7694059344,
    "rank_at_budget": 5,
    "rank_lc_auc": 12,
    "at_budget": {
     "tp": 10,
     "fp": 13,
     "fn": 426,
     "tn": 3368,
     "threshold": 0.6772987561633471,
     "precision": 0.43478260869565216,
     "recall": 0.022935779816513763,
     "f1": 0.04357298474945534,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.01834862385321101,
      0.07114624505928854
     ],
     "recall_ci95": [
      0.0125050447575181,
      0.041699619997409745
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.36122331217541837,
     "fpr": 0.29103815439219166,
     "precision": 0.2413261372397841,
     "recall": 0.7178899082568807,
     "tp": 313,
     "fp": 984,
     "fn": 123,
     "tn": 2397,
     "flagged_cases": 1297,
     "flag_rate": 0.3397956510348441
    },
    "oracle_f1": 0.34765366820885657,
    "oracle": {
     "f1": 0.34765366820885657,
     "threshold": 0.39099131515943186,
     "tp": 263,
     "fp": 814,
     "fn": 173,
     "tn": 2567,
     "precision": 0.24419684308263695,
     "recall": 0.6032110091743119,
     "fpr": 0.24075717243419106
    },
    "auc_raw": 0.7417631990969503,
    "auc_lc": 0.4998868914761416,
    "zero_fp": {
     "tp": 2,
     "recall": 0.0045871559633027525,
     "f1": 0.0091324200913242,
     "threshold": 0.8439652274928927,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "7388e2b13551ae7d04f808f5212248f2cc444b093ec86e11d88e7d5ee0e6e51b",
    "distinct_thresholds": 166,
    "rows_per_min": 10.08,
    "q4_k_m_bytes": 2493840704,
    "peak_rss_bytes": 3368312832
   },
   {
    "id": "shieldstral-1.0-3b",
    "repo": "mistralai/Shieldstral-1.0-3B",
    "revision": "003ec7e2b0bab5f0e6307edbaf186fa5822b76f5",
    "is_control": false,
    "class": "safety",
    "class_label": "Purpose-built safety classifier",
    "class_structure": "2-head (block, confirm; no allow)",
    "output_shape": "dual_binary_heads",
    "readout": "shieldstral",
    "params": 3849090048,
    "params_short": "3.85B",
    "size_band": "3B to 6B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "France (Mistral)",
    "note": "Mistral3ForConditionalGeneration: ministral3 text tower + Pixtral vision encoder",
    "backbone": null,
    "snapshot_bytes": 7731632731,
    "rank_at_budget": 6,
    "rank_lc_auc": 8,
    "at_budget": {
     "tp": 10,
     "fp": 13,
     "fn": 426,
     "tn": 3368,
     "threshold": 0.29421497216298875,
     "precision": 0.43478260869565216,
     "recall": 0.022935779816513763,
     "f1": 0.04357298474945534,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.01834862385321101,
      0.07114624505928854
     ],
     "recall_ci95": [
      0.0125050447575181,
      0.041699619997409745
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.017937219730941704,
     "fpr": 0.001774622892635315,
     "precision": 0.4,
     "recall": 0.009174311926605505,
     "tp": 4,
     "fp": 6,
     "fn": 432,
     "tn": 3375,
     "flagged_cases": 10,
     "flag_rate": 0.0026198585276395073
    },
    "oracle_f1": 0.3202979515828678,
    "oracle": {
     "f1": 0.3202979515828678,
     "threshold": 0.006692850924284856,
     "tp": 172,
     "fp": 466,
     "fn": 264,
     "tn": 2915,
     "precision": 0.26959247648902823,
     "recall": 0.3944954128440367,
     "fpr": 0.1378290446613428
    },
    "auc_raw": 0.7431331726946862,
    "auc_lc": 0.5673693219521023,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "d370b60df1178d4658329a286464dfdc2095da7cd9040d8952a773b5da4d1537",
    "distinct_thresholds": 105,
    "rows_per_min": 9.91,
    "q4_k_m_bytes": 2146497312,
    "peak_rss_bytes": 2816339968
   },
   {
    "id": "falcon3-3b-instruct",
    "repo": "tiiuae/Falcon3-3B-Instruct",
    "revision": "411bb94318f94f7a5735b77109f456b1e74b42a1",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 3227655168,
    "params_short": "3.23B",
    "size_band": "3B to 6B",
    "licence": "other",
    "licence_name": "Falcon LLM licence",
    "gated": null,
    "origin": "UAE (TII)",
    "note": null,
    "backbone": null,
    "snapshot_bytes": 6465509473,
    "rank_at_budget": 7,
    "rank_lc_auc": 18,
    "at_budget": {
     "tp": 9,
     "fp": 13,
     "fn": 427,
     "tn": 3368,
     "threshold": 0.7711428895470333,
     "precision": 0.4090909090909091,
     "recall": 0.020642201834862386,
     "f1": 0.039301310043668124,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.01639344262295082,
      0.06578947368421052
     ],
     "recall_ci95": [
      0.010897182921883053,
      0.03876038887513579
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.12439418416801293,
     "fpr": 0.21443359952676722,
     "precision": 0.09600997506234414,
     "recall": 0.17660550458715596,
     "tp": 77,
     "fp": 725,
     "fn": 359,
     "tn": 2656,
     "flagged_cases": 802,
     "flag_rate": 0.21011265391668849
    },
    "oracle_f1": 0.20699500356887937,
    "oracle": {
     "f1": 0.20699500356887937,
     "threshold": 0.004013126532015012,
     "tp": 435,
     "fp": 3332,
     "fn": 1,
     "tn": 49,
     "precision": 0.11547650650384922,
     "recall": 0.9977064220183486,
     "fpr": 0.9855072463768116
    },
    "auc_raw": 0.44662936973752404,
    "auc_lc": 0.3576114495988418,
    "zero_fp": {
     "tp": 2,
     "recall": 0.0045871559633027525,
     "f1": 0.0091324200913242,
     "threshold": 0.9622441940257137,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "6a3791986e18ede8492206e633386cfbba832d18e86aaefb558a7d7d920bde54",
    "distinct_thresholds": 981,
    "rows_per_min": 13.68,
    "q4_k_m_bytes": 2005684480,
    "peak_rss_bytes": 2302959616
   },
   {
    "id": "gemma-3-4b-it",
    "repo": "google/gemma-3-4b-it",
    "revision": "093f9f388b31de276ce2de164bdc2081324b9767",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 4300079472,
    "params_short": "4.30B",
    "size_band": "3B to 6B",
    "licence": "gemma",
    "licence_name": null,
    "gated": "gemma",
    "origin": "USA (Google)",
    "note": "multimodal Gemma3ForConditionalGeneration",
    "backbone": null,
    "snapshot_bytes": 8639634218,
    "rank_at_budget": 8,
    "rank_lc_auc": 16,
    "at_budget": {
     "tp": 5,
     "fp": 8,
     "fn": 431,
     "tn": 3373,
     "threshold": 0.9999869397150666,
     "precision": 0.38461538461538464,
     "recall": 0.011467889908256881,
     "f1": 0.022271714922048998,
     "fpr": 0.0023661638568470865,
     "f1_ci95": [
      0.0045871559633027525,
      0.0437636761487965
     ],
     "recall_ci95": [
      0.0049080708525622106,
      0.026561129112820638
     ],
     "fpr_ci95": [
      0.0011994651692201863,
      0.0046623924870633805
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": null,
    "oracle_f1": 0.22759509993552546,
    "oracle": null,
    "auc_raw": 0.5761212143413409,
    "auc_lc": 0.428430581528624,
    "zero_fp": {
     "tp": 2,
     "recall": 0.0045871559633027525,
     "f1": 0.0091324200913242,
     "threshold": 0.9999948349299194,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "b945f36e33a4e0ea4f41b76b400b1e22901d8b0b6d7a034cf85faf587b112d5e",
    "distinct_thresholds": 1999,
    "rows_per_min": null,
    "q4_k_m_bytes": 2489894144,
    "peak_rss_bytes": 3413061632
   },
   {
    "id": "shieldgemma-2b",
    "repo": "google/shieldgemma-2b",
    "revision": "d1dffc9c8c9237a90aab09c61383791e718ef9e8",
    "is_control": false,
    "class": "safety",
    "class_label": "Purpose-built safety classifier",
    "class_structure": "2-class",
    "output_shape": "two_class_scalar",
    "readout": "shieldgemma",
    "params": 2614341888,
    "params_short": "2.61B",
    "size_band": "under 3B",
    "licence": "gemma",
    "licence_name": null,
    "gated": "gemma",
    "origin": "USA (Google)",
    "note": "purpose-built safety classifier; accepts a plain-language guideline",
    "backbone": null,
    "snapshot_bytes": 5250578613,
    "rank_at_budget": 9,
    "rank_lc_auc": 2,
    "at_budget": {
     "tp": 5,
     "fp": 11,
     "fn": 431,
     "tn": 3370,
     "threshold": 0.5312093733737563,
     "precision": 0.3125,
     "recall": 0.011467889908256881,
     "f1": 0.022123893805309734,
     "fpr": 0.003253475303164744,
     "f1_ci95": [
      0.0045662100456621,
      0.04357298474945534
     ],
     "recall_ci95": [
      0.0049080708525622106,
      0.026561129112820638
     ],
     "fpr_ci95": [
      0.001817687995029199,
      0.005816778533189674
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.02207505518763797,
     "fpr": 0.00354924578527063,
     "precision": 0.29411764705882354,
     "recall": 0.011467889908256881,
     "tp": 5,
     "fp": 12,
     "fn": 431,
     "tn": 3369,
     "flagged_cases": 17,
     "flag_rate": 0.004453759496987163
    },
    "oracle_f1": 0.4167289021657954,
    "oracle": {
     "f1": 0.4167289021657954,
     "threshold": 0.012431650853185818,
     "tp": 279,
     "fp": 624,
     "fn": 157,
     "tn": 2757,
     "precision": 0.3089700996677741,
     "recall": 0.6399082568807339,
     "fpr": 0.18456078083407276
    },
    "auc_raw": 0.8292098450868182,
    "auc_lc": 0.7544168878566688,
    "zero_fp": {
     "tp": 1,
     "recall": 0.0022935779816513763,
     "f1": 0.004576659038901602,
     "threshold": 0.9046505351008906,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "d016410646fe40fed8e8ee9b6dc4977b63991f4c94646e3329258c84eed8177f",
    "distinct_thresholds": 69,
    "rows_per_min": null,
    "q4_k_m_bytes": 1708583392,
    "peak_rss_bytes": 2493304832
   },
   {
    "id": "granite-guardian-3.2-3b-a800m",
    "repo": "ibm-granite/granite-guardian-3.2-3b-a800m",
    "revision": "3de033d89b499a18d9a573b5192bf3b967ef48c5",
    "is_control": false,
    "class": "safety",
    "class_label": "Purpose-built safety classifier",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 3298793472,
    "params_short": "3.30B",
    "size_band": "3B to 6B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA (IBM)",
    "note": "MoE, ~800M active; purpose-built guardrail",
    "backbone": null,
    "snapshot_bytes": 6602467360,
    "rank_at_budget": 10,
    "rank_lc_auc": 3,
    "at_budget": {
     "tp": 5,
     "fp": 13,
     "fn": 431,
     "tn": 3368,
     "threshold": 0.6984180052295147,
     "precision": 0.2777777777777778,
     "recall": 0.011467889908256881,
     "f1": 0.022026431718061675,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.0045351473922902496,
      0.04338394793926247
     ],
     "recall_ci95": [
      0.0049080708525622106,
      0.026561129112820638
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.36666666666666664,
     "fpr": 0.14640638864241348,
     "precision": 0.296875,
     "recall": 0.4793577981651376,
     "tp": 209,
     "fp": 495,
     "fn": 227,
     "tn": 2886,
     "flagged_cases": 704,
     "flag_rate": 0.1844380403458213
    },
    "oracle_f1": 0.4059900166389351,
    "oracle": {
     "f1": 0.4059900166389351,
     "threshold": 0.4243980293295841,
     "tp": 244,
     "fp": 522,
     "fn": 192,
     "tn": 2859,
     "precision": 0.3185378590078329,
     "recall": 0.5596330275229358,
     "fpr": 0.1543921916592724
    },
    "auc_raw": 0.7847822016720529,
    "auc_lc": 0.6556844573807082,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "d76cfce2377f9be9a3e05926ccf93a4798f11dd312d3ccead34121c18743138e",
    "distinct_thresholds": 1570,
    "rows_per_min": 11.98,
    "q4_k_m_bytes": 2016896640,
    "peak_rss_bytes": 2211803136
   },
   {
    "id": "control-modernbert-large",
    "repo": "answerdotai/ModernBERT-large",
    "revision": "45bb4654a4d5aaff24dd11d4781fa46d39bf8c13",
    "is_control": true,
    "class": "control",
    "class_label": "MLM negative control",
    "class_structure": "2-class",
    "output_shape": "two_class_scalar",
    "readout": "mlm_control",
    "params": 395881664,
    "params_short": "396M",
    "size_band": "under 3B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA/France (Answer.AI/LightOn)",
    "note": null,
    "backbone": "ModernBERT",
    "snapshot_bytes": 1585715090,
    "rank_at_budget": null,
    "rank_lc_auc": null,
    "at_budget": {
     "tp": 2,
     "fp": 2,
     "fn": 434,
     "tn": 3379,
     "threshold": 0.6224593312018547,
     "precision": 0.5,
     "recall": 0.0045871559633027525,
     "f1": 0.00909090909090909,
     "fpr": 0.0005915409642117716,
     "f1_ci95": [
      0.0,
      0.022727272727272728
     ],
     "recall_ci95": [
      0.0012588662919984766,
      0.01656905479965736
     ],
     "fpr_ci95": [
      0.00016223686034824215,
      0.002154403052064717
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.18932748538011696,
     "fpr": 0.603667553978113,
     "precision": 0.1126086956521739,
     "recall": 0.5940366972477065,
     "tp": 259,
     "fp": 2041,
     "fn": 177,
     "tn": 1340,
     "flagged_cases": 2300,
     "flag_rate": 0.6025674613570867
    },
    "oracle_f1": 0.22993612885309636,
    "oracle": {
     "f1": 0.22993612885309636,
     "threshold": 0.46879062662624377,
     "tp": 414,
     "fp": 2751,
     "fn": 22,
     "tn": 630,
     "precision": 0.13080568720379146,
     "recall": 0.9495412844036697,
     "fpr": 0.8136645962732919
    },
    "auc_raw": 0.46228994190416495,
    "auc_lc": 0.38895947698618566,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "46cbba9bab176aa69afcdd383025999ae112b20e201ea13d126b672ee282f33e",
    "distinct_thresholds": 46,
    "rows_per_min": null,
    "q4_k_m_bytes": null,
    "peak_rss_bytes": null
   },
   {
    "id": "llama-guard-3-1b",
    "repo": "meta-llama/Llama-Guard-3-1B",
    "revision": "acf7aafa60f0410f8f42b1fa35e077d705892029",
    "is_control": false,
    "class": "safety",
    "class_label": "Purpose-built safety classifier",
    "class_structure": "2-class",
    "output_shape": "two_class_scalar",
    "readout": "llamaguard",
    "params": 1498482688,
    "params_short": "1.50B",
    "size_band": "under 3B",
    "licence": "llama3.2",
    "licence_name": null,
    "gated": "llama3.2",
    "origin": "USA (Meta)",
    "note": "purpose-built safety classifier; default taxonomy S1-S13 has NO destructive-tool-call category",
    "backbone": null,
    "snapshot_bytes": 3006170568,
    "rank_at_budget": 11,
    "rank_lc_auc": 6,
    "at_budget": {
     "tp": 2,
     "fp": 12,
     "fn": 434,
     "tn": 3369,
     "threshold": 0.5,
     "precision": 0.14285714285714285,
     "recall": 0.0045871559633027525,
     "f1": 0.008888888888888889,
     "fpr": 0.00354924578527063,
     "f1_ci95": [
      0.0,
      0.022172949002217297
     ],
     "recall_ci95": [
      0.0012588662919984766,
      0.01656905479965736
     ],
     "fpr_ci95": [
      0.0020315170559029564,
      0.006193819096294351
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.008888888888888889,
     "fpr": 0.00354924578527063,
     "precision": 0.14285714285714285,
     "recall": 0.0045871559633027525,
     "tp": 2,
     "fp": 12,
     "fn": 434,
     "tn": 3369,
     "flagged_cases": 14,
     "flag_rate": 0.0036678019386953103
    },
    "oracle_f1": 0.32502396931927136,
    "oracle": {
     "f1": 0.32502396931927136,
     "threshold": 0.4073334000459302,
     "tp": 339,
     "fp": 1311,
     "fn": 97,
     "tn": 2070,
     "precision": 0.20545454545454545,
     "recall": 0.7775229357798165,
     "fpr": 0.3877551020408163
    },
    "auc_raw": 0.744483473485126,
    "auc_lc": 0.5743123001749412,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "68795ca2ee59c16a70f6b67b752ae751dcbc4e563f0ccfe7c30cd61e556c1e36",
    "distinct_thresholds": 14,
    "rows_per_min": null,
    "q4_k_m_bytes": 955443968,
    "peak_rss_bytes": 1148407808
   },
   {
    "id": "prompt-guard-2-22m",
    "repo": "meta-llama/Llama-Prompt-Guard-2-22M",
    "revision": "11614a155199674a0a95e6602d6ab0417b790ed0",
    "is_control": false,
    "class": "encoder",
    "class_label": "Trained encoder classifier",
    "class_structure": "2-class",
    "output_shape": "two_class_scalar",
    "readout": "seqcls",
    "params": 70830722,
    "params_short": "70.8M",
    "size_band": "under 3B",
    "licence": "other",
    "licence_name": null,
    "gated": "promptguard2",
    "origin": "USA (Meta)",
    "note": "trained injection/jailbreak encoder; 512-token limit; repo name understates params",
    "backbone": "DebertaV2",
    "snapshot_bytes": 292053522,
    "rank_at_budget": 12,
    "rank_lc_auc": 15,
    "at_budget": {
     "tp": 2,
     "fp": 13,
     "fn": 434,
     "tn": 3368,
     "threshold": 0.01855545458499391,
     "precision": 0.13333333333333333,
     "recall": 0.0045871559633027525,
     "f1": 0.008869179600886918,
     "fpr": 0.003845016267376516,
     "f1_ci95": [
      0.0,
      0.022172949002217297
     ],
     "recall_ci95": [
      0.0012588662919984766,
      0.01656905479965736
     ],
     "fpr_ci95": [
      0.0022484734019615113,
      0.0065677323742142305
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.0,
     "fpr": 0.0,
     "precision": null,
     "recall": 0.0,
     "tp": 0,
     "fp": 0,
     "fn": 436,
     "tn": 3381,
     "flagged_cases": 0,
     "flag_rate": 0.0
    },
    "oracle_f1": 0.27893422148209823,
    "oracle": {
     "f1": 0.27893422148209823,
     "threshold": 0.00414302382887012,
     "tp": 335,
     "fp": 1631,
     "fn": 101,
     "tn": 1750,
     "precision": 0.1703967446592065,
     "recall": 0.768348623853211,
     "fpr": 0.4824016563146998
    },
    "auc_raw": 0.6248005584363782,
    "auc_lc": 0.4334488297038065,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 510,
    "shrunk_rows": 17202,
    "frac_shrunk": 0.5675354668426262,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "b446c1297e78490c40697800cbd656adf8e3587ae6e1b7c930a25501e27ed1fa",
    "distinct_thresholds": 2068,
    "rows_per_min": null,
    "q4_k_m_bytes": null,
    "peak_rss_bytes": null
   },
   {
    "id": "olmo-2-1b-instruct",
    "repo": "allenai/OLMo-2-0425-1B-Instruct",
    "revision": "48d788eca847d4d7548f375ad03d3c9312f6139e",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 1484916736,
    "params_short": "1.48B",
    "size_band": "under 3B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA (Ai2)",
    "note": null,
    "backbone": null,
    "snapshot_bytes": 2979535483,
    "rank_at_budget": 13,
    "rank_lc_auc": 7,
    "at_budget": {
     "tp": 1,
     "fp": 1,
     "fn": 435,
     "tn": 3380,
     "threshold": 0.24289531114035928,
     "precision": 0.5,
     "recall": 0.0022935779816513763,
     "f1": 0.0045662100456621,
     "fpr": 0.0002957704821058858,
     "f1_ci95": [
      0.0,
      0.014218009478672985
     ],
     "recall_ci95": [
      0.0004049876311643646,
      0.012875840502582472
     ],
     "fpr_ci95": [
      5.2212660819723426e-05,
      0.001673557627614801
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.0,
     "fpr": 0.0,
     "precision": null,
     "recall": 0.0,
     "tp": 0,
     "fp": 0,
     "fn": 436,
     "tn": 3381,
     "flagged_cases": 0,
     "flag_rate": 0.0
    },
    "oracle_f1": 0.3479236812570146,
    "oracle": {
     "f1": 0.3479236812570146,
     "threshold": 0.1863237232258476,
     "tp": 310,
     "fp": 1036,
     "fn": 126,
     "tn": 2345,
     "precision": 0.2303120356612184,
     "recall": 0.7110091743119266,
     "fpr": 0.3064182194616977
    },
    "auc_raw": 0.7539871353407738,
    "auc_lc": 0.569252578874344,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "f7a32cfd6eb06a790c22de827891123deb9f5c455043f27cb4c437ecdbda0db1",
    "distinct_thresholds": 47,
    "rows_per_min": 30.38,
    "q4_k_m_bytes": 935514976,
    "peak_rss_bytes": 1237389312
   },
   {
    "id": "smollm3-3b",
    "repo": "HuggingFaceTB/SmolLM3-3B",
    "revision": "a07cc9a04f16550a088caea529712d1d335b0ac1",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 3075098624,
    "params_short": "3.08B",
    "size_band": "3B to 6B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "France/USA (HuggingFace)",
    "note": null,
    "backbone": null,
    "snapshot_bytes": 6167868975,
    "rank_at_budget": 14,
    "rank_lc_auc": 17,
    "at_budget": {
     "tp": 1,
     "fp": 4,
     "fn": 435,
     "tn": 3377,
     "threshold": 0.05944623924176597,
     "precision": 0.2,
     "recall": 0.0022935779816513763,
     "f1": 0.0045351473922902496,
     "fpr": 0.0011830819284235432,
     "f1_ci95": [
      0.0,
      0.014150943396226415
     ],
     "recall_ci95": [
      0.0004049876311643646,
      0.012875840502582472
     ],
     "fpr_ci95": [
      0.0004601706357698699,
      0.0030382085245999587
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.0,
     "fpr": 0.0,
     "precision": null,
     "recall": 0.0,
     "tp": 0,
     "fp": 0,
     "fn": 436,
     "tn": 3381,
     "flagged_cases": 0,
     "flag_rate": 0.0
    },
    "oracle_f1": 0.3075070252910478,
    "oracle": {
     "f1": 0.3075070252910478,
     "threshold": 0.005885615743717063,
     "tp": 383,
     "fp": 1672,
     "fn": 53,
     "tn": 1709,
     "precision": 0.18637469586374697,
     "recall": 0.8784403669724771,
     "fpr": 0.4945282460810411
    },
    "auc_raw": 0.6742288259539955,
    "auc_lc": 0.4167068830307052,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "0a84032a29f5d510fbee50cdb33afd97e45f34c5670fbb8f337c5375db7cba1f",
    "distinct_thresholds": 2512,
    "rows_per_min": 11.4,
    "q4_k_m_bytes": 1915305888,
    "peak_rss_bytes": 2422538240
   },
   {
    "id": "smollm2-1.7b-instruct",
    "repo": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
    "revision": "31b70e2e869a7173562077fd711b654946d38674",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 1711376384,
    "params_short": "1.71B",
    "size_band": "under 3B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "France/USA (HuggingFace)",
    "note": null,
    "backbone": null,
    "snapshot_bytes": 3426390374,
    "rank_at_budget": 15,
    "rank_lc_auc": 4,
    "at_budget": {
     "tp": 1,
     "fp": 7,
     "fn": 435,
     "tn": 3374,
     "threshold": 0.5122626854994002,
     "precision": 0.125,
     "recall": 0.0022935779816513763,
     "f1": 0.0045045045045045045,
     "fpr": 0.002070393374741201,
     "f1_ci95": [
      0.0,
      0.01405152224824356
     ],
     "recall_ci95": [
      0.0004049876311643646,
      0.012875840502582472
     ],
     "fpr_ci95": [
      0.0010032670811636636,
      0.004267720951141469
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.20790419161676646,
     "fpr": 0.9775214433599527,
     "precision": 0.11607381652848354,
     "recall": 0.9954128440366973,
     "tp": 434,
     "fp": 3305,
     "fn": 2,
     "tn": 76,
     "flagged_cases": 3739,
     "flag_rate": 0.9795651034844118
    },
    "oracle_f1": 0.32369230769230767,
    "oracle": {
     "f1": 0.32369230769230767,
     "threshold": 0.44993349160089036,
     "tp": 263,
     "fp": 926,
     "fn": 173,
     "tn": 2455,
     "precision": 0.22119428090832632,
     "recall": 0.6032110091743119,
     "fpr": 0.2738834664300503
    },
    "auc_raw": 0.7171477685609545,
    "auc_lc": 0.5985156391385655,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "4fe2c3af3206fe2ee67727b7c89c1c2feb15047facb84bc7fbb41a1b3a093f19",
    "distinct_thresholds": 64,
    "rows_per_min": 19.35,
    "q4_k_m_bytes": 1055609952,
    "peak_rss_bytes": 1517502464
   },
   {
    "id": "deberta-v3-prompt-injection-v2",
    "repo": "protectai/deberta-v3-base-prompt-injection-v2",
    "revision": "90c9989b1a342275dd0d1a95aad283c04e075671",
    "is_control": false,
    "class": "encoder",
    "class_label": "Trained encoder classifier",
    "class_structure": "2-class",
    "output_shape": "two_class_scalar",
    "readout": "seqcls",
    "params": 184423682,
    "params_short": "184M",
    "size_band": "under 3B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA (ProtectAI)",
    "note": "trained 2-class injection head; 512-token limit",
    "backbone": "DebertaV2",
    "snapshot_bytes": 748866413,
    "rank_at_budget": 16,
    "rank_lc_auc": 1,
    "at_budget": {
     "tp": 1,
     "fp": 11,
     "fn": 435,
     "tn": 3370,
     "threshold": 0.999999142076462,
     "precision": 0.08333333333333333,
     "recall": 0.0022935779816513763,
     "f1": 0.004464285714285714,
     "fpr": 0.003253475303164744,
     "f1_ci95": [
      0.0,
      0.013986013986013986
     ],
     "recall_ci95": [
      0.0004049876311643646,
      0.012875840502582472
     ],
     "fpr_ci95": [
      0.001817687995029199,
      0.005816778533189674
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.21057810578105782,
     "fpr": 0.9467613132209406,
     "precision": 0.11793882612289887,
     "recall": 0.981651376146789,
     "tp": 428,
     "fp": 3201,
     "fn": 8,
     "tn": 180,
     "flagged_cases": 3629,
     "flag_rate": 0.9507466596803773
    },
    "oracle_f1": 0.48,
    "oracle": {
     "f1": 0.48,
     "threshold": 0.9987551566979116,
     "tp": 174,
     "fp": 115,
     "fn": 262,
     "tn": 3266,
     "precision": 0.6020761245674741,
     "recall": 0.39908256880733944,
     "fpr": 0.034013605442176874
    },
    "auc_raw": 0.8390509973434926,
    "auc_lc": 0.8347427912167461,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 510,
    "shrunk_rows": 17202,
    "frac_shrunk": 0.5675354668426262,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "927baa5ccc8bbf9b7885381ed50781264f848ab1c666e053166f9aab1721eeb4",
    "distinct_thresholds": 864,
    "rows_per_min": 186.7,
    "q4_k_m_bytes": null,
    "peak_rss_bytes": null
   },
   {
    "id": "control-modernbert-base",
    "repo": "answerdotai/ModernBERT-base",
    "revision": "8949b909ec900327062f0ebf497f51aef5e6f0c8",
    "is_control": true,
    "class": "control",
    "class_label": "MLM negative control",
    "class_structure": "2-class",
    "output_shape": "two_class_scalar",
    "readout": "mlm_control",
    "params": 149655232,
    "params_short": "150M",
    "size_band": "under 3B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA/France (Answer.AI/LightOn)",
    "note": null,
    "backbone": "ModernBERT",
    "snapshot_bytes": 600805271,
    "rank_at_budget": null,
    "rank_lc_auc": null,
    "at_budget": {
     "tp": 0,
     "fp": 1,
     "fn": 436,
     "tn": 3380,
     "threshold": 0.7185943925708562,
     "precision": 0.0,
     "recall": 0.0,
     "f1": 0.0,
     "fpr": 0.0002957704821058858,
     "f1_ci95": [
      0.0,
      0.0
     ],
     "recall_ci95": [
      0.0,
      0.008733735175837835
     ],
     "fpr_ci95": [
      5.2212660819723426e-05,
      0.001673557627614801
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.2496318114874816,
     "fpr": 0.5740905057675244,
     "precision": 0.14868421052631578,
     "recall": 0.7775229357798165,
     "tp": 339,
     "fp": 1941,
     "fn": 97,
     "tn": 1440,
     "flagged_cases": 2280,
     "flag_rate": 0.5973277443018077
    },
    "oracle_f1": 0.30849478390462,
    "oracle": {
     "f1": 0.30849478390462,
     "threshold": 0.5774953651858118,
     "tp": 207,
     "fp": 699,
     "fn": 229,
     "tn": 2682,
     "precision": 0.22847682119205298,
     "recall": 0.47477064220183485,
     "fpr": 0.2067435669920142
    },
    "auc_raw": 0.6622467295653802,
    "auc_lc": 0.4876428937684744,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "9d62074edd4ca545839b6f4246e4e7638804b3dc1c96cd2d940c8f1eae74a984",
    "distinct_thresholds": 53,
    "rows_per_min": 44.6,
    "q4_k_m_bytes": null,
    "peak_rss_bytes": null
   },
   {
    "id": "prompt-guard-2-86m",
    "repo": "meta-llama/Llama-Prompt-Guard-2-86M",
    "revision": "a8ded8e697ce7c355e395a0df51f94adb4a2fd27",
    "is_control": false,
    "class": "encoder",
    "class_label": "Trained encoder classifier",
    "class_structure": "2-class",
    "output_shape": "two_class_scalar",
    "readout": "seqcls",
    "params": 278810882,
    "params_short": "279M",
    "size_band": "under 3B",
    "licence": "other",
    "licence_name": null,
    "gated": "promptguard2",
    "origin": "USA (Meta)",
    "note": "trained injection/jailbreak encoder; 512-token limit; repo name understates params",
    "backbone": "DebertaV2",
    "snapshot_bytes": 1131677308,
    "rank_at_budget": 17,
    "rank_lc_auc": 19,
    "at_budget": {
     "tp": 0,
     "fp": 1,
     "fn": 436,
     "tn": 3380,
     "threshold": 0.9940428462362503,
     "precision": 0.0,
     "recall": 0.0,
     "f1": 0.0,
     "fpr": 0.0002957704821058858,
     "f1_ci95": [
      0.0,
      0.0
     ],
     "recall_ci95": [
      0.0,
      0.008733735175837835
     ],
     "fpr_ci95": [
      5.2212660819723426e-05,
      0.001673557627614801
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.0,
     "fpr": 0.0008873114463176575,
     "precision": 0.0,
     "recall": 0.0,
     "tp": 0,
     "fp": 3,
     "fn": 436,
     "tn": 3378,
     "flagged_cases": 3,
     "flag_rate": 0.0007859575582918522
    },
    "oracle_f1": 0.30047289923608583,
    "oracle": {
     "f1": 0.30047289923608583,
     "threshold": 0.0024927265619265125,
     "tp": 413,
     "fp": 1900,
     "fn": 23,
     "tn": 1481,
     "precision": 0.1785559878945093,
     "recall": 0.9472477064220184,
     "fpr": 0.5619639160011831
    },
    "auc_raw": 0.6278932594178477,
    "auc_lc": 0.3430242203052422,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 510,
    "shrunk_rows": 5396,
    "frac_shrunk": 0.17802705377763114,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "6b34ce108bb324043d5094cd32d2c282882312acc2bccbfbdcef5d460cdd25e5",
    "distinct_thresholds": 3076,
    "rows_per_min": null,
    "q4_k_m_bytes": null,
    "peak_rss_bytes": null
   },
   {
    "id": "gemma-3-1b-it",
    "repo": "google/gemma-3-1b-it",
    "revision": "dcc83ea841ab6100d6b47a070329e1ba4cf78752",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 999885952,
    "params_short": "1000M",
    "size_band": "under 3B",
    "licence": "gemma",
    "licence_name": null,
    "gated": "gemma",
    "origin": "USA (Google)",
    "note": "",
    "backbone": null,
    "snapshot_bytes": 2039072537,
    "rank_at_budget": 18,
    "rank_lc_auc": 5,
    "at_budget": {
     "tp": 0,
     "fp": 1,
     "fn": 436,
     "tn": 3380,
     "threshold": 0.999999463410758,
     "precision": 0.0,
     "recall": 0.0,
     "f1": 0.0,
     "fpr": 0.0002957704821058858,
     "f1_ci95": [
      0.0,
      0.0
     ],
     "recall_ci95": [
      0.0,
      0.008733735175837835
     ],
     "fpr_ci95": [
      5.2212660819723426e-05,
      0.001673557627614801
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.20748867159551634,
     "fpr": 0.9825495415557527,
     "precision": 0.11578387010912962,
     "recall": 0.9977064220183486,
     "tp": 435,
     "fp": 3322,
     "fn": 1,
     "tn": 59,
     "flagged_cases": 3757,
     "flag_rate": 0.984280848834163
    },
    "oracle_f1": 0.3463251670378619,
    "oracle": {
     "f1": 0.3463251670378619,
     "threshold": 0.9985169609154315,
     "tp": 311,
     "fp": 1049,
     "fn": 125,
     "tn": 2332,
     "precision": 0.2286764705882353,
     "recall": 0.713302752293578,
     "fpr": 0.3102632357290742
    },
    "auc_raw": 0.7427346287537752,
    "auc_lc": 0.5804258158894854,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "f4719aab1ba8b9497c09d63850bafe07598aa5fd652cd8fadd3031939273e38d",
    "distinct_thresholds": 1404,
    "rows_per_min": null,
    "q4_k_m_bytes": 806058400,
    "peak_rss_bytes": 1307176960
   },
   {
    "id": "granite-guardian-3.1-2b",
    "repo": "ibm-granite/granite-guardian-3.1-2b",
    "revision": "81145486e85c6c82c01e759c0356d9d6da4d21a5",
    "is_control": false,
    "class": "safety",
    "class_label": "Purpose-built safety classifier",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 2533531648,
    "params_short": "2.53B",
    "size_band": "under 3B",
    "licence": "apache-2.0",
    "licence_name": null,
    "gated": null,
    "origin": "USA (IBM)",
    "note": "purpose-built guardrail, fixed taxonomy",
    "backbone": null,
    "snapshot_bytes": 5071957207,
    "rank_at_budget": 19,
    "rank_lc_auc": 20,
    "at_budget": {
     "tp": 0,
     "fp": 1,
     "fn": 436,
     "tn": 3380,
     "threshold": 0.1713716318071269,
     "precision": 0.0,
     "recall": 0.0,
     "f1": 0.0,
     "fpr": 0.0002957704821058858,
     "f1_ci95": [
      0.0,
      0.0
     ],
     "recall_ci95": [
      0.0,
      0.008733735175837835
     ],
     "fpr_ci95": [
      5.2212660819723426e-05,
      0.001673557627614801
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.0,
     "fpr": 0.0,
     "precision": null,
     "recall": 0.0,
     "tp": 0,
     "fp": 0,
     "fn": 436,
     "tn": 3381,
     "flagged_cases": 0,
     "flag_rate": 0.0
    },
    "oracle_f1": 0.2926646706586826,
    "oracle": {
     "f1": 0.2926646706586826,
     "threshold": 0.017529003798401006,
     "tp": 391,
     "fp": 1845,
     "fn": 45,
     "tn": 1536,
     "precision": 0.17486583184257604,
     "recall": 0.8967889908256881,
     "fpr": 0.5456965394853593
    },
    "auc_raw": 0.6619360348846359,
    "auc_lc": 0.3008762140314894,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "10abb332428f9da6ce3c200395a421ae5cecd1c2e26a776fd3eb9c970dcd780a",
    "distinct_thresholds": 807,
    "rows_per_min": 12.45,
    "q4_k_m_bytes": 1545303040,
    "peak_rss_bytes": 1789657088
   },
   {
    "id": "llama-3.2-3b-instruct",
    "repo": "meta-llama/Llama-3.2-3B-Instruct",
    "revision": "0cb88a4f764b7a12671c53f0838cd831a0843b95",
    "is_control": false,
    "class": "general",
    "class_label": "General decoder",
    "class_structure": "3-class",
    "output_shape": "three_key_disposition",
    "readout": "letter3",
    "params": 3212749824,
    "params_short": "3.21B",
    "size_band": "3B to 6B",
    "licence": "llama3.2",
    "licence_name": null,
    "gated": "llama3.2",
    "origin": "USA (Meta)",
    "note": "",
    "backbone": null,
    "snapshot_bytes": 6434752520,
    "rank_at_budget": 20,
    "rank_lc_auc": 14,
    "at_budget": {
     "tp": 0,
     "fp": 1,
     "fn": 436,
     "tn": 3380,
     "threshold": 0.5178172830852636,
     "precision": 0.0,
     "recall": 0.0,
     "f1": 0.0,
     "fpr": 0.0002957704821058858,
     "f1_ci95": [
      0.0,
      0.0
     ],
     "recall_ci95": [
      0.0,
      0.008733735175837835
     ],
     "fpr_ci95": [
      5.2212660819723426e-05,
      0.001673557627614801
     ],
     "max_false_positives_allowed": 13
    },
    "default_decision": {
     "f1": 0.3548387096774194,
     "fpr": 0.2028985507246377,
     "precision": 0.2607758620689655,
     "recall": 0.555045871559633,
     "tp": 242,
     "fp": 686,
     "fn": 194,
     "tn": 2695,
     "flagged_cases": 928,
     "flag_rate": 0.24312287136494629
    },
    "oracle_f1": 0.318342151675485,
    "oracle": {
     "f1": 0.318342151675485,
     "threshold": 0.3071958857184984,
     "tp": 361,
     "fp": 1471,
     "fn": 75,
     "tn": 1910,
     "precision": 0.1970524017467249,
     "recall": 0.8279816513761468,
     "fpr": 0.43507837917775805
    },
    "auc_raw": 0.7197089645590985,
    "auc_lc": 0.44718397478433974,
    "zero_fp": {
     "tp": 0,
     "recall": 0.0,
     "f1": 0.0,
     "threshold": null,
     "rule_of_three_upper_bound": 0.0008873114463176575
    },
    "cap_tokens": 6144,
    "shrunk_rows": 0,
    "frac_shrunk": 0.0,
    "rows": 30310,
    "errors": 0,
    "settled": true,
    "digest": "2501574f34cb99d8271c64a0d94a624762c35260f5d5dc35fd18f5498e2c97ca",
    "distinct_thresholds": 155,
    "rows_per_min": null,
    "q4_k_m_bytes": 2019377600,
    "peak_rss_bytes": 2678620160
   }
  ],
  "baselines": {
   "block_everything": {
    "id": "block-everything",
    "label": "block every case",
    "f1": 0.20503174229955326,
    "precision": 0.11422583180508253,
    "recall": 1.0,
    "fpr": 1.0,
    "tp": 436,
    "fp": 3381,
    "fn": 0,
    "tn": 0
   },
   "allow_everything": {
    "id": "allow-everything",
    "label": "allow every case",
    "f1": 0.0,
    "precision": 0.0,
    "recall": 0.0,
    "fpr": 0.0,
    "tp": 0,
    "fp": 0,
    "fn": 436,
    "tn": 3381,
    "accuracy": 0.8857741681949175
   }
  },
  "counts": {
   "models": 22,
   "candidates": 20,
   "controls": 2,
   "with_default_decision": 21,
   "without_default_decision": [
    "gemma-3-4b-it"
   ],
   "bands": {
    "under 3B": 14,
    "3B to 6B": 8,
    "6B and up": 0
   }
  },
  "interval_methods": {
   "f1": "percentile bootstrap over cases at the arm's fixed threshold",
   "f1_resamples": 2000,
   "f1_seed": 20260924,
   "recall_and_fpr": "Wilson 95% at z = 1.959963984540054"
  }
 }
}