{ "task": "injection", "threshold": 0.02, "per_language": { "az": { "support": 42, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9943, "n": 174 }, "bg": { "support": 41, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9884, "n": 172 }, "cs": { "support": 42, "precision": 0.9545, "recall": 1.0, "f1": 0.9767, "fpr": 0.0152, "accuracy": 0.9828, "n": 174 }, "da": { "support": 41, "precision": 0.9762, "recall": 1.0, "f1": 0.988, "fpr": 0.0076, "accuracy": 0.9942, "n": 173 }, "de": { "support": 42, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9943, "n": 174 }, "el": { "support": 42, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9884, "n": 173 }, "en": { "support": 41, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9942, "n": 173 }, "es": { "support": 40, "precision": 0.9756, "recall": 1.0, "f1": 0.9877, "fpr": 0.0076, "accuracy": 0.9884, "n": 172 }, "et": { "support": 41, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9942, "n": 173 }, "fi": { "support": 42, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 1.0, "n": 174 }, "fr": { "support": 41, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9884, "n": 173 }, "ga": { "support": 42, "precision": 0.9762, "recall": 0.9762, "f1": 0.9762, "fpr": 0.0076, "accuracy": 0.977, "n": 174 }, "hr": { "support": 42, "precision": 0.9767, "recall": 1.0, "f1": 0.9882, "fpr": 0.0076, "accuracy": 0.9943, "n": 174 }, "hu": { "support": 42, "precision": 0.9545, "recall": 1.0, "f1": 0.9767, "fpr": 0.0152, "accuracy": 0.977, "n": 174 }, "it": { "support": 42, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 1.0, "n": 174 }, "lt": { "support": 42, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9943, "n": 174 }, "lv": { "support": 41, "precision": 0.9762, "recall": 1.0, "f1": 0.988, "fpr": 0.0076, "accuracy": 0.9884, "n": 173 }, "mt": { "support": 42, "precision": 0.8039, "recall": 0.9762, "f1": 0.8817, "fpr": 0.0758, "accuracy": 0.9195, "n": 174 }, "nl": { "support": 41, "precision": 1.0, "recall": 0.9756, "f1": 0.9877, "fpr": 0.0, "accuracy": 0.9942, "n": 173 }, "pl": { "support": 41, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 1.0, "n": 173 }, "pt": { "support": 42, "precision": 0.9767, "recall": 1.0, "f1": 0.9882, "fpr": 0.0076, "accuracy": 0.977, "n": 174 }, "ro": { "support": 42, "precision": 0.9767, "recall": 1.0, "f1": 0.9882, "fpr": 0.0076, "accuracy": 0.9884, "n": 173 }, "sk": { "support": 41, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9942, "n": 173 }, "sl": { "support": 42, "precision": 0.9767, "recall": 1.0, "f1": 0.9882, "fpr": 0.0076, "accuracy": 0.977, "n": 174 }, "sv": { "support": 41, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9884, "n": 173 }, "tr": { "support": 42, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9943, "n": 174 } }, "per_register": { "encoded_payload": { "support": 155, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9677, "n": 155 }, "hypothetical_framing": { "support": 156, "precision": 1.0, "recall": 0.9936, "f1": 0.9968, "fpr": 0.0, "accuracy": 0.9808, "n": 156 }, "ignore_instructions": { "support": 155, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9419, "n": 155 }, "instruction_in_document": { "support": 155, "precision": 1.0, "recall": 0.9935, "f1": 0.9968, "fpr": 0.0, "accuracy": 0.9677, "n": 155 }, "instruction_in_tool_output": { "support": 148, "precision": 1.0, "recall": 0.9932, "f1": 0.9966, "fpr": 0.0, "accuracy": 0.973, "n": 148 }, "meta_question": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0118, "accuracy": 0.9882, "n": 338 }, "mundane_account_access": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 182 }, "mundane_informational": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 182 }, "mundane_operational": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0055, "accuracy": 0.9945, "n": 182 }, "mundane_transactional": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0055, "accuracy": 0.9945, "n": 182 }, "ordinary_instruction": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 338 }, "ordinary_question": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0059, "accuracy": 0.9941, "n": 338 }, "persona_override": { "support": 155, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9484, "n": 155 }, "quoted_attack": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0089, "accuracy": 0.9911, "n": 338 }, "roleplay_benign": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0118, "accuracy": 0.9882, "n": 338 }, "security_discussion": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0089, "accuracy": 0.9911, "n": 338 }, "system_prompt_extraction": { "support": 156, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9936, "n": 156 }, "technical_identifiers": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.006, "accuracy": 0.994, "n": 336 }, "technical_payload": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0059, "accuracy": 0.9941, "n": 337 } }, "per_label": { "direct_injection": { "support": 466, "precision": 0.9528, "recall": 0.9957, "f1": 0.9738, "fpr": 0.0057, "accuracy": 0.0, "n": 0 }, "indirect_injection": { "support": 303, "precision": 0.9709, "recall": 0.9901, "f1": 0.9804, "fpr": 0.0021, "accuracy": 0.0, "n": 0 }, "jailbreak": { "support": 466, "precision": 0.9367, "recall": 0.985, "f1": 0.9603, "fpr": 0.0077, "accuracy": 0.0, "n": 0 } }, "worst_languages": [ [ "mt", 0.8817204301075269 ], [ "ga", 0.9761904761904762 ], [ "cs", 0.9767441860465117 ], [ "hu", 0.9767441860465117 ], [ "es", 0.9876543209876543 ] ], "calibration": { "threshold": 0.02, "objective": "recall_at_fpr", "value_on_val": 0.9907, "default_threshold": 0.5, "default_value_on_val": 0.9861, "improvement": 0.0046, "chosen_on": "validation split, never test", "curve": [ { "threshold": 0.02, "macro_f1": 0.9678, "any_label_recall": 0.9907, "any_label_fpr": 0.007 }, { "threshold": 0.03, "macro_f1": 0.9693, "any_label_recall": 0.9898, "any_label_fpr": 0.0064 }, { "threshold": 0.04, "macro_f1": 0.9705, "any_label_recall": 0.9898, "any_label_fpr": 0.0058 }, { "threshold": 0.05, "macro_f1": 0.9712, "any_label_recall": 0.9898, "any_label_fpr": 0.0055 }, { "threshold": 0.06, "macro_f1": 0.9729, "any_label_recall": 0.9898, "any_label_fpr": 0.0052 }, { "threshold": 0.07, "macro_f1": 0.9725, "any_label_recall": 0.9898, "any_label_fpr": 0.0052 }, { "threshold": 0.08, "macro_f1": 0.9723, "any_label_recall": 0.9898, "any_label_fpr": 0.005 }, { "threshold": 0.09, "macro_f1": 0.9727, "any_label_recall": 0.9898, "any_label_fpr": 0.005 }, { "threshold": 0.1, "macro_f1": 0.9737, "any_label_recall": 0.9898, "any_label_fpr": 0.0044 }, { "threshold": 0.11, "macro_f1": 0.9733, "any_label_recall": 0.9898, "any_label_fpr": 0.0044 }, { "threshold": 0.12, "macro_f1": 0.9733, "any_label_recall": 0.9898, "any_label_fpr": 0.0044 }, { "threshold": 0.13, "macro_f1": 0.9733, "any_label_recall": 0.9898, "any_label_fpr": 0.0044 }, { "threshold": 0.14, "macro_f1": 0.9733, "any_label_recall": 0.9889, "any_label_fpr": 0.0044 }, { "threshold": 0.15, "macro_f1": 0.9731, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.16, "macro_f1": 0.9731, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.17, "macro_f1": 0.9727, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.18, "macro_f1": 0.9722, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.19, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.2, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.21, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.22, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.23, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.24, "macro_f1": 0.9714, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.25, "macro_f1": 0.9714, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.26, "macro_f1": 0.9714, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.27, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.28, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.29, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.3, "macro_f1": 0.9718, "any_label_recall": 0.988, "any_label_fpr": 0.0041 }, { "threshold": 0.31, "macro_f1": 0.9714, "any_label_recall": 0.987, "any_label_fpr": 0.0041 }, { "threshold": 0.32, "macro_f1": 0.9714, "any_label_recall": 0.987, "any_label_fpr": 0.0041 }, { "threshold": 0.33, "macro_f1": 0.9717, "any_label_recall": 0.987, "any_label_fpr": 0.0038 }, { "threshold": 0.34, "macro_f1": 0.9717, "any_label_recall": 0.987, "any_label_fpr": 0.0038 }, { "threshold": 0.35, "macro_f1": 0.9714, "any_label_recall": 0.987, "any_label_fpr": 0.0038 }, { "threshold": 0.36, "macro_f1": 0.9713, "any_label_recall": 0.987, "any_label_fpr": 0.0038 }, { "threshold": 0.37, "macro_f1": 0.9713, "any_label_recall": 0.987, "any_label_fpr": 0.0038 }, { "threshold": 0.38, "macro_f1": 0.971, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.39, "macro_f1": 0.971, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.4, "macro_f1": 0.971, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.41, "macro_f1": 0.971, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.42, "macro_f1": 0.971, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.43, "macro_f1": 0.9706, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.44, "macro_f1": 0.9706, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.45, "macro_f1": 0.971, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.46, "macro_f1": 0.971, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.47, "macro_f1": 0.9704, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.48, "macro_f1": 0.9704, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.49, "macro_f1": 0.9704, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.5, "macro_f1": 0.9704, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.51, "macro_f1": 0.9704, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.52, "macro_f1": 0.9704, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.53, "macro_f1": 0.9704, "any_label_recall": 0.9861, "any_label_fpr": 0.0038 }, { "threshold": 0.54, "macro_f1": 0.9707, "any_label_recall": 0.9861, "any_label_fpr": 0.0035 }, { "threshold": 0.55, "macro_f1": 0.9707, "any_label_recall": 0.9861, "any_label_fpr": 0.0035 }, { "threshold": 0.56, "macro_f1": 0.9707, "any_label_recall": 0.9861, "any_label_fpr": 0.0035 }, { "threshold": 0.57, "macro_f1": 0.9707, "any_label_recall": 0.9861, "any_label_fpr": 0.0035 }, { "threshold": 0.58, "macro_f1": 0.9702, "any_label_recall": 0.9852, "any_label_fpr": 0.0035 }, { "threshold": 0.59, "macro_f1": 0.9702, "any_label_recall": 0.9852, "any_label_fpr": 0.0035 }, { "threshold": 0.6, "macro_f1": 0.9705, "any_label_recall": 0.9843, "any_label_fpr": 0.0035 }, { "threshold": 0.61, "macro_f1": 0.9705, "any_label_recall": 0.9843, "any_label_fpr": 0.0035 }, { "threshold": 0.62, "macro_f1": 0.9705, "any_label_recall": 0.9843, "any_label_fpr": 0.0035 }, { "threshold": 0.63, "macro_f1": 0.9705, "any_label_recall": 0.9843, "any_label_fpr": 0.0035 }, { "threshold": 0.64, "macro_f1": 0.9705, "any_label_recall": 0.9843, "any_label_fpr": 0.0035 }, { "threshold": 0.65, "macro_f1": 0.9709, "any_label_recall": 0.9843, "any_label_fpr": 0.0035 }, { "threshold": 0.66, "macro_f1": 0.9709, "any_label_recall": 0.9843, "any_label_fpr": 0.0035 }, { "threshold": 0.67, "macro_f1": 0.9712, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.68, "macro_f1": 0.9712, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.69, "macro_f1": 0.9716, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.7, "macro_f1": 0.9712, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.71, "macro_f1": 0.9712, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.72, "macro_f1": 0.9708, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.73, "macro_f1": 0.9708, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.74, "macro_f1": 0.9708, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.75, "macro_f1": 0.9708, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.76, "macro_f1": 0.9708, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.77, "macro_f1": 0.9708, "any_label_recall": 0.9843, "any_label_fpr": 0.0032 }, { "threshold": 0.78, "macro_f1": 0.9704, "any_label_recall": 0.9833, "any_label_fpr": 0.0032 }, { "threshold": 0.79, "macro_f1": 0.9704, "any_label_recall": 0.9833, "any_label_fpr": 0.0032 }, { "threshold": 0.8, "macro_f1": 0.9704, "any_label_recall": 0.9833, "any_label_fpr": 0.0032 }, { "threshold": 0.81, "macro_f1": 0.9708, "any_label_recall": 0.9833, "any_label_fpr": 0.0029 }, { "threshold": 0.82, "macro_f1": 0.9708, "any_label_recall": 0.9833, "any_label_fpr": 0.0029 }, { "threshold": 0.83, "macro_f1": 0.9704, "any_label_recall": 0.9833, "any_label_fpr": 0.0029 }, { "threshold": 0.84, "macro_f1": 0.971, "any_label_recall": 0.9833, "any_label_fpr": 0.0029 }, { "threshold": 0.85, "macro_f1": 0.971, "any_label_recall": 0.9833, "any_label_fpr": 0.0029 }, { "threshold": 0.86, "macro_f1": 0.9706, "any_label_recall": 0.9833, "any_label_fpr": 0.0029 }, { "threshold": 0.87, "macro_f1": 0.9696, "any_label_recall": 0.9824, "any_label_fpr": 0.0029 }, { "threshold": 0.88, "macro_f1": 0.9693, "any_label_recall": 0.9815, "any_label_fpr": 0.0029 }, { "threshold": 0.89, "macro_f1": 0.9692, "any_label_recall": 0.9815, "any_label_fpr": 0.0026 }, { "threshold": 0.9, "macro_f1": 0.9692, "any_label_recall": 0.9815, "any_label_fpr": 0.0026 }, { "threshold": 0.91, "macro_f1": 0.9692, "any_label_recall": 0.9806, "any_label_fpr": 0.0023 }, { "threshold": 0.92, "macro_f1": 0.9692, "any_label_recall": 0.9806, "any_label_fpr": 0.0023 }, { "threshold": 0.93, "macro_f1": 0.9688, "any_label_recall": 0.9806, "any_label_fpr": 0.0023 }, { "threshold": 0.94, "macro_f1": 0.9688, "any_label_recall": 0.9806, "any_label_fpr": 0.0023 }, { "threshold": 0.95, "macro_f1": 0.9688, "any_label_recall": 0.9806, "any_label_fpr": 0.0023 } ], "notes": [ "the chosen threshold 0.02 is the lowest value in the sweep, so the scores are compressed toward zero and this is underfitting rather than a calibration success. Check average precision before trusting the F1." ] } }