Upload eval/topic_scope_eval.json with huggingface_hub

eval/topic_scope_eval.json

@@ -0,0 +1,153 @@
+{
+  "measured_on": "the shipped INT8 artifact rather than the torch checkpoint",
+  "not_evaluated": "the 78 out_of_taxonomy rows in the test split are generated and never scored, here or in train_embed.py, so nothing measures whether this detector rejects input that belongs to no node, which is half of what it is for",
+  "per_language": {
+    "az": {
+      "accuracy": 0.8571,
+      "n": 7
+    },
+    "bg": {
+      "accuracy": 0.8571,
+      "n": 7
+    },
+    "cs": {
+      "accuracy": 1.0,
+      "n": 6
+    },
+    "da": {
+      "accuracy": 0.8571,
+      "n": 7
+    },
+    "de": {
+      "accuracy": 1.0,
+      "n": 7
+    },
+    "el": {
+      "accuracy": 0.8571,
+      "n": 7
+    },
+    "en": {
+      "accuracy": 1.0,
+      "n": 6
+    },
+    "es": {
+      "accuracy": 1.0,
+      "n": 7
+    },
+    "et": {
+      "accuracy": 1.0,
+      "n": 7
+    },
+    "fi": {
+      "accuracy": 0.8333,
+      "n": 6
+    },
+    "fr": {
+      "accuracy": 1.0,
+      "n": 6
+    },
+    "ga": {
+      "accuracy": 0.375,
+      "n": 8
+    },
+    "hr": {
+      "accuracy": 1.0,
+      "n": 6
+    },
+    "hu": {
+      "accuracy": 0.625,
+      "n": 8
+    },
+    "it": {
+      "accuracy": 0.8333,
+      "n": 6
+    },
+    "lt": {
+      "accuracy": 0.7143,
+      "n": 7
+    },
+    "lv": {
+      "accuracy": 1.0,
+      "n": 7
+    },
+    "mt": {
+      "accuracy": 0.5,
+      "n": 8
+    },
+    "nl": {
+      "accuracy": 0.8333,
+      "n": 6
+    },
+    "pl": {
+      "accuracy": 0.8333,
+      "n": 6
+    },
+    "pt": {
+      "accuracy": 1.0,
+      "n": 7
+    },
+    "ro": {
+      "accuracy": 0.8333,
+      "n": 6
+    },
+    "sk": {
+      "accuracy": 1.0,
+      "n": 7
+    },
+    "sl": {
+      "accuracy": 0.8571,
+      "n": 7
+    },
+    "sv": {
+      "accuracy": 0.8333,
+      "n": 6
+    },
+    "tr": {
+      "accuracy": 1.0,
+      "n": 7
+    }
+  },
+  "per_node": {
+    "banking/accounts/closing": 0.75,
+    "banking/accounts/opening": 0.6364,
+    "banking/cards/limits": 1.0,
+    "banking/cards/lost_or_stolen": 0.9091,
+    "banking/loans/consumer": 0.6667,
+    "banking/loans/mortgage": 0.9474,
+    "banking/payments/failed": 0.9286,
+    "banking/payments/transfer": 0.7692,
+    "health/appointments": 1.0,
+    "health/prescriptions": 0.7778,
+    "insurance/claims/home": 1.0,
+    "insurance/claims/motor": 0.9231,
+    "insurance/policy/renewal": 0.8182,
+    "public/identity/documents": 0.9286,
+    "public/tax/filing": 0.8
+  },
+  "sibling_near_miss_n": 155,
+  "sibling_rejection_rate": 0.9161,
+  "top1_accuracy": 0.8571,
+  "total_examples": 175,
+  "weakest_nodes": [
+    [
+      "banking/accounts/opening",
+      0.6364
+    ],
+    [
+      "banking/loans/consumer",
+      0.6667
+    ],
+    [
+      "banking/accounts/closing",
+      0.75
+    ],
+    [
+      "banking/payments/transfer",
+      0.7692
+    ],
+    [
+      "health/prescriptions",
+      0.7778
+    ]
+  ]
+}