{
  "model_dir": "/Users/hwp/Documents/trae_projects/intelligent_cabin/models/local_bert_intent",
  "threshold": 0.0,
  "test_path": "/Users/hwp/Documents/trae_projects/intelligent_cabin/app/data/bert_intent_eval_independent.jsonl",
  "test_case_count": 42,
  "accuracy": 0.9762,
  "train_summary": {
    "base_model": "hfl/chinese-macbert-base",
    "epochs": 16,
    "batch_size": 8,
    "learning_rate": 2e-05,
    "train_size": 1557,
    "dev_size": 401,
    "best_dev_accuracy": 0.9875,
    "device": "mps"
  },
  "per_category": [
    {
      "category": "business",
      "total": 34,
      "correct": 33,
      "accuracy": 0.9706
    },
    {
      "category": "out_of_scope",
      "total": 4,
      "correct": 4,
      "accuracy": 1.0
    },
    {
      "category": "social",
      "total": 4,
      "correct": 4,
      "accuracy": 1.0
    }
  ],
  "per_label": [
    {
      "label": "__out_of_scope__",
      "category": "out_of_scope",
      "total": 4,
      "correct": 4,
      "accuracy": 1.0
    },
    {
      "label": "__social__",
      "category": "social",
      "total": 4,
      "correct": 4,
      "accuracy": 1.0
    },
    {
      "label": "cabin_ac_off",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_ac_on",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_defog_front_on",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_defog_rear_on",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_fan_down",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_fan_up",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_lights_off",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_lights_on",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_lock_doors",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_mirror_fold",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_mirror_unfold",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_nav_cancel",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_nav_to",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_next_track",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_pause_music",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_play_music",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_previous_track",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_seat_heat_off",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_seat_heat_on",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_set_ac",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_sunroof_close",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_sunroof_open",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_unlock_doors",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_volume_down",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_volume_mute",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_volume_up",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_window_close",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_window_open",
      "category": "business",
      "total": 1,
      "correct": 0,
      "accuracy": 0.0
    },
    {
      "label": "cabin_wiper_off",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cabin_wiper_on",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cs_cancel_order",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cs_query_logistics",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cs_query_order",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    },
    {
      "label": "cs_transfer_human",
      "category": "business",
      "total": 1,
      "correct": 1,
      "accuracy": 1.0
    }
  ],
  "errors": [
    {
      "text": "左前窗打开一点",
      "category": "business",
      "expected_label": "cabin_window_open",
      "predicted_label": "cabin_defog_front_on",
      "score": 0.9951,
      "raw_label": "cabin_defog_front_on",
      "ok": false,
      "top_candidates": [
        {
          "intent_id": "cabin_defog_front_on",
          "score": 0.9951
        },
        {
          "intent_id": "cabin_sunroof_open",
          "score": 0.0005
        },
        {
          "intent_id": "cabin_lights_on",
          "score": 0.0004
        }
      ]
    }
  ],
  "confusion": {
    "cabin_ac_off": {
      "cabin_ac_off": 1
    },
    "cabin_ac_on": {
      "cabin_ac_on": 1
    },
    "cabin_defog_front_on": {
      "cabin_defog_front_on": 1
    },
    "cabin_defog_rear_on": {
      "cabin_defog_rear_on": 1
    },
    "cabin_fan_down": {
      "cabin_fan_down": 1
    },
    "cabin_fan_up": {
      "cabin_fan_up": 1
    },
    "cabin_lights_off": {
      "cabin_lights_off": 1
    },
    "cabin_lights_on": {
      "cabin_lights_on": 1
    },
    "cabin_lock_doors": {
      "cabin_lock_doors": 1
    },
    "cabin_mirror_fold": {
      "cabin_mirror_fold": 1
    },
    "cabin_mirror_unfold": {
      "cabin_mirror_unfold": 1
    },
    "cabin_nav_cancel": {
      "cabin_nav_cancel": 1
    },
    "cabin_nav_to": {
      "cabin_nav_to": 1
    },
    "cabin_next_track": {
      "cabin_next_track": 1
    },
    "cabin_pause_music": {
      "cabin_pause_music": 1
    },
    "cabin_play_music": {
      "cabin_play_music": 1
    },
    "cabin_previous_track": {
      "cabin_previous_track": 1
    },
    "cabin_seat_heat_off": {
      "cabin_seat_heat_off": 1
    },
    "cabin_seat_heat_on": {
      "cabin_seat_heat_on": 1
    },
    "cabin_set_ac": {
      "cabin_set_ac": 1
    },
    "cabin_sunroof_close": {
      "cabin_sunroof_close": 1
    },
    "cabin_sunroof_open": {
      "cabin_sunroof_open": 1
    },
    "cabin_unlock_doors": {
      "cabin_unlock_doors": 1
    },
    "cabin_volume_down": {
      "cabin_volume_down": 1
    },
    "cabin_volume_mute": {
      "cabin_volume_mute": 1
    },
    "cabin_volume_up": {
      "cabin_volume_up": 1
    },
    "cabin_window_close": {
      "cabin_window_close": 1
    },
    "cabin_window_open": {
      "cabin_defog_front_on": 1
    },
    "cabin_wiper_off": {
      "cabin_wiper_off": 1
    },
    "cabin_wiper_on": {
      "cabin_wiper_on": 1
    },
    "cs_cancel_order": {
      "cs_cancel_order": 1
    },
    "cs_query_logistics": {
      "cs_query_logistics": 1
    },
    "cs_query_order": {
      "cs_query_order": 1
    },
    "cs_transfer_human": {
      "cs_transfer_human": 1
    },
    "__social__": {
      "__social__": 4
    },
    "__out_of_scope__": {
      "__out_of_scope__": 4
    }
  }
}