{ "model_path": "/Users/hwp/Documents/trae_projects/intelligent_cabin/models/local_joint_bert_nlu", "test_path": "/Users/hwp/Documents/trae_projects/intelligent_cabin/app/data/joint_nlu_eval_independent.jsonl", "sample_count": 43, "metrics": { "intent_accuracy": 0.9302, "slot_exact_match": 1.0, "joint_exact_match": 0.9302, "slot_micro_precision": 1.0, "slot_micro_recall": 1.0, "slot_micro_f1": 1.0 }, "per_category": [ { "category": "failure_replay", "sample_count": 12, "metrics": { "intent_accuracy": 0.75, "slot_exact_match": 1.0, "joint_exact_match": 0.75, "slot_micro_precision": 0.0, "slot_micro_recall": 0.0, "slot_micro_f1": 0.0 } }, { "category": "no_slot_control", "sample_count": 14, "metrics": { "intent_accuracy": 1.0, "slot_exact_match": 1.0, "joint_exact_match": 1.0, "slot_micro_precision": 0.0, "slot_micro_recall": 0.0, "slot_micro_f1": 0.0 } }, { "category": "slot_destination", "sample_count": 4, "metrics": { "intent_accuracy": 1.0, "slot_exact_match": 1.0, "joint_exact_match": 1.0, "slot_micro_precision": 1.0, "slot_micro_recall": 1.0, "slot_micro_f1": 1.0 } }, { "category": "slot_music", "sample_count": 5, "metrics": { "intent_accuracy": 1.0, "slot_exact_match": 1.0, "joint_exact_match": 1.0, "slot_micro_precision": 1.0, "slot_micro_recall": 1.0, "slot_micro_f1": 1.0 } }, { "category": "slot_order", "sample_count": 4, "metrics": { "intent_accuracy": 1.0, "slot_exact_match": 1.0, "joint_exact_match": 1.0, "slot_micro_precision": 1.0, "slot_micro_recall": 1.0, "slot_micro_f1": 1.0 } }, { "category": "slot_temperature", "sample_count": 4, "metrics": { "intent_accuracy": 1.0, "slot_exact_match": 1.0, "joint_exact_match": 1.0, "slot_micro_precision": 1.0, "slot_micro_recall": 1.0, "slot_micro_f1": 1.0 } } ], "top_confusions": [ { "expected": "cabin_window_open", "predicted": "None", "count": 1 }, { "expected": "cabin_window_open", "predicted": "cabin_play_music", "count": 1 }, { "expected": "cabin_fan_up", "predicted": "cabin_fan_down", "count": 1 } ], "failure_examples": [ { "text": "把左前窗降一点", "category": "failure_replay", "expected_intent_id": "cabin_window_open", "predicted_intent_id": "None", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.2812, "intent_ok": false, "slot_exact": true, "joint_ok": false, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_defog_front_on", "score": 0.2812 }, { "intent_id": "cabin_window_open", "score": 0.1715 }, { "intent_id": "cabin_mirror_unfold", "score": 0.0986 } ] }, { "text": "给我透个气", "category": "failure_replay", "expected_intent_id": "cabin_window_open", "predicted_intent_id": "cabin_play_music", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9331, "intent_ok": false, "slot_exact": true, "joint_ok": false, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_play_music", "score": 0.9331 }, { "intent_id": "cabin_next_track", "score": 0.0138 }, { "intent_id": "cabin_pause_music", "score": 0.0031 } ] }, { "text": "风别太小", "category": "failure_replay", "expected_intent_id": "cabin_fan_up", "predicted_intent_id": "cabin_fan_down", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.6053, "intent_ok": false, "slot_exact": true, "joint_ok": false, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_fan_down", "score": 0.6053 }, { "intent_id": "cabin_fan_up", "score": 0.1549 }, { "intent_id": "cabin_volume_down", "score": 0.0286 } ] } ], "train_summary": { "train_size": 337, "eval_size": 10, "metrics": { "intent_accuracy": 1.0, "slot_token_accuracy": 0.9412, "slot_exact_match": 0.8, "intent_label_count": 34.0, "slot_label_count": 11.0, "train_loss": 0.2596 } }, "results": [ { "text": "把空调调到22度", "category": "slot_temperature", "expected_intent_id": "cabin_set_ac", "predicted_intent_id": "cabin_set_ac", "expected_slots": { "temperature": 22 }, "predicted_slots": { "temperature": 22 }, "intent_score": 0.9879, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_set_ac", "score": 0.9879 }, { "intent_id": "cabin_fan_up", "score": 0.0017 }, { "intent_id": "cabin_fan_down", "score": 0.0011 } ] }, { "text": "空调给我调到20度", "category": "slot_temperature", "expected_intent_id": "cabin_set_ac", "predicted_intent_id": "cabin_set_ac", "expected_slots": { "temperature": 20 }, "predicted_slots": { "temperature": 20 }, "intent_score": 0.9897, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_set_ac", "score": 0.9897 }, { "intent_id": "cabin_fan_up", "score": 0.0014 }, { "intent_id": "cabin_fan_down", "score": 0.0011 } ] }, { "text": "车里温度设成24度", "category": "slot_temperature", "expected_intent_id": "cabin_set_ac", "predicted_intent_id": "cabin_set_ac", "expected_slots": { "temperature": 24 }, "predicted_slots": { "temperature": 24 }, "intent_score": 0.9908, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_set_ac", "score": 0.9908 }, { "intent_id": "cabin_fan_up", "score": 0.0009 }, { "intent_id": "cabin_fan_down", "score": 0.0006 } ] }, { "text": "把温度打到21度", "category": "slot_temperature", "expected_intent_id": "cabin_set_ac", "predicted_intent_id": "cabin_set_ac", "expected_slots": { "temperature": 21 }, "predicted_slots": { "temperature": 21 }, "intent_score": 0.9886, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_set_ac", "score": 0.9886 }, { "intent_id": "cabin_fan_up", "score": 0.0011 }, { "intent_id": "cabin_nav_to", "score": 0.0008 } ] }, { "text": "导航去公司停车场", "category": "slot_destination", "expected_intent_id": "cabin_nav_to", "predicted_intent_id": "cabin_nav_to", "expected_slots": { "destination": "公司停车场" }, "predicted_slots": { "destination": "公司停车场" }, "intent_score": 0.9858, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_nav_to", "score": 0.9858 }, { "intent_id": "cabin_nav_cancel", "score": 0.0018 }, { "intent_id": "cabin_lights_on", "score": 0.001 } ] }, { "text": "带我去浦东机场", "category": "slot_destination", "expected_intent_id": "cabin_nav_to", "predicted_intent_id": "cabin_nav_to", "expected_slots": { "destination": "浦东机场" }, "predicted_slots": { "destination": "浦东机场" }, "intent_score": 0.9893, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_nav_to", "score": 0.9893 }, { "intent_id": "cs_query_logistics", "score": 0.0008 }, { "intent_id": "cabin_play_music", "score": 0.0007 } ] }, { "text": "导航到南京东路", "category": "slot_destination", "expected_intent_id": "cabin_nav_to", "predicted_intent_id": "cabin_nav_to", "expected_slots": { "destination": "南京东路" }, "predicted_slots": { "destination": "南京东路" }, "intent_score": 0.9893, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_nav_to", "score": 0.9893 }, { "intent_id": "cabin_nav_cancel", "score": 0.0012 }, { "intent_id": "cabin_play_music", "score": 0.0007 } ] }, { "text": "去虹桥机场", "category": "slot_destination", "expected_intent_id": "cabin_nav_to", "predicted_intent_id": "cabin_nav_to", "expected_slots": { "destination": "虹桥机场" }, "predicted_slots": { "destination": "虹桥机场" }, "intent_score": 0.9862, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_nav_to", "score": 0.9862 }, { "intent_id": "cabin_nav_cancel", "score": 0.0012 }, { "intent_id": "cabin_play_music", "score": 0.0011 } ] }, { "text": "查一下订单A123456", "category": "slot_order", "expected_intent_id": "cs_query_order", "predicted_intent_id": "cs_query_order", "expected_slots": { "order_id": "A123456" }, "predicted_slots": { "order_id": "A123456" }, "intent_score": 0.9921, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cs_query_order", "score": 0.9921 }, { "intent_id": "cs_cancel_order", "score": 0.001 }, { "intent_id": "cs_query_logistics", "score": 0.0007 } ] }, { "text": "帮我看看订单A808001", "category": "slot_order", "expected_intent_id": "cs_query_order", "predicted_intent_id": "cs_query_order", "expected_slots": { "order_id": "A808001" }, "predicted_slots": { "order_id": "A808001" }, "intent_score": 0.9922, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cs_query_order", "score": 0.9922 }, { "intent_id": "cs_cancel_order", "score": 0.0013 }, { "intent_id": "cs_query_logistics", "score": 0.0007 } ] }, { "text": "快递A998877到哪了", "category": "slot_order", "expected_intent_id": "cs_query_logistics", "predicted_intent_id": "cs_query_logistics", "expected_slots": { "order_id": "A998877" }, "predicted_slots": { "order_id": "A998877" }, "intent_score": 0.9931, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cs_query_logistics", "score": 0.9931 }, { "intent_id": "cabin_volume_mute", "score": 0.0005 }, { "intent_id": "cabin_defog_rear_on", "score": 0.0005 } ] }, { "text": "取消订单A556677", "category": "slot_order", "expected_intent_id": "cs_cancel_order", "predicted_intent_id": "cs_cancel_order", "expected_slots": { "order_id": "A556677" }, "predicted_slots": { "order_id": "A556677" }, "intent_score": 0.9911, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cs_cancel_order", "score": 0.9911 }, { "intent_id": "cabin_pause_music", "score": 0.001 }, { "intent_id": "cs_query_order", "score": 0.0007 } ] }, { "text": "来一首青花瓷", "category": "slot_music", "expected_intent_id": "cabin_play_music", "predicted_intent_id": "cabin_play_music", "expected_slots": { "song": "青花瓷" }, "predicted_slots": { "song": "青花瓷" }, "intent_score": 0.9881, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_play_music", "score": 0.9881 }, { "intent_id": "cabin_next_track", "score": 0.002 }, { "intent_id": "cabin_pause_music", "score": 0.0005 } ] }, { "text": "播放夜的第七章", "category": "slot_music", "expected_intent_id": "cabin_play_music", "predicted_intent_id": "cabin_play_music", "expected_slots": { "song": "夜的第七章" }, "predicted_slots": { "song": "夜的第七章" }, "intent_score": 0.9828, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_play_music", "score": 0.9828 }, { "intent_id": "cabin_nav_to", "score": 0.0015 }, { "intent_id": "cabin_pause_music", "score": 0.0012 } ] }, { "text": "来点爵士", "category": "slot_music", "expected_intent_id": "cabin_play_music", "predicted_intent_id": "cabin_play_music", "expected_slots": { "genre": "爵士" }, "predicted_slots": { "genre": "爵士" }, "intent_score": 0.9908, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_play_music", "score": 0.9908 }, { "intent_id": "cabin_next_track", "score": 0.0007 }, { "intent_id": "cabin_pause_music", "score": 0.0006 } ] }, { "text": "放点摇滚", "category": "slot_music", "expected_intent_id": "cabin_play_music", "predicted_intent_id": "cabin_play_music", "expected_slots": { "genre": "摇滚" }, "predicted_slots": { "genre": "摇滚" }, "intent_score": 0.9915, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_play_music", "score": 0.9915 }, { "intent_id": "cabin_pause_music", "score": 0.0006 }, { "intent_id": "cabin_next_track", "score": 0.0006 } ] }, { "text": "给我播点民谣", "category": "slot_music", "expected_intent_id": "cabin_play_music", "predicted_intent_id": "cabin_play_music", "expected_slots": { "genre": "民谣" }, "predicted_slots": { "genre": "民谣" }, "intent_score": 0.9917, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 1, "exact": true }, "top_candidates": [ { "intent_id": "cabin_play_music", "score": 0.9917 }, { "intent_id": "cabin_next_track", "score": 0.0007 }, { "intent_id": "cabin_sunroof_open", "score": 0.0005 } ] }, { "text": "把车窗打开", "category": "no_slot_control", "expected_intent_id": "cabin_window_open", "predicted_intent_id": "cabin_window_open", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9474, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_window_open", "score": 0.9474 }, { "intent_id": "cabin_window_close", "score": 0.0074 }, { "intent_id": "cabin_sunroof_open", "score": 0.0053 } ] }, { "text": "把车窗关上", "category": "no_slot_control", "expected_intent_id": "cabin_window_close", "predicted_intent_id": "cabin_window_close", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9544, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_window_close", "score": 0.9544 }, { "intent_id": "cabin_lock_doors", "score": 0.0088 }, { "intent_id": "cabin_window_open", "score": 0.0065 } ] }, { "text": "把天窗打开", "category": "no_slot_control", "expected_intent_id": "cabin_sunroof_open", "predicted_intent_id": "cabin_sunroof_open", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9715, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_sunroof_open", "score": 0.9715 }, { "intent_id": "cabin_sunroof_close", "score": 0.0032 }, { "intent_id": "cabin_wiper_on", "score": 0.0018 } ] }, { "text": "把天窗合上", "category": "no_slot_control", "expected_intent_id": "cabin_sunroof_close", "predicted_intent_id": "cabin_sunroof_close", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.7244, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_sunroof_close", "score": 0.7244 }, { "intent_id": "cabin_sunroof_open", "score": 0.204 }, { "intent_id": "cabin_mirror_unfold", "score": 0.008 } ] }, { "text": "把空调打开", "category": "no_slot_control", "expected_intent_id": "cabin_ac_on", "predicted_intent_id": "cabin_ac_on", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9545, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_ac_on", "score": 0.9545 }, { "intent_id": "cabin_ac_off", "score": 0.0079 }, { "intent_id": "cabin_sunroof_open", "score": 0.0029 } ] }, { "text": "把空调关掉", "category": "no_slot_control", "expected_intent_id": "cabin_ac_off", "predicted_intent_id": "cabin_ac_off", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9673, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_ac_off", "score": 0.9673 }, { "intent_id": "cabin_ac_on", "score": 0.0035 }, { "intent_id": "cabin_lights_off", "score": 0.0032 } ] }, { "text": "把风量调大一点", "category": "no_slot_control", "expected_intent_id": "cabin_fan_up", "predicted_intent_id": "cabin_fan_up", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.5975, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_fan_up", "score": 0.5975 }, { "intent_id": "cabin_fan_down", "score": 0.3425 }, { "intent_id": "cabin_volume_up", "score": 0.0044 } ] }, { "text": "风太大了，往小调一点", "category": "no_slot_control", "expected_intent_id": "cabin_fan_down", "predicted_intent_id": "cabin_fan_down", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.6892, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_fan_down", "score": 0.6892 }, { "intent_id": "cabin_fan_up", "score": 0.2052 }, { "intent_id": "cabin_volume_down", "score": 0.014 } ] }, { "text": "把音乐暂停", "category": "no_slot_control", "expected_intent_id": "cabin_pause_music", "predicted_intent_id": "cabin_pause_music", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9672, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_pause_music", "score": 0.9672 }, { "intent_id": "cabin_volume_mute", "score": 0.0039 }, { "intent_id": "cabin_next_track", "score": 0.0025 } ] }, { "text": "帮我切到下一首", "category": "no_slot_control", "expected_intent_id": "cabin_next_track", "predicted_intent_id": "cabin_next_track", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9667, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_next_track", "score": 0.9667 }, { "intent_id": "cabin_previous_track", "score": 0.0104 }, { "intent_id": "cabin_play_music", "score": 0.0022 } ] }, { "text": "切回上一首", "category": "no_slot_control", "expected_intent_id": "cabin_previous_track", "predicted_intent_id": "cabin_previous_track", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9694, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_previous_track", "score": 0.9694 }, { "intent_id": "cabin_next_track", "score": 0.0096 }, { "intent_id": "cabin_nav_cancel", "score": 0.0018 } ] }, { "text": "先静音", "category": "no_slot_control", "expected_intent_id": "cabin_volume_mute", "predicted_intent_id": "cabin_volume_mute", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9763, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_volume_mute", "score": 0.9763 }, { "intent_id": "cabin_pause_music", "score": 0.0022 }, { "intent_id": "cs_query_logistics", "score": 0.0016 } ] }, { "text": "把音量调大一点", "category": "no_slot_control", "expected_intent_id": "cabin_volume_up", "predicted_intent_id": "cabin_volume_up", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.6867, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_volume_up", "score": 0.6867 }, { "intent_id": "cabin_volume_down", "score": 0.2533 }, { "intent_id": "cabin_fan_up", "score": 0.0067 } ] }, { "text": "把音量调小一点", "category": "no_slot_control", "expected_intent_id": "cabin_volume_down", "predicted_intent_id": "cabin_volume_down", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.6048, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_volume_down", "score": 0.6048 }, { "intent_id": "cabin_volume_up", "score": 0.327 }, { "intent_id": "cabin_fan_down", "score": 0.009 } ] }, { "text": "把后视镜收起来", "category": "failure_replay", "expected_intent_id": "cabin_mirror_fold", "predicted_intent_id": "cabin_mirror_fold", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9329, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_mirror_fold", "score": 0.9329 }, { "intent_id": "cabin_mirror_unfold", "score": 0.0315 }, { "intent_id": "cabin_wiper_off", "score": 0.0046 } ] }, { "text": "把镜子展开", "category": "failure_replay", "expected_intent_id": "cabin_mirror_unfold", "predicted_intent_id": "cabin_mirror_unfold", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.7236, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_mirror_unfold", "score": 0.7236 }, { "intent_id": "cabin_play_music", "score": 0.025 }, { "intent_id": "cabin_mirror_fold", "score": 0.0239 } ] }, { "text": "锁车门", "category": "failure_replay", "expected_intent_id": "cabin_lock_doors", "predicted_intent_id": "cabin_lock_doors", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9345, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_lock_doors", "score": 0.9345 }, { "intent_id": "cabin_unlock_doors", "score": 0.0328 }, { "intent_id": "cabin_window_close", "score": 0.0061 } ] }, { "text": "把车门解锁", "category": "failure_replay", "expected_intent_id": "cabin_unlock_doors", "predicted_intent_id": "cabin_unlock_doors", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9194, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_unlock_doors", "score": 0.9194 }, { "intent_id": "cabin_lock_doors", "score": 0.0488 }, { "intent_id": "cabin_seat_heat_off", "score": 0.0032 } ] }, { "text": "路线别导了", "category": "failure_replay", "expected_intent_id": "cabin_nav_cancel", "predicted_intent_id": "cabin_nav_cancel", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.6978, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_nav_cancel", "score": 0.6978 }, { "intent_id": "cabin_pause_music", "score": 0.069 }, { "intent_id": "cs_cancel_order", "score": 0.0353 } ] }, { "text": "音乐停一下", "category": "failure_replay", "expected_intent_id": "cabin_pause_music", "predicted_intent_id": "cabin_pause_music", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9077, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_pause_music", "score": 0.9077 }, { "intent_id": "cabin_volume_mute", "score": 0.012 }, { "intent_id": "cabin_lights_off", "score": 0.009 } ] }, { "text": "雨刮关掉", "category": "failure_replay", "expected_intent_id": "cabin_wiper_off", "predicted_intent_id": "cabin_wiper_off", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9638, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_wiper_off", "score": 0.9638 }, { "intent_id": "cabin_wiper_on", "score": 0.0044 }, { "intent_id": "cabin_lights_off", "score": 0.0033 } ] }, { "text": "把雨刮打开", "category": "failure_replay", "expected_intent_id": "cabin_wiper_on", "predicted_intent_id": "cabin_wiper_on", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9066, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_wiper_on", "score": 0.9066 }, { "intent_id": "cabin_wiper_off", "score": 0.0363 }, { "intent_id": "cabin_sunroof_open", "score": 0.0069 } ] }, { "text": "把左前窗降一点", "category": "failure_replay", "expected_intent_id": "cabin_window_open", "predicted_intent_id": "None", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.2812, "intent_ok": false, "slot_exact": true, "joint_ok": false, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_defog_front_on", "score": 0.2812 }, { "intent_id": "cabin_window_open", "score": 0.1715 }, { "intent_id": "cabin_mirror_unfold", "score": 0.0986 } ] }, { "text": "给我透个气", "category": "failure_replay", "expected_intent_id": "cabin_window_open", "predicted_intent_id": "cabin_play_music", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9331, "intent_ok": false, "slot_exact": true, "joint_ok": false, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_play_music", "score": 0.9331 }, { "intent_id": "cabin_next_track", "score": 0.0138 }, { "intent_id": "cabin_pause_music", "score": 0.0031 } ] }, { "text": "风别太小", "category": "failure_replay", "expected_intent_id": "cabin_fan_up", "predicted_intent_id": "cabin_fan_down", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.6053, "intent_ok": false, "slot_exact": true, "joint_ok": false, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cabin_fan_down", "score": 0.6053 }, { "intent_id": "cabin_fan_up", "score": 0.1549 }, { "intent_id": "cabin_volume_down", "score": 0.0286 } ] }, { "text": "要是太慢就转人工", "category": "failure_replay", "expected_intent_id": "cs_transfer_human", "predicted_intent_id": "cs_transfer_human", "expected_slots": {}, "predicted_slots": {}, "intent_score": 0.9849, "intent_ok": true, "slot_exact": true, "joint_ok": true, "slot_diff": { "missing_keys": [], "extra_keys": [], "wrong_values": [], "matched_keys": 0, "exact": true }, "top_candidates": [ { "intent_id": "cs_transfer_human", "score": 0.9849 }, { "intent_id": "cs_query_logistics", "score": 0.0018 }, { "intent_id": "cs_cancel_order", "score": 0.0016 } ] } ] }