1706 lines
40 KiB
JSON
1706 lines
40 KiB
JSON
{
|
|
"model_path": "/Users/hwp/Documents/trae_projects/intelligent_cabin/models/local_joint_bert_nlu",
|
|
"test_path": "/Users/hwp/Documents/trae_projects/intelligent_cabin/app/data/joint_nlu_eval_independent.jsonl",
|
|
"sample_count": 43,
|
|
"metrics": {
|
|
"intent_accuracy": 0.9302,
|
|
"slot_exact_match": 1.0,
|
|
"joint_exact_match": 0.9302,
|
|
"slot_micro_precision": 1.0,
|
|
"slot_micro_recall": 1.0,
|
|
"slot_micro_f1": 1.0
|
|
},
|
|
"per_category": [
|
|
{
|
|
"category": "failure_replay",
|
|
"sample_count": 12,
|
|
"metrics": {
|
|
"intent_accuracy": 0.75,
|
|
"slot_exact_match": 1.0,
|
|
"joint_exact_match": 0.75,
|
|
"slot_micro_precision": 0.0,
|
|
"slot_micro_recall": 0.0,
|
|
"slot_micro_f1": 0.0
|
|
}
|
|
},
|
|
{
|
|
"category": "no_slot_control",
|
|
"sample_count": 14,
|
|
"metrics": {
|
|
"intent_accuracy": 1.0,
|
|
"slot_exact_match": 1.0,
|
|
"joint_exact_match": 1.0,
|
|
"slot_micro_precision": 0.0,
|
|
"slot_micro_recall": 0.0,
|
|
"slot_micro_f1": 0.0
|
|
}
|
|
},
|
|
{
|
|
"category": "slot_destination",
|
|
"sample_count": 4,
|
|
"metrics": {
|
|
"intent_accuracy": 1.0,
|
|
"slot_exact_match": 1.0,
|
|
"joint_exact_match": 1.0,
|
|
"slot_micro_precision": 1.0,
|
|
"slot_micro_recall": 1.0,
|
|
"slot_micro_f1": 1.0
|
|
}
|
|
},
|
|
{
|
|
"category": "slot_music",
|
|
"sample_count": 5,
|
|
"metrics": {
|
|
"intent_accuracy": 1.0,
|
|
"slot_exact_match": 1.0,
|
|
"joint_exact_match": 1.0,
|
|
"slot_micro_precision": 1.0,
|
|
"slot_micro_recall": 1.0,
|
|
"slot_micro_f1": 1.0
|
|
}
|
|
},
|
|
{
|
|
"category": "slot_order",
|
|
"sample_count": 4,
|
|
"metrics": {
|
|
"intent_accuracy": 1.0,
|
|
"slot_exact_match": 1.0,
|
|
"joint_exact_match": 1.0,
|
|
"slot_micro_precision": 1.0,
|
|
"slot_micro_recall": 1.0,
|
|
"slot_micro_f1": 1.0
|
|
}
|
|
},
|
|
{
|
|
"category": "slot_temperature",
|
|
"sample_count": 4,
|
|
"metrics": {
|
|
"intent_accuracy": 1.0,
|
|
"slot_exact_match": 1.0,
|
|
"joint_exact_match": 1.0,
|
|
"slot_micro_precision": 1.0,
|
|
"slot_micro_recall": 1.0,
|
|
"slot_micro_f1": 1.0
|
|
}
|
|
}
|
|
],
|
|
"top_confusions": [
|
|
{
|
|
"expected": "cabin_window_open",
|
|
"predicted": "None",
|
|
"count": 1
|
|
},
|
|
{
|
|
"expected": "cabin_window_open",
|
|
"predicted": "cabin_play_music",
|
|
"count": 1
|
|
},
|
|
{
|
|
"expected": "cabin_fan_up",
|
|
"predicted": "cabin_fan_down",
|
|
"count": 1
|
|
}
|
|
],
|
|
"failure_examples": [
|
|
{
|
|
"text": "把左前窗降一点",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_window_open",
|
|
"predicted_intent_id": "None",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.2812,
|
|
"intent_ok": false,
|
|
"slot_exact": true,
|
|
"joint_ok": false,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_defog_front_on",
|
|
"score": 0.2812
|
|
},
|
|
{
|
|
"intent_id": "cabin_window_open",
|
|
"score": 0.1715
|
|
},
|
|
{
|
|
"intent_id": "cabin_mirror_unfold",
|
|
"score": 0.0986
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "给我透个气",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_window_open",
|
|
"predicted_intent_id": "cabin_play_music",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9331,
|
|
"intent_ok": false,
|
|
"slot_exact": true,
|
|
"joint_ok": false,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.9331
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.0138
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.0031
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "风别太小",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_fan_up",
|
|
"predicted_intent_id": "cabin_fan_down",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.6053,
|
|
"intent_ok": false,
|
|
"slot_exact": true,
|
|
"joint_ok": false,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.6053
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.1549
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_down",
|
|
"score": 0.0286
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"train_summary": {
|
|
"train_size": 337,
|
|
"eval_size": 10,
|
|
"metrics": {
|
|
"intent_accuracy": 1.0,
|
|
"slot_token_accuracy": 0.9412,
|
|
"slot_exact_match": 0.8,
|
|
"intent_label_count": 34.0,
|
|
"slot_label_count": 11.0,
|
|
"train_loss": 0.2596
|
|
}
|
|
},
|
|
"results": [
|
|
{
|
|
"text": "把空调调到22度",
|
|
"category": "slot_temperature",
|
|
"expected_intent_id": "cabin_set_ac",
|
|
"predicted_intent_id": "cabin_set_ac",
|
|
"expected_slots": {
|
|
"temperature": 22
|
|
},
|
|
"predicted_slots": {
|
|
"temperature": 22
|
|
},
|
|
"intent_score": 0.9879,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_set_ac",
|
|
"score": 0.9879
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.0017
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.0011
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "空调给我调到20度",
|
|
"category": "slot_temperature",
|
|
"expected_intent_id": "cabin_set_ac",
|
|
"predicted_intent_id": "cabin_set_ac",
|
|
"expected_slots": {
|
|
"temperature": 20
|
|
},
|
|
"predicted_slots": {
|
|
"temperature": 20
|
|
},
|
|
"intent_score": 0.9897,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_set_ac",
|
|
"score": 0.9897
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.0014
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.0011
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "车里温度设成24度",
|
|
"category": "slot_temperature",
|
|
"expected_intent_id": "cabin_set_ac",
|
|
"predicted_intent_id": "cabin_set_ac",
|
|
"expected_slots": {
|
|
"temperature": 24
|
|
},
|
|
"predicted_slots": {
|
|
"temperature": 24
|
|
},
|
|
"intent_score": 0.9908,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_set_ac",
|
|
"score": 0.9908
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.0009
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.0006
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把温度打到21度",
|
|
"category": "slot_temperature",
|
|
"expected_intent_id": "cabin_set_ac",
|
|
"predicted_intent_id": "cabin_set_ac",
|
|
"expected_slots": {
|
|
"temperature": 21
|
|
},
|
|
"predicted_slots": {
|
|
"temperature": 21
|
|
},
|
|
"intent_score": 0.9886,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_set_ac",
|
|
"score": 0.9886
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.0011
|
|
},
|
|
{
|
|
"intent_id": "cabin_nav_to",
|
|
"score": 0.0008
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "导航去公司停车场",
|
|
"category": "slot_destination",
|
|
"expected_intent_id": "cabin_nav_to",
|
|
"predicted_intent_id": "cabin_nav_to",
|
|
"expected_slots": {
|
|
"destination": "公司停车场"
|
|
},
|
|
"predicted_slots": {
|
|
"destination": "公司停车场"
|
|
},
|
|
"intent_score": 0.9858,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_nav_to",
|
|
"score": 0.9858
|
|
},
|
|
{
|
|
"intent_id": "cabin_nav_cancel",
|
|
"score": 0.0018
|
|
},
|
|
{
|
|
"intent_id": "cabin_lights_on",
|
|
"score": 0.001
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "带我去浦东机场",
|
|
"category": "slot_destination",
|
|
"expected_intent_id": "cabin_nav_to",
|
|
"predicted_intent_id": "cabin_nav_to",
|
|
"expected_slots": {
|
|
"destination": "浦东机场"
|
|
},
|
|
"predicted_slots": {
|
|
"destination": "浦东机场"
|
|
},
|
|
"intent_score": 0.9893,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_nav_to",
|
|
"score": 0.9893
|
|
},
|
|
{
|
|
"intent_id": "cs_query_logistics",
|
|
"score": 0.0008
|
|
},
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.0007
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "导航到南京东路",
|
|
"category": "slot_destination",
|
|
"expected_intent_id": "cabin_nav_to",
|
|
"predicted_intent_id": "cabin_nav_to",
|
|
"expected_slots": {
|
|
"destination": "南京东路"
|
|
},
|
|
"predicted_slots": {
|
|
"destination": "南京东路"
|
|
},
|
|
"intent_score": 0.9893,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_nav_to",
|
|
"score": 0.9893
|
|
},
|
|
{
|
|
"intent_id": "cabin_nav_cancel",
|
|
"score": 0.0012
|
|
},
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.0007
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "去虹桥机场",
|
|
"category": "slot_destination",
|
|
"expected_intent_id": "cabin_nav_to",
|
|
"predicted_intent_id": "cabin_nav_to",
|
|
"expected_slots": {
|
|
"destination": "虹桥机场"
|
|
},
|
|
"predicted_slots": {
|
|
"destination": "虹桥机场"
|
|
},
|
|
"intent_score": 0.9862,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_nav_to",
|
|
"score": 0.9862
|
|
},
|
|
{
|
|
"intent_id": "cabin_nav_cancel",
|
|
"score": 0.0012
|
|
},
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.0011
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "查一下订单A123456",
|
|
"category": "slot_order",
|
|
"expected_intent_id": "cs_query_order",
|
|
"predicted_intent_id": "cs_query_order",
|
|
"expected_slots": {
|
|
"order_id": "A123456"
|
|
},
|
|
"predicted_slots": {
|
|
"order_id": "A123456"
|
|
},
|
|
"intent_score": 0.9921,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cs_query_order",
|
|
"score": 0.9921
|
|
},
|
|
{
|
|
"intent_id": "cs_cancel_order",
|
|
"score": 0.001
|
|
},
|
|
{
|
|
"intent_id": "cs_query_logistics",
|
|
"score": 0.0007
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "帮我看看订单A808001",
|
|
"category": "slot_order",
|
|
"expected_intent_id": "cs_query_order",
|
|
"predicted_intent_id": "cs_query_order",
|
|
"expected_slots": {
|
|
"order_id": "A808001"
|
|
},
|
|
"predicted_slots": {
|
|
"order_id": "A808001"
|
|
},
|
|
"intent_score": 0.9922,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cs_query_order",
|
|
"score": 0.9922
|
|
},
|
|
{
|
|
"intent_id": "cs_cancel_order",
|
|
"score": 0.0013
|
|
},
|
|
{
|
|
"intent_id": "cs_query_logistics",
|
|
"score": 0.0007
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "快递A998877到哪了",
|
|
"category": "slot_order",
|
|
"expected_intent_id": "cs_query_logistics",
|
|
"predicted_intent_id": "cs_query_logistics",
|
|
"expected_slots": {
|
|
"order_id": "A998877"
|
|
},
|
|
"predicted_slots": {
|
|
"order_id": "A998877"
|
|
},
|
|
"intent_score": 0.9931,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cs_query_logistics",
|
|
"score": 0.9931
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_mute",
|
|
"score": 0.0005
|
|
},
|
|
{
|
|
"intent_id": "cabin_defog_rear_on",
|
|
"score": 0.0005
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "取消订单A556677",
|
|
"category": "slot_order",
|
|
"expected_intent_id": "cs_cancel_order",
|
|
"predicted_intent_id": "cs_cancel_order",
|
|
"expected_slots": {
|
|
"order_id": "A556677"
|
|
},
|
|
"predicted_slots": {
|
|
"order_id": "A556677"
|
|
},
|
|
"intent_score": 0.9911,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cs_cancel_order",
|
|
"score": 0.9911
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.001
|
|
},
|
|
{
|
|
"intent_id": "cs_query_order",
|
|
"score": 0.0007
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "来一首青花瓷",
|
|
"category": "slot_music",
|
|
"expected_intent_id": "cabin_play_music",
|
|
"predicted_intent_id": "cabin_play_music",
|
|
"expected_slots": {
|
|
"song": "青花瓷"
|
|
},
|
|
"predicted_slots": {
|
|
"song": "青花瓷"
|
|
},
|
|
"intent_score": 0.9881,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.9881
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.002
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.0005
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "播放夜的第七章",
|
|
"category": "slot_music",
|
|
"expected_intent_id": "cabin_play_music",
|
|
"predicted_intent_id": "cabin_play_music",
|
|
"expected_slots": {
|
|
"song": "夜的第七章"
|
|
},
|
|
"predicted_slots": {
|
|
"song": "夜的第七章"
|
|
},
|
|
"intent_score": 0.9828,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.9828
|
|
},
|
|
{
|
|
"intent_id": "cabin_nav_to",
|
|
"score": 0.0015
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.0012
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "来点爵士",
|
|
"category": "slot_music",
|
|
"expected_intent_id": "cabin_play_music",
|
|
"predicted_intent_id": "cabin_play_music",
|
|
"expected_slots": {
|
|
"genre": "爵士"
|
|
},
|
|
"predicted_slots": {
|
|
"genre": "爵士"
|
|
},
|
|
"intent_score": 0.9908,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.9908
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.0007
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.0006
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "放点摇滚",
|
|
"category": "slot_music",
|
|
"expected_intent_id": "cabin_play_music",
|
|
"predicted_intent_id": "cabin_play_music",
|
|
"expected_slots": {
|
|
"genre": "摇滚"
|
|
},
|
|
"predicted_slots": {
|
|
"genre": "摇滚"
|
|
},
|
|
"intent_score": 0.9915,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.9915
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.0006
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.0006
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "给我播点民谣",
|
|
"category": "slot_music",
|
|
"expected_intent_id": "cabin_play_music",
|
|
"predicted_intent_id": "cabin_play_music",
|
|
"expected_slots": {
|
|
"genre": "民谣"
|
|
},
|
|
"predicted_slots": {
|
|
"genre": "民谣"
|
|
},
|
|
"intent_score": 0.9917,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 1,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.9917
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.0007
|
|
},
|
|
{
|
|
"intent_id": "cabin_sunroof_open",
|
|
"score": 0.0005
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把车窗打开",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_window_open",
|
|
"predicted_intent_id": "cabin_window_open",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9474,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_window_open",
|
|
"score": 0.9474
|
|
},
|
|
{
|
|
"intent_id": "cabin_window_close",
|
|
"score": 0.0074
|
|
},
|
|
{
|
|
"intent_id": "cabin_sunroof_open",
|
|
"score": 0.0053
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把车窗关上",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_window_close",
|
|
"predicted_intent_id": "cabin_window_close",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9544,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_window_close",
|
|
"score": 0.9544
|
|
},
|
|
{
|
|
"intent_id": "cabin_lock_doors",
|
|
"score": 0.0088
|
|
},
|
|
{
|
|
"intent_id": "cabin_window_open",
|
|
"score": 0.0065
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把天窗打开",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_sunroof_open",
|
|
"predicted_intent_id": "cabin_sunroof_open",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9715,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_sunroof_open",
|
|
"score": 0.9715
|
|
},
|
|
{
|
|
"intent_id": "cabin_sunroof_close",
|
|
"score": 0.0032
|
|
},
|
|
{
|
|
"intent_id": "cabin_wiper_on",
|
|
"score": 0.0018
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把天窗合上",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_sunroof_close",
|
|
"predicted_intent_id": "cabin_sunroof_close",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.7244,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_sunroof_close",
|
|
"score": 0.7244
|
|
},
|
|
{
|
|
"intent_id": "cabin_sunroof_open",
|
|
"score": 0.204
|
|
},
|
|
{
|
|
"intent_id": "cabin_mirror_unfold",
|
|
"score": 0.008
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把空调打开",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_ac_on",
|
|
"predicted_intent_id": "cabin_ac_on",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9545,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_ac_on",
|
|
"score": 0.9545
|
|
},
|
|
{
|
|
"intent_id": "cabin_ac_off",
|
|
"score": 0.0079
|
|
},
|
|
{
|
|
"intent_id": "cabin_sunroof_open",
|
|
"score": 0.0029
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把空调关掉",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_ac_off",
|
|
"predicted_intent_id": "cabin_ac_off",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9673,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_ac_off",
|
|
"score": 0.9673
|
|
},
|
|
{
|
|
"intent_id": "cabin_ac_on",
|
|
"score": 0.0035
|
|
},
|
|
{
|
|
"intent_id": "cabin_lights_off",
|
|
"score": 0.0032
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把风量调大一点",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_fan_up",
|
|
"predicted_intent_id": "cabin_fan_up",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.5975,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.5975
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.3425
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_up",
|
|
"score": 0.0044
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "风太大了，往小调一点",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_fan_down",
|
|
"predicted_intent_id": "cabin_fan_down",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.6892,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.6892
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.2052
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_down",
|
|
"score": 0.014
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把音乐暂停",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_pause_music",
|
|
"predicted_intent_id": "cabin_pause_music",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9672,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.9672
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_mute",
|
|
"score": 0.0039
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.0025
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "帮我切到下一首",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_next_track",
|
|
"predicted_intent_id": "cabin_next_track",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9667,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.9667
|
|
},
|
|
{
|
|
"intent_id": "cabin_previous_track",
|
|
"score": 0.0104
|
|
},
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.0022
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "切回上一首",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_previous_track",
|
|
"predicted_intent_id": "cabin_previous_track",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9694,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_previous_track",
|
|
"score": 0.9694
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.0096
|
|
},
|
|
{
|
|
"intent_id": "cabin_nav_cancel",
|
|
"score": 0.0018
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "先静音",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_volume_mute",
|
|
"predicted_intent_id": "cabin_volume_mute",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9763,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_volume_mute",
|
|
"score": 0.9763
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.0022
|
|
},
|
|
{
|
|
"intent_id": "cs_query_logistics",
|
|
"score": 0.0016
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把音量调大一点",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_volume_up",
|
|
"predicted_intent_id": "cabin_volume_up",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.6867,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_volume_up",
|
|
"score": 0.6867
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_down",
|
|
"score": 0.2533
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.0067
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把音量调小一点",
|
|
"category": "no_slot_control",
|
|
"expected_intent_id": "cabin_volume_down",
|
|
"predicted_intent_id": "cabin_volume_down",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.6048,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_volume_down",
|
|
"score": 0.6048
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_up",
|
|
"score": 0.327
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.009
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把后视镜收起来",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_mirror_fold",
|
|
"predicted_intent_id": "cabin_mirror_fold",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9329,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_mirror_fold",
|
|
"score": 0.9329
|
|
},
|
|
{
|
|
"intent_id": "cabin_mirror_unfold",
|
|
"score": 0.0315
|
|
},
|
|
{
|
|
"intent_id": "cabin_wiper_off",
|
|
"score": 0.0046
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把镜子展开",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_mirror_unfold",
|
|
"predicted_intent_id": "cabin_mirror_unfold",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.7236,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_mirror_unfold",
|
|
"score": 0.7236
|
|
},
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.025
|
|
},
|
|
{
|
|
"intent_id": "cabin_mirror_fold",
|
|
"score": 0.0239
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "锁车门",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_lock_doors",
|
|
"predicted_intent_id": "cabin_lock_doors",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9345,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_lock_doors",
|
|
"score": 0.9345
|
|
},
|
|
{
|
|
"intent_id": "cabin_unlock_doors",
|
|
"score": 0.0328
|
|
},
|
|
{
|
|
"intent_id": "cabin_window_close",
|
|
"score": 0.0061
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把车门解锁",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_unlock_doors",
|
|
"predicted_intent_id": "cabin_unlock_doors",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9194,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_unlock_doors",
|
|
"score": 0.9194
|
|
},
|
|
{
|
|
"intent_id": "cabin_lock_doors",
|
|
"score": 0.0488
|
|
},
|
|
{
|
|
"intent_id": "cabin_seat_heat_off",
|
|
"score": 0.0032
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "路线别导了",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_nav_cancel",
|
|
"predicted_intent_id": "cabin_nav_cancel",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.6978,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_nav_cancel",
|
|
"score": 0.6978
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.069
|
|
},
|
|
{
|
|
"intent_id": "cs_cancel_order",
|
|
"score": 0.0353
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "音乐停一下",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_pause_music",
|
|
"predicted_intent_id": "cabin_pause_music",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9077,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.9077
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_mute",
|
|
"score": 0.012
|
|
},
|
|
{
|
|
"intent_id": "cabin_lights_off",
|
|
"score": 0.009
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "雨刮关掉",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_wiper_off",
|
|
"predicted_intent_id": "cabin_wiper_off",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9638,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_wiper_off",
|
|
"score": 0.9638
|
|
},
|
|
{
|
|
"intent_id": "cabin_wiper_on",
|
|
"score": 0.0044
|
|
},
|
|
{
|
|
"intent_id": "cabin_lights_off",
|
|
"score": 0.0033
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把雨刮打开",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_wiper_on",
|
|
"predicted_intent_id": "cabin_wiper_on",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9066,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_wiper_on",
|
|
"score": 0.9066
|
|
},
|
|
{
|
|
"intent_id": "cabin_wiper_off",
|
|
"score": 0.0363
|
|
},
|
|
{
|
|
"intent_id": "cabin_sunroof_open",
|
|
"score": 0.0069
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "把左前窗降一点",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_window_open",
|
|
"predicted_intent_id": "None",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.2812,
|
|
"intent_ok": false,
|
|
"slot_exact": true,
|
|
"joint_ok": false,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_defog_front_on",
|
|
"score": 0.2812
|
|
},
|
|
{
|
|
"intent_id": "cabin_window_open",
|
|
"score": 0.1715
|
|
},
|
|
{
|
|
"intent_id": "cabin_mirror_unfold",
|
|
"score": 0.0986
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "给我透个气",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_window_open",
|
|
"predicted_intent_id": "cabin_play_music",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9331,
|
|
"intent_ok": false,
|
|
"slot_exact": true,
|
|
"joint_ok": false,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_play_music",
|
|
"score": 0.9331
|
|
},
|
|
{
|
|
"intent_id": "cabin_next_track",
|
|
"score": 0.0138
|
|
},
|
|
{
|
|
"intent_id": "cabin_pause_music",
|
|
"score": 0.0031
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "风别太小",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cabin_fan_up",
|
|
"predicted_intent_id": "cabin_fan_down",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.6053,
|
|
"intent_ok": false,
|
|
"slot_exact": true,
|
|
"joint_ok": false,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_fan_down",
|
|
"score": 0.6053
|
|
},
|
|
{
|
|
"intent_id": "cabin_fan_up",
|
|
"score": 0.1549
|
|
},
|
|
{
|
|
"intent_id": "cabin_volume_down",
|
|
"score": 0.0286
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"text": "要是太慢就转人工",
|
|
"category": "failure_replay",
|
|
"expected_intent_id": "cs_transfer_human",
|
|
"predicted_intent_id": "cs_transfer_human",
|
|
"expected_slots": {},
|
|
"predicted_slots": {},
|
|
"intent_score": 0.9849,
|
|
"intent_ok": true,
|
|
"slot_exact": true,
|
|
"joint_ok": true,
|
|
"slot_diff": {
|
|
"missing_keys": [],
|
|
"extra_keys": [],
|
|
"wrong_values": [],
|
|
"matched_keys": 0,
|
|
"exact": true
|
|
},
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cs_transfer_human",
|
|
"score": 0.9849
|
|
},
|
|
{
|
|
"intent_id": "cs_query_logistics",
|
|
"score": 0.0018
|
|
},
|
|
{
|
|
"intent_id": "cs_cancel_order",
|
|
"score": 0.0016
|
|
}
|
|
]
|
|
}
|
|
]
|
|
} |