426 lines
8.4 KiB
JSON
426 lines
8.4 KiB
JSON
{
|
|
"model_dir": "/Users/hwp/Documents/trae_projects/intelligent_cabin/models/local_bert_intent",
|
|
"threshold": 0.0,
|
|
"test_path": "/Users/hwp/Documents/trae_projects/intelligent_cabin/app/data/bert_intent_eval_independent.jsonl",
|
|
"test_case_count": 42,
|
|
"accuracy": 0.9762,
|
|
"train_summary": {
|
|
"base_model": "hfl/chinese-macbert-base",
|
|
"epochs": 16,
|
|
"batch_size": 8,
|
|
"learning_rate": 2e-05,
|
|
"train_size": 1557,
|
|
"dev_size": 401,
|
|
"best_dev_accuracy": 0.9875,
|
|
"device": "mps"
|
|
},
|
|
"per_category": [
|
|
{
|
|
"category": "business",
|
|
"total": 34,
|
|
"correct": 33,
|
|
"accuracy": 0.9706
|
|
},
|
|
{
|
|
"category": "out_of_scope",
|
|
"total": 4,
|
|
"correct": 4,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"category": "social",
|
|
"total": 4,
|
|
"correct": 4,
|
|
"accuracy": 1.0
|
|
}
|
|
],
|
|
"per_label": [
|
|
{
|
|
"label": "__out_of_scope__",
|
|
"category": "out_of_scope",
|
|
"total": 4,
|
|
"correct": 4,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "__social__",
|
|
"category": "social",
|
|
"total": 4,
|
|
"correct": 4,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_ac_off",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_ac_on",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_defog_front_on",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_defog_rear_on",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_fan_down",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_fan_up",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_lights_off",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_lights_on",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_lock_doors",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_mirror_fold",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_mirror_unfold",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_nav_cancel",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_nav_to",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_next_track",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_pause_music",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_play_music",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_previous_track",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_seat_heat_off",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_seat_heat_on",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_set_ac",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_sunroof_close",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_sunroof_open",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_unlock_doors",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_volume_down",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_volume_mute",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_volume_up",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_window_close",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_window_open",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 0,
|
|
"accuracy": 0.0
|
|
},
|
|
{
|
|
"label": "cabin_wiper_off",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cabin_wiper_on",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cs_cancel_order",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cs_query_logistics",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cs_query_order",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
},
|
|
{
|
|
"label": "cs_transfer_human",
|
|
"category": "business",
|
|
"total": 1,
|
|
"correct": 1,
|
|
"accuracy": 1.0
|
|
}
|
|
],
|
|
"errors": [
|
|
{
|
|
"text": "左前窗打开一点",
|
|
"category": "business",
|
|
"expected_label": "cabin_window_open",
|
|
"predicted_label": "cabin_defog_front_on",
|
|
"score": 0.9951,
|
|
"raw_label": "cabin_defog_front_on",
|
|
"ok": false,
|
|
"top_candidates": [
|
|
{
|
|
"intent_id": "cabin_defog_front_on",
|
|
"score": 0.9951
|
|
},
|
|
{
|
|
"intent_id": "cabin_sunroof_open",
|
|
"score": 0.0005
|
|
},
|
|
{
|
|
"intent_id": "cabin_lights_on",
|
|
"score": 0.0004
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"confusion": {
|
|
"cabin_ac_off": {
|
|
"cabin_ac_off": 1
|
|
},
|
|
"cabin_ac_on": {
|
|
"cabin_ac_on": 1
|
|
},
|
|
"cabin_defog_front_on": {
|
|
"cabin_defog_front_on": 1
|
|
},
|
|
"cabin_defog_rear_on": {
|
|
"cabin_defog_rear_on": 1
|
|
},
|
|
"cabin_fan_down": {
|
|
"cabin_fan_down": 1
|
|
},
|
|
"cabin_fan_up": {
|
|
"cabin_fan_up": 1
|
|
},
|
|
"cabin_lights_off": {
|
|
"cabin_lights_off": 1
|
|
},
|
|
"cabin_lights_on": {
|
|
"cabin_lights_on": 1
|
|
},
|
|
"cabin_lock_doors": {
|
|
"cabin_lock_doors": 1
|
|
},
|
|
"cabin_mirror_fold": {
|
|
"cabin_mirror_fold": 1
|
|
},
|
|
"cabin_mirror_unfold": {
|
|
"cabin_mirror_unfold": 1
|
|
},
|
|
"cabin_nav_cancel": {
|
|
"cabin_nav_cancel": 1
|
|
},
|
|
"cabin_nav_to": {
|
|
"cabin_nav_to": 1
|
|
},
|
|
"cabin_next_track": {
|
|
"cabin_next_track": 1
|
|
},
|
|
"cabin_pause_music": {
|
|
"cabin_pause_music": 1
|
|
},
|
|
"cabin_play_music": {
|
|
"cabin_play_music": 1
|
|
},
|
|
"cabin_previous_track": {
|
|
"cabin_previous_track": 1
|
|
},
|
|
"cabin_seat_heat_off": {
|
|
"cabin_seat_heat_off": 1
|
|
},
|
|
"cabin_seat_heat_on": {
|
|
"cabin_seat_heat_on": 1
|
|
},
|
|
"cabin_set_ac": {
|
|
"cabin_set_ac": 1
|
|
},
|
|
"cabin_sunroof_close": {
|
|
"cabin_sunroof_close": 1
|
|
},
|
|
"cabin_sunroof_open": {
|
|
"cabin_sunroof_open": 1
|
|
},
|
|
"cabin_unlock_doors": {
|
|
"cabin_unlock_doors": 1
|
|
},
|
|
"cabin_volume_down": {
|
|
"cabin_volume_down": 1
|
|
},
|
|
"cabin_volume_mute": {
|
|
"cabin_volume_mute": 1
|
|
},
|
|
"cabin_volume_up": {
|
|
"cabin_volume_up": 1
|
|
},
|
|
"cabin_window_close": {
|
|
"cabin_window_close": 1
|
|
},
|
|
"cabin_window_open": {
|
|
"cabin_defog_front_on": 1
|
|
},
|
|
"cabin_wiper_off": {
|
|
"cabin_wiper_off": 1
|
|
},
|
|
"cabin_wiper_on": {
|
|
"cabin_wiper_on": 1
|
|
},
|
|
"cs_cancel_order": {
|
|
"cs_cancel_order": 1
|
|
},
|
|
"cs_query_logistics": {
|
|
"cs_query_logistics": 1
|
|
},
|
|
"cs_query_order": {
|
|
"cs_query_order": 1
|
|
},
|
|
"cs_transfer_human": {
|
|
"cs_transfer_human": 1
|
|
},
|
|
"__social__": {
|
|
"__social__": 4
|
|
},
|
|
"__out_of_scope__": {
|
|
"__out_of_scope__": 4
|
|
}
|
|
}
|
|
} |