Files
ai-device/intelligent_cabin/archive/reports/bert_local_test_result.json
2026-06-11 16:28:00 +08:00

426 lines
8.4 KiB
JSON

{
"model_dir": "/Users/hwp/Documents/trae_projects/intelligent_cabin/models/local_bert_intent",
"threshold": 0.0,
"test_path": "/Users/hwp/Documents/trae_projects/intelligent_cabin/app/data/bert_intent_eval_independent.jsonl",
"test_case_count": 42,
"accuracy": 0.9762,
"train_summary": {
"base_model": "hfl/chinese-macbert-base",
"epochs": 16,
"batch_size": 8,
"learning_rate": 2e-05,
"train_size": 1557,
"dev_size": 401,
"best_dev_accuracy": 0.9875,
"device": "mps"
},
"per_category": [
{
"category": "business",
"total": 34,
"correct": 33,
"accuracy": 0.9706
},
{
"category": "out_of_scope",
"total": 4,
"correct": 4,
"accuracy": 1.0
},
{
"category": "social",
"total": 4,
"correct": 4,
"accuracy": 1.0
}
],
"per_label": [
{
"label": "__out_of_scope__",
"category": "out_of_scope",
"total": 4,
"correct": 4,
"accuracy": 1.0
},
{
"label": "__social__",
"category": "social",
"total": 4,
"correct": 4,
"accuracy": 1.0
},
{
"label": "cabin_ac_off",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_ac_on",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_defog_front_on",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_defog_rear_on",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_fan_down",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_fan_up",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_lights_off",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_lights_on",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_lock_doors",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_mirror_fold",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_mirror_unfold",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_nav_cancel",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_nav_to",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_next_track",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_pause_music",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_play_music",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_previous_track",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_seat_heat_off",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_seat_heat_on",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_set_ac",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_sunroof_close",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_sunroof_open",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_unlock_doors",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_volume_down",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_volume_mute",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_volume_up",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_window_close",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_window_open",
"category": "business",
"total": 1,
"correct": 0,
"accuracy": 0.0
},
{
"label": "cabin_wiper_off",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cabin_wiper_on",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cs_cancel_order",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cs_query_logistics",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cs_query_order",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
},
{
"label": "cs_transfer_human",
"category": "business",
"total": 1,
"correct": 1,
"accuracy": 1.0
}
],
"errors": [
{
"text": "左前窗打开一点",
"category": "business",
"expected_label": "cabin_window_open",
"predicted_label": "cabin_defog_front_on",
"score": 0.9951,
"raw_label": "cabin_defog_front_on",
"ok": false,
"top_candidates": [
{
"intent_id": "cabin_defog_front_on",
"score": 0.9951
},
{
"intent_id": "cabin_sunroof_open",
"score": 0.0005
},
{
"intent_id": "cabin_lights_on",
"score": 0.0004
}
]
}
],
"confusion": {
"cabin_ac_off": {
"cabin_ac_off": 1
},
"cabin_ac_on": {
"cabin_ac_on": 1
},
"cabin_defog_front_on": {
"cabin_defog_front_on": 1
},
"cabin_defog_rear_on": {
"cabin_defog_rear_on": 1
},
"cabin_fan_down": {
"cabin_fan_down": 1
},
"cabin_fan_up": {
"cabin_fan_up": 1
},
"cabin_lights_off": {
"cabin_lights_off": 1
},
"cabin_lights_on": {
"cabin_lights_on": 1
},
"cabin_lock_doors": {
"cabin_lock_doors": 1
},
"cabin_mirror_fold": {
"cabin_mirror_fold": 1
},
"cabin_mirror_unfold": {
"cabin_mirror_unfold": 1
},
"cabin_nav_cancel": {
"cabin_nav_cancel": 1
},
"cabin_nav_to": {
"cabin_nav_to": 1
},
"cabin_next_track": {
"cabin_next_track": 1
},
"cabin_pause_music": {
"cabin_pause_music": 1
},
"cabin_play_music": {
"cabin_play_music": 1
},
"cabin_previous_track": {
"cabin_previous_track": 1
},
"cabin_seat_heat_off": {
"cabin_seat_heat_off": 1
},
"cabin_seat_heat_on": {
"cabin_seat_heat_on": 1
},
"cabin_set_ac": {
"cabin_set_ac": 1
},
"cabin_sunroof_close": {
"cabin_sunroof_close": 1
},
"cabin_sunroof_open": {
"cabin_sunroof_open": 1
},
"cabin_unlock_doors": {
"cabin_unlock_doors": 1
},
"cabin_volume_down": {
"cabin_volume_down": 1
},
"cabin_volume_mute": {
"cabin_volume_mute": 1
},
"cabin_volume_up": {
"cabin_volume_up": 1
},
"cabin_window_close": {
"cabin_window_close": 1
},
"cabin_window_open": {
"cabin_defog_front_on": 1
},
"cabin_wiper_off": {
"cabin_wiper_off": 1
},
"cabin_wiper_on": {
"cabin_wiper_on": 1
},
"cs_cancel_order": {
"cs_cancel_order": 1
},
"cs_query_logistics": {
"cs_query_logistics": 1
},
"cs_query_order": {
"cs_query_order": 1
},
"cs_transfer_human": {
"cs_transfer_human": 1
},
"__social__": {
"__social__": 4
},
"__out_of_scope__": {
"__out_of_scope__": 4
}
}
}