from __future__ import annotations

import argparse
import json
from collections import Counter, defaultdict
from pathlib import Path
import sys


PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from app.core.bootstrap import build_intent_registry
from app.services.multi_intent_detector import BertMultiIntentDetector


TEST_PATH = PROJECT_ROOT / "app/data/bert_intent_multilabel_eval_independent.jsonl"
MODEL_DIR = PROJECT_ROOT / "models/local_bert_multi_intent"
REPORT_DIR = PROJECT_ROOT / "reports"
RESULT_PATH = REPORT_DIR / "bert_multi_intent_independent_result.json"
REPORT_PATH = REPORT_DIR / "bert_multi_intent_independent_report.md"
THRESHOLD = 0.45
TOP_K = 8
MAX_LABELS = 4


def load_cases(file_path: Path) -> list[dict[str, object]]:
    cases: list[dict[str, object]] = []
    for line in file_path.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if not line:
            continue
        payload = json.loads(line)
        expected = sorted({str(item).strip() for item in payload.get("expected_intent_ids") or [] if str(item).strip()})
        if not expected:
            continue
        cases.append(
            {
                "text": str(payload["text"]),
                "expected_intent_ids": expected,
                "category": str(payload.get("category") or "unknown"),
            }
        )
    return cases


def compute_set_metrics(results: list[dict[str, object]]) -> dict[str, float]:
    true_positive = 0
    false_positive = 0
    false_negative = 0
    exact_match = 0
    multi_recall_hit = 0
    single_false_alarm = 0
    total = len(results)
    single_guard_total = 0
    for item in results:
        expected = set(item["expected_intent_ids"])
        predicted = set(item["predicted_intent_ids"])
        if expected == predicted:
            exact_match += 1
        true_positive += len(expected & predicted)
        false_positive += len(predicted - expected)
        false_negative += len(expected - predicted)
        if len(expected) >= 2 and expected.issubset(predicted):
            multi_recall_hit += 1
        if len(expected) == 1:
            single_guard_total += 1
            if len(predicted) > 1:
                single_false_alarm += 1
    precision = true_positive / (true_positive + false_positive) if (true_positive + false_positive) else 0.0
    recall = true_positive / (true_positive + false_negative) if (true_positive + false_negative) else 0.0
    micro_f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0
    multi_total = sum(1 for item in results if len(item["expected_intent_ids"]) >= 2)
    return {
        "micro_precision": round(precision, 4),
        "micro_recall": round(recall, 4),
        "micro_f1": round(micro_f1, 4),
        "exact_match": round(exact_match / total, 4) if total else 0.0,
        "multi_sentence_recall": round(multi_recall_hit / multi_total, 4) if multi_total else 0.0,
        "single_guard_false_alarm_rate": round(single_false_alarm / single_guard_total, 4) if single_guard_total else 0.0,
    }


def summarize_by_category(results: list[dict[str, object]]) -> list[dict[str, object]]:
    grouped: dict[str, list[dict[str, object]]] = defaultdict(list)
    for item in results:
        grouped[str(item["category"])].append(item)
    summary: list[dict[str, object]] = []
    for category, items in sorted(grouped.items()):
        summary.append(
            {
                "category": category,
                "sample_count": len(items),
                "metrics": compute_set_metrics(items),
            }
        )
    return summary


def collect_error_examples(results: list[dict[str, object]], limit: int = 15) -> list[dict[str, object]]:
    errors = [item for item in results if set(item["expected_intent_ids"]) != set(item["predicted_intent_ids"])]
    def sort_key(item: dict[str, object]) -> tuple[int, int]:
        expected = set(item["expected_intent_ids"])
        predicted = set(item["predicted_intent_ids"])
        miss = len(expected - predicted)
        extra = len(predicted - expected)
        return (miss + extra, miss)
    return sorted(errors, key=sort_key, reverse=True)[:limit]


def top_confusions(results: list[dict[str, object]], limit: int = 12) -> list[dict[str, object]]:
    counter: Counter[tuple[str, str]] = Counter()
    for item in results:
        expected = set(item["expected_intent_ids"])
        predicted = set(item["predicted_intent_ids"])
        for miss in sorted(expected - predicted):
            for extra in sorted(predicted - expected):
                counter[(miss, extra)] += 1
    return [
        {"expected_missing": pair[0], "wrong_extra": pair[1], "count": count}
        for pair, count in counter.most_common(limit)
    ]


def main() -> None:
    parser = argparse.ArgumentParser(description="本地多标签 detector 独立评测脚本")
    parser.add_argument("--test-path", type=str, default=str(TEST_PATH), help="独立评测集路径")
    parser.add_argument("--model-path", type=str, default=str(MODEL_DIR), help="多标签模型路径")
    parser.add_argument("--threshold", type=float, default=THRESHOLD, help="检测阈值")
    parser.add_argument("--top-k", type=int, default=TOP_K, help="输出 top-k 原始分数")
    parser.add_argument("--max-labels", type=int, default=MAX_LABELS, help="最多返回标签数")
    parser.add_argument("--result-path", type=str, default=str(RESULT_PATH), help="结构化结果输出路径")
    parser.add_argument("--report-path", type=str, default=str(REPORT_PATH), help="Markdown 报告输出路径")
    args = parser.parse_args()

    cases = load_cases(Path(args.test_path))
    intents = build_intent_registry().list()
    detector = BertMultiIntentDetector(
        model_path=args.model_path,
        threshold=args.threshold,
        top_k=args.top_k,
        max_labels=args.max_labels,
    )

    results: list[dict[str, object]] = []
    for case in cases:
        detection = detector.detect(str(case["text"]), intents)
        predicted = [candidate.intent_id for candidate in detection.candidates]
        raw_top = [
            {
                "intent_id": str(item.get("intent_id") or item.get("label") or ""),
                "score": round(float(item.get("score", 0.0)), 4),
            }
            for item in detection.raw_scores
        ]
        results.append(
            {
                "text": case["text"],
                "category": case["category"],
                "expected_intent_ids": case["expected_intent_ids"],
                "predicted_intent_ids": predicted,
                "detected": detection.detected,
                "backend_name": detection.backend_name,
                "reason": detection.reason,
                "raw_top_scores": raw_top,
            }
        )

    summary = {
        "model_path": args.model_path,
        "test_path": args.test_path,
        "threshold": args.threshold,
        "top_k": args.top_k,
        "max_labels": args.max_labels,
        "sample_count": len(results),
        "metrics": compute_set_metrics(results),
        "per_category": summarize_by_category(results),
        "top_confusions": top_confusions(results),
        "error_examples": collect_error_examples(results),
        "results": results,
    }

    REPORT_DIR.mkdir(parents=True, exist_ok=True)
    Path(args.result_path).write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
    Path(args.report_path).write_text(render_report(summary), encoding="utf-8")
    print(json.dumps({"sample_count": len(results), "metrics": summary["metrics"]}, ensure_ascii=False))


def render_report(summary: dict[str, object]) -> str:
    metrics = summary["metrics"]
    per_category = summary["per_category"]
    confusions = summary["top_confusions"]
    errors = summary["error_examples"]
    lines = [
        "# 本地多标签 Detector 独立评测报告",
        "",
        "## 概览",
        f"- 模型目录：`{summary['model_path']}`",
        f"- 评测集：`{summary['test_path']}`",
        f"- 样本数：`{summary['sample_count']}`",
        f"- 阈值 / top_k / max_labels：`{summary['threshold']} / {summary['top_k']} / {summary['max_labels']}`",
        f"- `micro_precision`：`{metrics['micro_precision']}`",
        f"- `micro_recall`：`{metrics['micro_recall']}`",
        f"- `micro_f1`：`{metrics['micro_f1']}`",
        f"- `exact_match`：`{metrics['exact_match']}`",
        f"- `multi_sentence_recall`：`{metrics['multi_sentence_recall']}`",
        f"- `single_guard_false_alarm_rate`：`{metrics['single_guard_false_alarm_rate']}`",
        "",
        "## 分类别结果",
    ]
    for item in per_category:
        category_metrics = item["metrics"]
        lines.append(
            f"- `{item['category']}`: count={item['sample_count']} micro_f1={category_metrics['micro_f1']} exact_match={category_metrics['exact_match']}"
        )
    lines.extend(["", "## 主要混淆"])
    if not confusions:
        lines.append("- 未发现明显混淆对。")
    else:
        for item in confusions:
            lines.append(
                f"- 漏掉 `{item['expected_missing']}`，同时误报 `{item['wrong_extra']}`：`{item['count']}` 次"
            )
    lines.extend(["", "## 错误样例"])
    if not errors:
        lines.append("- 无错误样例。")
    else:
        for item in errors:
            lines.append(
                f"- 文本：`{item['text']}` | 类别：`{item['category']}` | 期望：`{item['expected_intent_ids']}` | 预测：`{item['predicted_intent_ids']}`"
            )
    lines.extend(
        [
            "",
            "## 结论建议",
            "- 先看多意图句是否存在系统性漏召回，再看单意图是否被误报成多意图。",
            "- 若 `single_guard_false_alarm_rate` 偏高，需要先收紧 detector 阈值或补单意图负样本，再考虑进入 NER。",
            "- 若 `multi_sentence_recall` 不稳定，应继续补条件句、弱连接句和口语化多动作语料。",
            "",
        ]
    )
    return "\n".join(lines)


if __name__ == "__main__":
    main()
