251 lines
10 KiB
Python
251 lines
10 KiB
Python
"""
|
||
BOM 成本 - 终极树状结构抓取脚本 (全站 1400+ 父件及 5 层嵌套子件)
|
||
目标:
|
||
1. 抓取所有父件（成本核算表主页）
|
||
2. 暗网请求所有父件下对应的 BOM 成本数据（扁平的 5 层数据）
|
||
3. 实时清洗并重组为完美嵌套的 JSON 树
|
||
"""
|
||
import sys
|
||
import json
|
||
import time
|
||
import subprocess
|
||
import random
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).parent))
|
||
from login import get_page, log
|
||
from config import OUTPUT_DIR
|
||
|
||
PAGE_URL = "https://yunmes.tftykj.cn/PartBomCostAccounting"
|
||
API_PARENT = "PartBomCostAccounting_SearchList_Proxy"
|
||
|
||
# 最终保存的文件名
|
||
TREE_FILE_PATH = OUTPUT_DIR / "bom_cost_full_tree_final.json"
|
||
|
||
def build_nested_tree(flat_items):
|
||
"""将包含 _parentId 的扁平列表转换为嵌套树"""
|
||
if not flat_items:
|
||
return []
|
||
|
||
node_dict = {}
|
||
for item in flat_items:
|
||
son_no = item.get("sonNO")
|
||
if not son_no:
|
||
continue
|
||
|
||
clean_node = {
|
||
"sonNO": son_no,
|
||
"_parentId": item.get("_parentId"),
|
||
"bomLevel": item.get("bomLevel"),
|
||
"childMaterialCode": item.get("childMaterialCode"),
|
||
"childMaterialName": item.get("childMaterialName"),
|
||
"usageQty": item.get("childrenMaterialConsumption", 1),
|
||
"sub_items": []
|
||
}
|
||
node_dict[son_no] = clean_node
|
||
|
||
roots = []
|
||
for son_no, node in node_dict.items():
|
||
parent_id = node.get("_parentId")
|
||
if parent_id is None:
|
||
roots.append(node)
|
||
else:
|
||
parent_node = node_dict.get(parent_id)
|
||
if parent_node:
|
||
parent_node["sub_items"].append(node)
|
||
|
||
# 清理建树临时字段
|
||
def clean_temp(node_list):
|
||
for node in node_list:
|
||
node.pop("sonNO", None)
|
||
node.pop("_parentId", None)
|
||
if node["sub_items"]:
|
||
clean_temp(node["sub_items"])
|
||
else:
|
||
node.pop("sub_items", None)
|
||
|
||
clean_temp(roots)
|
||
return roots
|
||
|
||
|
||
def fetch_bom_cost_tree():
|
||
log("INFO", "=== 🌳 启动 BOM 成本终极抓取 (多层嵌套自动重组) ===")
|
||
page = get_page(port=9222)
|
||
clean_parents_list = []
|
||
|
||
try:
|
||
# =========================================================
|
||
# 第一阶段：获取父件基础信息
|
||
# =========================================================
|
||
log("INFO", f"正在访问安全的父件页面: {PAGE_URL}")
|
||
page.get(PAGE_URL)
|
||
page.wait.load_start()
|
||
|
||
log("INFO", f"开启父件 API 网络监听: {API_PARENT}")
|
||
page.listen.start(API_PARENT)
|
||
page.refresh()
|
||
|
||
current_page = 1
|
||
total_records = 0
|
||
|
||
while True:
|
||
log("INFO", f"等待第 {current_page} 页父件 API 响应...")
|
||
packet = page.listen.wait(timeout=20)
|
||
|
||
if not packet:
|
||
log("ERR", f"超时未收到第 {current_page} 页数据，父件扫荡结束。")
|
||
break
|
||
|
||
body = packet.response.body
|
||
data = body if isinstance(body, (dict, list)) else json.loads(body)
|
||
|
||
if isinstance(data, dict) and "result" in data:
|
||
result_data = data.get("result") or {}
|
||
items = result_data.get("items", [])
|
||
total_records = result_data.get("totalCount", 0)
|
||
|
||
for item in items:
|
||
if not item:
|
||
continue
|
||
# 注意：我们要拿的是 parentMaterialId，因为这是传给 BOM 成本 API 的关键参数 materialId
|
||
clean_parent = {
|
||
"_id": item.get("id"), # 这个是 partBomCostAccountingId
|
||
"_materialId": item.get("parentMaterialId"), # 这个是传给子件的 materialId
|
||
"parentMaterialCode": item.get("parentMaterialCode"),
|
||
"parentMaterialName": item.get("parentMaterialName"),
|
||
"bom_cost_tree": [] # 准备挂载这棵树
|
||
}
|
||
clean_parents_list.append(clean_parent)
|
||
|
||
log("OK", f"提取了 {len(items)} 个父件。总进度: {len(clean_parents_list)}/{total_records}")
|
||
|
||
if len(clean_parents_list) >= total_records or len(items) == 0:
|
||
break
|
||
else:
|
||
break
|
||
|
||
# 准备翻页
|
||
next_btn_xpath = "xpath:/html/body/div[1]/div/div[3]/table/tbody/tr/td[10]/a/span/span[2]"
|
||
next_btn = page.ele(next_btn_xpath, timeout=5)
|
||
|
||
if next_btn:
|
||
parent_a = next_btn.parent(2)
|
||
if parent_a and "disabled" in parent_a.attr("class"):
|
||
log("INFO", "已到达最后一页。")
|
||
break
|
||
page.run_js("arguments[0].click();", next_btn)
|
||
time.sleep(1.5)
|
||
else:
|
||
log("WARN", "未找到下一页按钮，停止翻页。")
|
||
break
|
||
|
||
current_page += 1
|
||
|
||
page.listen.stop()
|
||
|
||
# =========================================================
|
||
# 第二阶段：暗网递归注入，重组 5 层嵌套树
|
||
# =========================================================
|
||
log("INFO", f"=== 🚀 开始为 {len(clean_parents_list)} 个父件抓取 BOM 成本树 ===")
|
||
|
||
js_template = """
|
||
return new Promise((resolve, reject) => {
|
||
if (typeof $ !== 'undefined' && $.ajax) {
|
||
$.ajax({
|
||
url: '/api/services/TfTechApi/PartBom/PartBom_SearchByTreeCost',
|
||
type: 'POST',
|
||
data: {
|
||
materialId: MATERIAL_ID_PLACEHOLDER,
|
||
partBomCostAccountingId: ACCOUNTING_ID_PLACEHOLDER,
|
||
childMaterialCode: '',
|
||
childMaterialName: '',
|
||
childMaterialSpecification: '',
|
||
childMaterialModel: ''
|
||
},
|
||
headers: {
|
||
'referer': 'https://yunmes.tftykj.cn/PartBomCostAccounting/Detail?id=ACCOUNTING_ID_PLACEHOLDER'
|
||
},
|
||
success: function(response) {
|
||
resolve({status: 'success', data: response});
|
||
},
|
||
error: function(xhr, status, error) {
|
||
resolve({status: 'error', data: xhr.responseText || error});
|
||
}
|
||
});
|
||
} else {
|
||
resolve({status: 'error', data: 'No jQuery'});
|
||
}
|
||
});
|
||
"""
|
||
|
||
for index, parent in enumerate(clean_parents_list):
|
||
accounting_id = parent.get("_id")
|
||
material_id = parent.get("_materialId")
|
||
parent_code = parent.get("parentMaterialCode", "未知")
|
||
|
||
if not accounting_id or not material_id:
|
||
continue
|
||
|
||
log("INFO", f"[{index+1}/{len(clean_parents_list)}] 正在请求 BOM 成本树 (Code: {parent_code})...")
|
||
|
||
js_code = js_template.replace("MATERIAL_ID_PLACEHOLDER", str(material_id)).replace("ACCOUNTING_ID_PLACEHOLDER", str(accounting_id))
|
||
result = page.run_js(js_code)
|
||
|
||
if result and result.get('status') == 'success':
|
||
data = result.get('data')
|
||
if isinstance(data, str):
|
||
try: data = json.loads(data)
|
||
except: pass
|
||
|
||
if isinstance(data, dict) and "result" in data:
|
||
flat_items = data["result"]
|
||
if isinstance(flat_items, list):
|
||
# 核心：调用刚才验证成功的重组函数，把扁平列表变成 5 层树
|
||
nested_tree = build_nested_tree(flat_items)
|
||
parent["bom_cost_tree"] = nested_tree
|
||
log("OK", f" └── 成功重组了一棵包含 {len(flat_items)} 个节点的多层树。")
|
||
else:
|
||
log("ERR", f" └── 请求失败: {result.get('data') if result else '未知错误'}")
|
||
|
||
time.sleep(random.uniform(0.3, 0.7))
|
||
|
||
if (index + 1) % 10 == 0 or (index + 1) == len(clean_parents_list):
|
||
# 最终保存前，清理一下用于请求的临时字段
|
||
clean_save_list = []
|
||
for p in clean_parents_list[:index+1]:
|
||
clean_p = dict(p)
|
||
clean_p.pop("_id", None)
|
||
clean_p.pop("_materialId", None)
|
||
clean_save_list.append(clean_p)
|
||
|
||
with open(TREE_FILE_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(clean_save_list, f, ensure_ascii=False, indent=2)
|
||
log("INFO", f"💾 进度已实时保存至 JSON ({index+1}/{len(clean_parents_list)})")
|
||
|
||
log("OK", f"=== 🏆 终极 BOM 成本多层树状抓取完成！文件路径: {TREE_FILE_PATH} ===")
|
||
|
||
# 抓取完成后，自动调用入库脚本将 JSON 导入 SQLite
|
||
log("INFO", "⏳ 正在自动将 JSON 数据同步至 SQLite 数据库...")
|
||
try:
|
||
import_script = Path(__file__).parent / "import_to_sqlite.py"
|
||
# 使用 sys.executable 确保使用当前的 Python 环境
|
||
import sys
|
||
result = subprocess.run([sys.executable, str(import_script), "--bom-only"], capture_output=True, text=True)
|
||
if result.returncode == 0:
|
||
log("OK", "✅ 数据库同步成功！")
|
||
print(result.stdout)
|
||
else:
|
||
log("ERR", f"❌ 数据库同步失败: {result.stderr}")
|
||
except Exception as db_err:
|
||
log("ERR", f"❌ 自动触发入库脚本失败: {db_err}")
|
||
|
||
except Exception as e:
|
||
import traceback
|
||
err_msg = f"发生异常: {e}\n{traceback.format_exc()}"
|
||
with open("error.log", "w") as f:
|
||
f.write(err_msg)
|
||
print(err_msg, flush=True)
|
||
log("ERR", err_msg)
|
||
|
||
if __name__ == "__main__":
|
||
fetch_bom_cost_tree() |