""" 发料单报表 - 导航测试脚本 目标: 模拟点击菜单，进入“发料单报表”页面。 """ import sys import json import time import random from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from login import get_page, log from config import OUTPUT_DIR HOME_URL = "https://yunmes.tftykj.cn/" API_TARGET = "WorkOrdersDetailed_SearchListAll_Proxy" SAVE_PATH = OUTPUT_DIR / "issue_receipt_details_full.json" def fetch_issue_receipt_details(): log("INFO", "=== 🚀 启动发料单报表全量数据抓取 ===") # 强制复用 9222 端口，不关闭浏览器 page = get_page(port=9222) all_clean_items = [] if SAVE_PATH.exists(): try: with open(SAVE_PATH, "r", encoding="utf-8") as f: all_clean_items = json.load(f) log("INFO", f"📦 已加载本地历史存档，包含 {len(all_clean_items)} 条数据。") except Exception as e: log("WARN", f"加载本地存档失败: {e}，将从空列表开始。") all_clean_items = [] try: log("INFO", f"正在回到主页起点: {HOME_URL}") page.get(HOME_URL) page.wait.load_start() time.sleep(2) menus = [ ("第一层: 业务统计报表", 'xpath://*[@id="app"]/div/div[1]/div[1]/div[2]/div/div[1]/div/div[10]/div/p'), ("第二层: 生产业务报表(推测)", 'xpath:/html/body/div[7]/div/div[1]/div/div[9]/div/p'), ("第三层: 发料单报表", 'xpath:/html/body/div[8]/div/div[1]/div/div[6]/div/p') ] log("INFO", "开始模拟人工点击左侧导航菜单...") for name, xpath in menus: ele = page.ele(xpath, timeout=5) if ele: try: ele.click() except: page.run_js("arguments[0].click();", ele) time.sleep(1.5) else: log("ERR", f"找不到菜单元素: {name}") return log("OK", "✅ 成功点开发料单报表界面！") # 点击空白处隐藏菜单 blank_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[1]/div[2]/div[2]/div/div[1]/div' blank_ele = page.ele(blank_xpath, timeout=3) if blank_ele: try: blank_ele.click() except: page.run_js("arguments[0].click();", blank_ele) time.sleep(0.5) log("INFO", f"开启底层数据拦截网: {API_TARGET}") page.listen.start(API_TARGET) # 等待页面自动发起的请求 packet = page.listen.wait(timeout=10) if not packet: log("INFO", "尝试寻找并点击页面上的【查询】按钮...") query_btn_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[2]/div[1]/div[1]/div/button[1]/span' query_btn = page.ele(query_btn_xpath, timeout=3) if query_btn: try: query_btn.click() except: page.run_js("arguments[0].click();", query_btn) packet = page.listen.wait(timeout=15) if not packet: log("ERR", "未能拦截到数据请求，可能网络超时或查询未触发。") return # 设定开始抓取的页码，如果因为中断需要断点续传，请修改此变量 # 刚才抓到了 95 页，我们需要从 96 页开始继续 target_resume_page = 1 # ========================================================= # 第一页数据处理 # ========================================================= log("OK", f"🎉 成功拦截到第一页数据！HTTP 状态码: {packet.response.status}") body = packet.response.body data = body if isinstance(body, (dict, list)) else json.loads(body) total_count = 0 if isinstance(data, dict) and "result" in data: total_count = data["result"].get("totalCount", 0) items = data["result"].get("items", []) log("OK", f"后端报告总条数: {total_count}") # 只有当不是断点续传（即从第1页开始）时，才把第一页的数据加入列表 if target_resume_page <= 1: # 由于可能触发断点，如果是重新抓取，这里直接覆盖 if not all_clean_items: for item in items: all_clean_items.append(_extract_fields(item)) log("OK", f"第一页清洗完成，提取了 {len(items)} 条数据。") else: log("INFO", f"本地已有数据，跳过第一页保存，走翻页逻辑（注意：发料单可能需要您清空旧存档才能从头抓，这里先保留累加）") else: log("INFO", f"触发断点续传，跳过第一页的数据保存。后端报告总条数: {total_count}") else: log("ERR", "第一页返回的数据结构异常。") return page_num = 1 # ========================================================= # 断点续传逻辑跳转 # ========================================================= if target_resume_page > 1: log("INFO", f"🚀 触发断点续传机制！准备直接跳转到第 {target_resume_page} 页...") # 尝试找页码输入框 jumper_input_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[2]/div[1]/div[2]/div/div[2]/div[1]/span[3]/div/div//input' input_ele = page.ele(jumper_input_xpath, timeout=5) if not input_ele: jumper_input_xpath = 'xpath://input[@type="number" and @aria-label="页"]' input_ele = page.ele(jumper_input_xpath, timeout=5) if input_ele: input_ele.clear() input_ele.input(str(target_resume_page)) time.sleep(0.5) input_ele.input('\n') packet = page.listen.wait(timeout=15) if not packet: log("ERR", "断点跳转失败，未拦截到目标页的数据请求。") return log("OK", f"✅ 成功跳转至第 {target_resume_page} 页并截获数据！") page_num = target_resume_page # 读取并解析断点页的数据 body = packet.response.body data = body if isinstance(body, (dict, list)) else json.loads(body) if isinstance(data, dict) and "result" in data: items = data["result"].get("items", []) for item in items: all_clean_items.append(_extract_fields(item)) log("OK", f"第 {page_num} 页清洗完成，累计提取 {len(all_clean_items)} 条数据。") else: log("ERR", "找不到页码输入框，断点跳转失败，将从第 1 页继续！") # ========================================================= # 循环翻页抓取 # ========================================================= while True: # 引入“类人”随机延迟 delay = random.uniform(2.5, 5.5) log("INFO", f"⏳ 模拟真人停顿 {delay:.2f} 秒后，准备点击下一页...") time.sleep(delay) if page_num > 1 and page_num % 50 == 0: long_delay = random.uniform(10.0, 20.0) log("INFO", f"☕️ 已经连续高强度翻了 {page_num} 页，触发风控规避机制，假装喝水休息 {long_delay:.2f} 秒...") time.sleep(long_delay) # 用户指定的下一页按钮 xpath next_btn_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[2]/div[1]/div[2]/div/div[2]/div[1]/button[2]' next_btn = page.ele(next_btn_xpath, timeout=3) if not next_btn: log("ERR", "找不到下一页按钮，尝试强制刷新页面或终止。") break # 检查按钮是否被禁用 class_str = str(next_btn.attr("class")) aria_disabled = next_btn.attr("aria-disabled") is_disabled_attr = next_btn.attr("disabled") is not None if "disabled" in class_str or is_disabled_attr or aria_disabled == "true": log("OK", "🏁 下一页按钮已被禁用，说明已经到达最后一页！") break page_num += 1 log("INFO", f"正在点击【下一页】抓取第 {page_num} 页...") try: next_btn.click() except Exception as e: log("ERR", f"普通点击失败: {e}，尝试 JS 点击...") page.run_js("arguments[0].click();", next_btn) # 等待新一页的 API 响应 packet = page.listen.wait(timeout=15) if not packet: log("ERR", f"第 {page_num} 页请求超时或未触发，中止抓取。") break body = packet.response.body data = body if isinstance(body, (dict, list)) else json.loads(body) if isinstance(data, dict) and "result" in data: items = data["result"].get("items", []) if not items: log("WARN", f"第 {page_num} 页返回了空列表，可能已无数据。") break for item in items: all_clean_items.append(_extract_fields(item)) log("OK", f"第 {page_num} 页清洗完成，累计提取 {len(all_clean_items)} 条数据。") # 每 10 页自动保存一次 if page_num % 10 == 0: with open(SAVE_PATH, "w", encoding="utf-8") as f: json.dump(all_clean_items, f, ensure_ascii=False, indent=2) log("INFO", f"💾 自动存档: 已保存 {len(all_clean_items)} 条记录至本地。") else: log("ERR", f"第 {page_num} 页数据结构异常，中止。") break page.listen.stop() # 最终保存 if all_clean_items: with open(SAVE_PATH, "w", encoding="utf-8") as f: json.dump(all_clean_items, f, ensure_ascii=False, indent=2) log("OK", f"🎉 全部抓取完成！总计成功提取 {len(all_clean_items)} 条数据。") log("OK", f"数据已保存至: {SAVE_PATH}") except Exception as e: log("ERR", f"发生全局异常: {e}") if all_clean_items: rescue_path = OUTPUT_DIR / "issue_receipt_details_RESCUE.json" with open(rescue_path, "w", encoding="utf-8") as f: json.dump(all_clean_items, f, ensure_ascii=False, indent=2) log("INFO", f"🆘 触发异常保存，抢救了 {len(all_clean_items)} 条数据。") finally: try: page.listen.stop() log("INFO", "🛑 已释放浏览器监听资源，保持浏览器开启。") except: pass def _extract_fields(item): """提取所需的字段""" return { "生产任务单号": item.get("productionOrderNo"), "生产物料代码": item.get("productMaterialCode"), "生产物料名称": item.get("productMaterialName"), "生产物料规格": item.get("productMaterialSpecification"), "发料单号": item.get("workOrdersNumber"), "状态": item.get("status"), "物料规格": item.get("materialSpecification"), "物料名称": item.get("materialName"), "物料代码": item.get("materialCode"), "发料数量": item.get("issueNumber"), "已发料数量": item.get("hasIssueNumber"), "金额": item.get("amount"), "成本价": item.get("costPrice"), "发料金额": item.get("issueAmount"), "生产订单备注": item.get("productionOrderRemark"), "明细备注": item.get("detailedRemark"), "单位名称": item.get("unitName"), "仓库名称": item.get("warehouseName"), "行号": item.get("lineNumber"), "发料单备注": item.get("workOrdersRemark"), "执行人名称": item.get("executorUserName"), "物料型号": item.get("materialModel"), "执行时间": item.get("executionTime"), "领料人": item.get("materialsUserName"), "生产物料型号": item.get("productMaterialModel"), "自定义字段": item.get("customField"), "部门代码": item.get("departmentInformationCode"), "部门名称": item.get("departmentInformationName"), "图片文件": item.get("imageFile"), "汇总金额": item.get("issueAmountTotal"), "物料组代码": item.get("materialGroupCode"), "物料组名称": item.get("materialGroupName"), "单价小数位数": item.get("numnberOfReservedDigits"), "单价进位策略": item.get("placeMentStrategy"), "单价": item.get("price"), "销售订单号": item.get("salesOrderCode") } if __name__ == "__main__": fetch_issue_receipt_details()