抓取生产工单，赚取发料异常

browser_login/fetch_issue_receipt_details.py

@@ -0,0 +1,297 @@
+"""
+发料单报表 - 导航测试脚本
+目标: 模拟点击菜单，进入“发料单报表”页面。
+"""
+import sys
+import json
+import time
+import random
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from login import get_page, log
+from config import OUTPUT_DIR
+
+HOME_URL = "https://yunmes.tftykj.cn/"
+API_TARGET = "WorkOrdersDetailed_SearchListAll_Proxy"
+SAVE_PATH = OUTPUT_DIR / "issue_receipt_details_full.json"
+
+def fetch_issue_receipt_details():
+    log("INFO", "=== 🚀 启动发料单报表全量数据抓取 ===")
+    # 强制复用 9222 端口，不关闭浏览器
+    page = get_page(port=9222)
+    
+    all_clean_items = []
+    if SAVE_PATH.exists():
+        try:
+            with open(SAVE_PATH, "r", encoding="utf-8") as f:
+                all_clean_items = json.load(f)
+            log("INFO", f"📦 已加载本地历史存档，包含 {len(all_clean_items)} 条数据。")
+        except Exception as e:
+            log("WARN", f"加载本地存档失败: {e}，将从空列表开始。")
+            all_clean_items = []
+            
+    try:
+        log("INFO", f"正在回到主页起点: {HOME_URL}")
+        page.get(HOME_URL)
+        page.wait.load_start()
+        time.sleep(2)
+        
+        menus = [
+            ("第一层: 业务统计报表", 'xpath://*[@id="app"]/div/div[1]/div[1]/div[2]/div/div[1]/div/div[10]/div/p'),
+            ("第二层: 生产业务报表(推测)", 'xpath:/html/body/div[7]/div/div[1]/div/div[9]/div/p'),
+            ("第三层: 发料单报表", 'xpath:/html/body/div[8]/div/div[1]/div/div[6]/div/p')
+        ]
+        
+        log("INFO", "开始模拟人工点击左侧导航菜单...")
+        for name, xpath in menus:
+            ele = page.ele(xpath, timeout=5)
+            if ele:
+                try: 
+                    ele.click()
+                except: 
+                    page.run_js("arguments[0].click();", ele)
+                time.sleep(1.5)
+            else:
+                log("ERR", f"找不到菜单元素: {name}")
+                return
+                
+        log("OK", "✅ 成功点开发料单报表界面！")
+        
+        # 点击空白处隐藏菜单
+        blank_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[1]/div[2]/div[2]/div/div[1]/div'
+        blank_ele = page.ele(blank_xpath, timeout=3)
+        if blank_ele:
+            try: 
+                blank_ele.click()
+            except: 
+                page.run_js("arguments[0].click();", blank_ele)
+            time.sleep(0.5)
+            
+        log("INFO", f"开启底层数据拦截网: {API_TARGET}")
+        page.listen.start(API_TARGET)
+        
+        # 等待页面自动发起的请求
+        packet = page.listen.wait(timeout=10)
+        
+        if not packet:
+            log("INFO", "尝试寻找并点击页面上的【查询】按钮...")
+            query_btn_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[2]/div[1]/div[1]/div/button[1]/span'
+            query_btn = page.ele(query_btn_xpath, timeout=3)
+            
+            if query_btn:
+                try: query_btn.click()
+                except: page.run_js("arguments[0].click();", query_btn)
+                packet = page.listen.wait(timeout=15)
+                
+        if not packet:
+            log("ERR", "未能拦截到数据请求，可能网络超时或查询未触发。")
+            return
+            
+        # 设定开始抓取的页码，如果因为中断需要断点续传，请修改此变量
+        # 刚才抓到了 95 页，我们需要从 96 页开始继续
+        target_resume_page = 1
+            
+
+        # =========================================================
+        # 第一页数据处理
+        # =========================================================
+        log("OK", f"🎉 成功拦截到第一页数据！HTTP 状态码: {packet.response.status}")
+        body = packet.response.body
+        data = body if isinstance(body, (dict, list)) else json.loads(body)
+        
+        total_count = 0
+        if isinstance(data, dict) and "result" in data:
+            total_count = data["result"].get("totalCount", 0)
+            items = data["result"].get("items", [])
+            log("OK", f"后端报告总条数: {total_count}")
+            
+            # 只有当不是断点续传（即从第1页开始）时，才把第一页的数据加入列表
+            if target_resume_page <= 1:
+                # 由于可能触发断点，如果是重新抓取，这里直接覆盖
+                if not all_clean_items:
+                    for item in items:
+                        all_clean_items.append(_extract_fields(item))
+                    log("OK", f"第一页清洗完成，提取了 {len(items)} 条数据。")
+                else:
+                    log("INFO", f"本地已有数据，跳过第一页保存，走翻页逻辑（注意：发料单可能需要您清空旧存档才能从头抓，这里先保留累加）")
+            else:
+                log("INFO", f"触发断点续传，跳过第一页的数据保存。后端报告总条数: {total_count}")
+        else:
+            log("ERR", "第一页返回的数据结构异常。")
+            return
+            
+        page_num = 1
+        
+        # =========================================================
+        # 断点续传逻辑跳转
+        # =========================================================
+        if target_resume_page > 1:
+            log("INFO", f"🚀 触发断点续传机制！准备直接跳转到第 {target_resume_page} 页...")
+            # 尝试找页码输入框
+            jumper_input_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[2]/div[1]/div[2]/div/div[2]/div[1]/span[3]/div/div//input'
+            input_ele = page.ele(jumper_input_xpath, timeout=5)
+            
+            if not input_ele:
+                jumper_input_xpath = 'xpath://input[@type="number" and @aria-label="页"]'
+                input_ele = page.ele(jumper_input_xpath, timeout=5)
+                
+            if input_ele:
+                input_ele.clear()
+                input_ele.input(str(target_resume_page))
+                time.sleep(0.5)
+                input_ele.input('\n')
+                
+                packet = page.listen.wait(timeout=15)
+                if not packet:
+                    log("ERR", "断点跳转失败，未拦截到目标页的数据请求。")
+                    return
+                    
+                log("OK", f"✅ 成功跳转至第 {target_resume_page} 页并截获数据！")
+                page_num = target_resume_page
+                
+                # 读取并解析断点页的数据
+                body = packet.response.body
+                data = body if isinstance(body, (dict, list)) else json.loads(body)
+                if isinstance(data, dict) and "result" in data:
+                    items = data["result"].get("items", [])
+                    for item in items:
+                        all_clean_items.append(_extract_fields(item))
+                    log("OK", f"第 {page_num} 页清洗完成，累计提取 {len(all_clean_items)} 条数据。")
+            else:
+                log("ERR", "找不到页码输入框，断点跳转失败，将从第 1 页继续！")
+        
+        # =========================================================
+        # 循环翻页抓取
+        # =========================================================
+        while True:
+            # 引入“类人”随机延迟
+            delay = random.uniform(2.5, 5.5)
+            log("INFO", f"⏳ 模拟真人停顿 {delay:.2f} 秒后，准备点击下一页...")
+            time.sleep(delay)
+            
+            if page_num > 1 and page_num % 50 == 0:
+                long_delay = random.uniform(10.0, 20.0)
+                log("INFO", f"☕️ 已经连续高强度翻了 {page_num} 页，触发风控规避机制，假装喝水休息 {long_delay:.2f} 秒...")
+                time.sleep(long_delay)
+            
+            # 用户指定的下一页按钮 xpath
+            next_btn_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[2]/div[1]/div[2]/div/div[2]/div[1]/button[2]'
+            next_btn = page.ele(next_btn_xpath, timeout=3)
+            
+            if not next_btn:
+                log("ERR", "找不到下一页按钮，尝试强制刷新页面或终止。")
+                break
+                
+            # 检查按钮是否被禁用
+            class_str = str(next_btn.attr("class"))
+            aria_disabled = next_btn.attr("aria-disabled")
+            is_disabled_attr = next_btn.attr("disabled") is not None
+            
+            if "disabled" in class_str or is_disabled_attr or aria_disabled == "true":
+                log("OK", "🏁 下一页按钮已被禁用，说明已经到达最后一页！")
+                break
+                
+            page_num += 1
+            log("INFO", f"正在点击【下一页】抓取第 {page_num} 页...")
+            
+            try: 
+                next_btn.click()
+            except Exception as e: 
+                log("ERR", f"普通点击失败: {e}，尝试 JS 点击...")
+                page.run_js("arguments[0].click();", next_btn)
+                
+            # 等待新一页的 API 响应
+            packet = page.listen.wait(timeout=15)
+            if not packet:
+                log("ERR", f"第 {page_num} 页请求超时或未触发，中止抓取。")
+                break
+                
+            body = packet.response.body
+            data = body if isinstance(body, (dict, list)) else json.loads(body)
+            
+            if isinstance(data, dict) and "result" in data:
+                items = data["result"].get("items", [])
+                if not items:
+                    log("WARN", f"第 {page_num} 页返回了空列表，可能已无数据。")
+                    break
+                    
+                for item in items:
+                    all_clean_items.append(_extract_fields(item))
+                log("OK", f"第 {page_num} 页清洗完成，累计提取 {len(all_clean_items)} 条数据。")
+                
+                # 每 10 页自动保存一次
+                if page_num % 10 == 0:
+                    with open(SAVE_PATH, "w", encoding="utf-8") as f:
+                        json.dump(all_clean_items, f, ensure_ascii=False, indent=2)
+                    log("INFO", f"💾 自动存档: 已保存 {len(all_clean_items)} 条记录至本地。")
+            else:
+                log("ERR", f"第 {page_num} 页数据结构异常，中止。")
+                break
+                
+        page.listen.stop()
+        
+        # 最终保存
+        if all_clean_items:
+            with open(SAVE_PATH, "w", encoding="utf-8") as f:
+                json.dump(all_clean_items, f, ensure_ascii=False, indent=2)
+            log("OK", f"🎉 全部抓取完成！总计成功提取 {len(all_clean_items)} 条数据。")
+            log("OK", f"数据已保存至: {SAVE_PATH}")
+            
+    except Exception as e:
+        log("ERR", f"发生全局异常: {e}")
+        if all_clean_items:
+            rescue_path = OUTPUT_DIR / "issue_receipt_details_RESCUE.json"
+            with open(rescue_path, "w", encoding="utf-8") as f:
+                json.dump(all_clean_items, f, ensure_ascii=False, indent=2)
+            log("INFO", f"🆘 触发异常保存，抢救了 {len(all_clean_items)} 条数据。")
+    finally:
+        try:
+            page.listen.stop()
+            log("INFO", "🛑 已释放浏览器监听资源，保持浏览器开启。")
+        except:
+            pass
+
+def _extract_fields(item):
+    """提取所需的字段"""
+    return {
+        "生产任务单号": item.get("productionOrderNo"),
+        "生产物料代码": item.get("productMaterialCode"),
+        "生产物料名称": item.get("productMaterialName"),
+        "生产物料规格": item.get("productMaterialSpecification"),
+        "发料单号": item.get("workOrdersNumber"),
+        "状态": item.get("status"),
+        "物料规格": item.get("materialSpecification"),
+        "物料名称": item.get("materialName"),
+        "物料代码": item.get("materialCode"),
+        "发料数量": item.get("issueNumber"),
+        "已发料数量": item.get("hasIssueNumber"),
+        "金额": item.get("amount"),
+        "成本价": item.get("costPrice"),
+        "发料金额": item.get("issueAmount"),
+        "生产订单备注": item.get("productionOrderRemark"),
+        "明细备注": item.get("detailedRemark"),
+        "单位名称": item.get("unitName"),
+        "仓库名称": item.get("warehouseName"),
+        "行号": item.get("lineNumber"),
+        "发料单备注": item.get("workOrdersRemark"),
+        "执行人名称": item.get("executorUserName"),
+        "物料型号": item.get("materialModel"),
+        "执行时间": item.get("executionTime"),
+        "领料人": item.get("materialsUserName"),
+        "生产物料型号": item.get("productMaterialModel"),
+        "自定义字段": item.get("customField"),
+        "部门代码": item.get("departmentInformationCode"),
+        "部门名称": item.get("departmentInformationName"),
+        "图片文件": item.get("imageFile"),
+        "汇总金额": item.get("issueAmountTotal"),
+        "物料组代码": item.get("materialGroupCode"),
+        "物料组名称": item.get("materialGroupName"),
+        "单价小数位数": item.get("numnberOfReservedDigits"),
+        "单价进位策略": item.get("placeMentStrategy"),
+        "单价": item.get("price"),
+        "销售订单号": item.get("salesOrderCode")
+    }
+
+if __name__ == "__main__":
+    fetch_issue_receipt_details()