""" 发料单报表 - 智能增量同步脚本 (从第一页开始抓，遇到旧数据即停) 目标: 1. 自动连接本地 SQLite 数据库查询是否存在某条记录。 2. 进入 ERP 系统截获发料单数据，由于新数据都在第一页，我们从第 1 页开始抓。 3. 逐条对比，如果发现某页的数据在本地已经存在，则认为增量部分已经抓取完毕，提前终止。 4. 将新增数据存入 SQLite。 """ import sys import json import time import math import random import sqlite3 from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from login import get_page, log from config import DB_PATH HOME_URL = "https://yunmes.tftykj.cn/" API_TARGET = "WorkOrdersDetailed_SearchListAll_Proxy" def get_local_count(conn): """获取本地数据库已有的总记录数""" cursor = conn.cursor() cursor.execute("SELECT COUNT(*) FROM issue_receipt_details") return cursor.fetchone()[0] def item_exists(cursor, item): """判断某条发料明细是否已在数据库中存在（基于发料单号+行号+物料代码组合判断）""" wo_number = item.get("workOrdersNumber") line_no = item.get("lineNumber") mat_code = item.get("materialCode") # 增加一个容错判断，如果其中有 None 就不当作重复 if not wo_number or not line_no or not mat_code: return False cursor.execute(''' SELECT 1 FROM issue_receipt_details WHERE work_orders_number = ? AND line_number = ? AND material_code = ? ''', (wo_number, line_no, mat_code)) return cursor.fetchone() is not None def _extract_fields(item): """提取所需的字段""" return { "生产任务单号": item.get("productionOrderNo"), "生产物料代码": item.get("productMaterialCode"), "生产物料名称": item.get("productMaterialName"), "生产物料规格": item.get("productMaterialSpecification"), "发料单号": item.get("workOrdersNumber"), "状态": item.get("status"), "物料规格": item.get("materialSpecification"), "物料名称": item.get("materialName"), "物料代码": item.get("materialCode"), "发料数量": item.get("issueNumber"), "已发料数量": item.get("hasIssueNumber"), "金额": item.get("amount"), "成本价": item.get("costPrice"), "发料金额": item.get("issueAmount"), "生产订单备注": item.get("productionOrderRemark"), "明细备注": item.get("detailedRemark"), "单位名称": item.get("unitName"), "仓库名称": item.get("warehouseName"), "行号": item.get("lineNumber"), "发料单备注": item.get("workOrdersRemark"), "执行人名称": item.get("executorUserName"), "物料型号": item.get("materialModel"), "执行时间": item.get("executionTime"), "领料人": item.get("materialsUserName"), "生产物料型号": item.get("productMaterialModel"), "自定义字段": item.get("customField"), "部门代码": item.get("departmentInformationCode"), "部门名称": item.get("departmentInformationName"), "图片文件": item.get("imageFile"), "汇总金额": item.get("issueAmountTotal"), "物料组代码": item.get("materialGroupCode"), "物料组名称": item.get("materialGroupName"), "单价小数位数": item.get("numnberOfReservedDigits"), "单价进位策略": item.get("placeMentStrategy"), "单价": item.get("price"), "销售订单号": item.get("salesOrderCode") } def fetch_issue_receipt_incremental(): log("INFO", "=== 🚀 启动发料单报表 - 智能增量同步 (首屏更新模式) ===") if not DB_PATH.exists(): log("ERR", f"找不到数据库文件: {DB_PATH}，请先执行全量导入！") return conn = sqlite3.connect(DB_PATH) local_count = get_local_count(conn) log("INFO", f"📦 本地数据库当前总计: {local_count} 条数据") page = get_page(port=9222) try: log("INFO", f"正在回到主页起点: {HOME_URL}") page.get(HOME_URL) page.wait.load_start() time.sleep(2) menus = [ ("第一层: 业务统计报表", 'xpath://*[@id="app"]/div/div[1]/div[1]/div[2]/div/div[1]/div/div[10]/div/p'), ("第二层: 生产业务报表(推测)", 'xpath:/html/body/div[7]/div/div[1]/div/div[9]/div/p'), ("第三层: 发料单报表", 'xpath:/html/body/div[8]/div/div[1]/div/div[6]/div/p') ] log("INFO", "模拟点击左侧导航菜单...") for name, xpath in menus: ele = page.ele(xpath, timeout=5) if ele: try: ele.click() except: page.run_js("arguments[0].click();", ele) time.sleep(1.5) else: log("ERR", f"找不到菜单元素: {name}") return log("OK", "✅ 成功点开发料单报表界面！") # 隐藏菜单 blank_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[1]/div[2]/div[2]/div/div[1]/div' blank_ele = page.ele(blank_xpath, timeout=3) if blank_ele: try: blank_ele.click() except: page.run_js("arguments[0].click();", blank_ele) time.sleep(0.5) log("INFO", f"开启底层数据拦截网: {API_TARGET}") page.listen.start(API_TARGET) packet = page.listen.wait(timeout=10) if not packet: query_btn_xpath = 'xpath://*[@id="app"]/div/div[1]/div[2]/div[2]/div[1]/div[1]/div/button[1]/span' query_btn = page.ele(query_btn_xpath, timeout=3) if query_btn: try: query_btn.click() except: page.run_js("arguments[0].click();", query_btn) packet = page.listen.wait(timeout=15) if not packet: log("ERR", "未能拦截到第一页数据，无法获取线上总条数。") return body = packet.response.body data = body if isinstance(body, (dict, list)) else json.loads(body) remote_count = 0 if isinstance(data, dict) and "result" in data: remote_count = data["result"].get("totalCount", 0) log("INFO", f"🌐 线上 ERP 系统当前总条数: {remote_count} 条") if remote_count == local_count: log("OK", "🎉 线上条数与本地一致，数据已是最新状态，无需抓取！") return new_items_count = remote_count - local_count if new_items_count > 0: log("INFO", f"🔥 发现大致 {new_items_count} 条新增数据！准备从第 1 页开始扫描录入...") else: log("INFO", f"⚠️ 线上条数 ({remote_count}) 少于本地条数 ({local_count})，可能存在数据删除。仍将扫描第一页验证更新。") # ========================================================= # 开始处理第一页，并循环往后翻，直到遇到重复数据 # ========================================================= current_page = 1 cursor = conn.cursor() total_inserted = 0 # 第一次的数据已经在上面的 packet 里了，直接处理 first_page_data = data while True: should_stop = False inserted_this_page = 0 if isinstance(first_page_data, dict) and "result" in first_page_data: items = first_page_data["result"].get("items", []) if not items: log("WARN", f"第 {current_page} 页返回了空列表，已无数据。") break # 打印第一条数据的信息，用于调试 if items: first_item = items[0] log("INFO", f"🔍 正在检查本页第一条数据: 发料单 {first_item.get('workOrdersNumber')} 行号 {first_item.get('lineNumber')} 物料 {first_item.get('materialCode')}") for raw_item in items: # 1. 检查是否存在 if item_exists(cursor, raw_item): # 发料单的新数据都在最前面。当我们遇到一条已经在数据库里的数据时， # 说明这之前的数据都是新的，这之后的数据肯定都抓过了，直接停止。 log("INFO", f"🛑 在第 {current_page} 页发现本地已存在的记录 (发料单: {raw_item.get('workOrdersNumber')} 行号: {raw_item.get('lineNumber')} 物料: {raw_item.get('materialCode')})，增量扫描结束！") should_stop = True break # 2. 如果不存在，提取并插入 item = _extract_fields(raw_item) cursor.execute(''' INSERT INTO issue_receipt_details ( production_order_no, product_material_code, product_material_name, product_material_specification, work_orders_number, status, material_specification, material_name, material_code, issue_number, has_issue_number, amount, cost_price, issue_amount, production_order_remark, detailed_remark, unit_name, warehouse_name, line_number, work_orders_remark, executor_user_name, material_model, execution_time, materials_user_name, product_material_model, custom_field, department_information_code, department_information_name, image_file, issue_amount_total, material_group_code, material_group_name, numnber_of_reserved_digits, place_ment_strategy, price, sales_order_code ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ''', ( item.get("生产任务单号"), item.get("生产物料代码"), item.get("生产物料名称"), item.get("生产物料规格"), item.get("发料单号"), item.get("状态"), item.get("物料规格"), item.get("物料名称"), item.get("物料代码"), item.get("发料数量"), item.get("已发料数量"), item.get("金额"), item.get("成本价"), item.get("发料金额"), item.get("生产订单备注"), item.get("明细备注"), item.get("单位名称"), item.get("仓库名称"), item.get("行号"), item.get("发料单备注"), item.get("执行人名称"), item.get("物料型号"), item.get("执行时间"), item.get("领料人"), item.get("生产物料型号"), item.get("自定义字段"), item.get("部门代码"), item.get("部门名称"), item.get("图片文件"), item.get("汇总金额"), item.get("物料组代码"), item.get("物料组名称"), item.get("单价小数位数"), item.get("单价进位策略"), item.get("单价"), item.get("销售订单号") )) inserted_this_page += 1 total_inserted += 1 conn.commit() log("OK", f"第 {current_page} 页处理完毕，成功插入 {inserted_this_page} 条新数据。") if should_stop: break else: log("ERR", f"第 {current_page} 页数据结构异常，中止。") break # 如果没遇到旧数据，继续点击下一页 delay = random.uniform(1.5, 3.5) log("INFO", f"⏳ 停顿 {delay:.2f} 秒后点击下一页...") time.sleep(delay) next_btn = None for _ in range(3): next_btn = page.ele('xpath://button[contains(@class, "btn-next")]', timeout=3) if next_btn: break time.sleep(1) if not next_btn: next_btn = page.ele('xpath://i[contains(@class, "el-icon-arrow-right")]/parent::button', timeout=3) if next_btn: # 检查按钮是否被禁用 class_str = str(next_btn.attr("class")) aria_disabled = next_btn.attr("aria-disabled") is_disabled_attr = next_btn.attr("disabled") is not None if "disabled" in class_str or is_disabled_attr or aria_disabled == "true": log("OK", "🏁 下一页按钮已被禁用，已经翻到最后一页。") break try: next_btn.click() except: page.run_js("arguments[0].click();", next_btn) packet = page.listen.wait(timeout=15) if not packet: log("ERR", f"第 {current_page + 1} 页请求超时！") break # 为下一轮循环准备数据 body = packet.response.body first_page_data = body if isinstance(body, (dict, list)) else json.loads(body) else: log("ERR", "重试 3 次后仍然找不到下一页按钮！") break current_page += 1 log("OK", f"🎉 发料单增量同步大功告成！总计新增了 {total_inserted} 条记录入库！") except Exception as e: log("ERR", f"发生全局异常: {e}") finally: if 'conn' in locals() and conn: conn.close() if 'page' in locals() and page: try: page.listen.stop() except Exception: pass if __name__ == "__main__": fetch_issue_receipt_incremental()