feat(xkgroup): 采集器切换到新站 120.70.100.128:8096

- 新站为 layui 服务端渲染(无 JSON API), 抓取层重写为 HTML 列表页解析
  (GET /front/notice/cgxx?category=bid|procure|auction 分页, pageNo 从 0 起)
- 旧站 kdcloud getPortalNoticeData API 已失效(302 到登录页)
- 类型标签映射: 默认抓商机类(招标/采购/竞价公告)+流标类(废标/竞价流拍),
  --all-types 额外抓结果/变更/其他类, 对应旧版 C+D / +B 语义
- 白名单过滤改为标题+发布单位联合匹配(新站发布单位多为代理机构,
  按单位匹配会漏掉白名单组织自己的公告)
- 详情增强改用 requests(新站纯 SSR 无需 Playwright): 存 HTML 源码 +
  提取金额/截止/地区 + 下载附件 PDF 直链
- 输出状态文件格式不变, CrawlerService 入库链路零改动(本地+生产端到端验证通过)
- 新增 --max-pages 参数(默认每板块 5 页)

scripts/tender-fetcher/fetch-xkgroup.py

@@ -1,23 +1,35 @@
 #!/usr/bin/env python3
 """
 新矿集团供应链平台 公告抓取工具
-数据源：https://xkgroup.kdcloud.com/isv/q5m9/dist/srmportal.html
+数据源：http://120.70.100.128:8096/（新站，2026-10 切换）
-API：getPortalNoticeData (REST, 无需登录即可访问 guest 数据)
+  - 纯服务端渲染（layui），无 JSON API，抓 HTML 列表页解析
+  - 列表页：/front/notice/cgxx?category=bid|procure|auction（GET 表单分页，pageNo 从 0 起，每页 10 条）
+  - 详情页：/front/index/v_notice_view?id=<32hex>（含正文 + 附件 PDF 直链）
 
-抓取类型：
+板块类型标签全集：
-  C - 寻源公告（核心采购公告）
+  bid     招标公告 / 变更&澄清公告 / 资格预审公告 / 中标候选人公示 / 中标公告 / 废标公告
-  D - 流标公告
+  procure 采购公告 / 变更公告 / 结果公示 / 其他公告
-  B - 结果公示
+  auction 竞价公告 / 竞价流拍
+
+默认抓取"商机类 + 流标类"（对应旧站 C 寻源 + D 流标语义）：
+  商机类 = 招标公告/采购公告/竞价公告/资格预审公告
+  流标类 = 废标公告/竞价流拍
+--all-types 额外抓结果/其他类（中标候选人公示/中标公告/变更&澄清/结果公示/其他公告）
 
 用法：
-  python3 fetch-xkgroup.py                     # 差量更新（默认抓取昨天的寻源公告）
+  python3 fetch-xkgroup.py                     # 差量更新（默认回溯到上次运行/7 天内）
   python3 fetch-xkgroup.py --all-types         # 抓取所有公告类型
-  python3 fetch-xkgroup.py --from 2026-05-01   # 指定起始日期
+  python3 fetch-xkgroup.py --from 2026-10-01   # 指定起始日期
   python3 fetch-xkgroup.py --force             # 强制重新抓取（不跳过已下载）
-  python3 fetch-xkgroup.py --keyword 水泥       # 按关键词过滤
+  python3 fetch-xkgroup.py --keyword 水泥       # 按关键词过滤（标题/发布单位）
+  python3 fetch-xkgroup.py --max-pages 10      # 每板块最大翻页数（默认 5 页 = 50 条）
+
+状态文件 xkgroup_downloaded.json 格式与旧版一致，CrawlerService.runGenericScript 依赖字段：
+  id/title/org/publish_time/detail_url/amount/deadline/region/region_detail/downloaded_at
 """
 
 import argparse
+import html as html_lib
 import json
 import os
 import re
@@ -31,17 +43,19 @@ except ImportError:
     sys.exit(1)
 
 # ============ 配置区 ============
-BASE_URL = "https://xkgroup.kdcloud.com/kapi/app/srm/srmwebapi"
+BASE_URL = "http://120.70.100.128:8096"
-ACCOUNT_ID = "2066845847806639104"
+LIST_URL = f"{BASE_URL}/front/notice/cgxx"
-PAGE_SIZE = 50
+PAGE_SIZE = 10          # 新站固定每页 10 条
+MAX_PAGES_DEFAULT = 5   # 每板块默认最多翻 5 页（日增量远小于此，历史回填可加大）
 
-# 公告类型映射
+# 板块与类型标签
-NOTICE_TYPES = {
+CATEGORIES = ["bid", "procure", "auction"]
-    "C": "寻源公告",
+
-    "D": "流标公告",
+# 商机类 + 流标类（默认抓取，对应旧版 C+D）
-    "B": "结果公示",
+BUSINESS_TAGS = {"招标公告", "采购公告", "竞价公告", "资格预审公告"}
-}
+FLOWN_TAGS = {"废标公告", "竞价流拍"}
-DEFAULT_TYPES = ["C", "D"]  # 抓取寻源公告 + 流标公告
+# 结果/其他类（--all-types 才抓）
+RESULT_TAGS = {"中标候选人公示", "中标公告", "变更&澄清公告", "变更公告", "结果公示", "其他公告"}
 
 # 多次询价标题正则 (匹配 二次/三次/四次/.../九次)
 RETRY_PATTERN = re.compile(r"[二三四五六七八九]次")
@@ -56,77 +70,99 @@ OUTPUT_BASE = os.path.expanduser("~/desktop/招标公告")
 
 HEADERS = {
     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
-    "Accept": "application/json, text/plain, */*",
+    "Accept": "text/html,application/xhtml+xml,*/*",
-    "Referer": "https://xkgroup.kdcloud.com/isv/q5m9/dist/srmportal.html",
+    "Referer": f"{BASE_URL}/",
 }
 
-# ============ 核心函数 ============
+# ============ 抓取层（新站 HTML 解析） ============
 
 
-def fetch_notices(notice_type, page=1, size=PAGE_SIZE):
+def fetch_list_page(category, page_no, timeout=30):
-    """调用 API 获取公告列表"""
+    """抓取列表页 HTML（GET 表单，pageNo 从 0 起）"""
-    params = {
+    resp = requests.get(
-        "page": page,
+        LIST_URL,
-        "size": size,
+        params={"category": category, "pageNo": page_no},
-        "type": notice_type,
+        headers=HEADERS,
-        "apiname": "getPortalNoticeData",
+        timeout=timeout,
-        "methodname": "getPortalNoticeData",
+    )
-        "accountId": ACCOUNT_ID,
-    }
-    resp = requests.get(BASE_URL, params=params, headers=HEADERS, timeout=30)
     resp.raise_for_status()
-    return resp.json()
+    return resp.text
 
 
-def fetch_all_notices(notice_type):
+def parse_list_html(html_text):
-    """翻页获取指定类型的所有公告"""
+    """解析列表页 tbody#listTbody 中的公告条目
+
+    返回 [{"id", "url_path", "title", "tag", "publish_date", "org"}]
+    """
+    items = []
+    tbody_m = re.search(r'<tbody id="listTbody">([\s\S]*?)</tbody>', html_text)
+    rows = re.split(r"<tr>", tbody_m.group(1)) if tbody_m else []
+    for row in rows[1:]:  # 第一段是 <thead> 残留
+        link_m = re.search(
+            r'<a href="(/front/index/v_notice_view\?id=([a-f0-9]{32})[^"]*)"[^>]*title="([^"]+)"', row
+        )
+        if not link_m:
+            continue
+        tag_m = re.search(r'<span class="tag[^"]*">([^<]+)</span>', row)
+        date_m = re.search(r"<td>\s*(20\d{2}-\d{2}-\d{2})\s*</td>", row)
+        tds = re.findall(r"<td>([^<]*)</td>", row)
+        items.append(
+            {
+                "id": link_m.group(2),
+                "url_path": html_lib.unescape(link_m.group(1)),
+                "title": html_lib.unescape(link_m.group(3)).strip(),
+                "tag": tag_m.group(1).strip() if tag_m else "",
+                "publish_date": date_m.group(1) if date_m else "",
+                "org": tds[-1].strip() if tds else "",
+            }
+        )
+    return items
+
+
+def fetch_all_notices(category, date_from, max_pages):
+    """翻页抓取指定板块的公告列表（按发布时间倒序，翻到早于 date_from 即停）"""
     all_items = []
-    page = 1
+    for page_no in range(max_pages):
-
+        html_text = fetch_list_page(category, page_no)
-    while True:
+        items = parse_list_html(html_text)
-        data = fetch_notices(notice_type, page=page)
+        if not items:
-        if not data.get("success"):
-            print(f"  ❌ API 请求失败: {data.get('errorCode', 'unknown')}")
             break
-
-        items = data.get("data", {}).get("noticedata", [])
-        total = data.get("data", {}).get("total", 0)
-
         all_items.extend(items)
-
+        print(f"    第 {page_no + 1} 页: {len(items)} 条")
-        total_page = (total + PAGE_SIZE - 1) // PAGE_SIZE
+        # 倒序排列：本页最旧一条都早于起始日期，无需继续翻页
-        print(f"    第 {page}/{total_page} 页: {len(items)} 条")
+        dates = [it["publish_date"] for it in items if it["publish_date"]]
-
+        if dates and max(dates) < str(date_from):
-        if len(items) < PAGE_SIZE or page >= total_page:
             break
-
-        page += 1
-
     return all_items
 
 
-def normalize_notice(item, notice_type_label):
+def fetch_detail(notice_id, timeout=30):
-    """统一公告数据格式"""
+    """抓取详情页，返回 (html_text, body_text, attach_list)
-    notice_id = item.get("noticeId", "")
-    pub_time = item.get("publishtime", "")
 
-    # 解析状态
+    body_text 为去标签后的正文纯文本（供金额/截止/地区提取），
-    status = item.get("status", "")
+    attach_list 为详情页附件直链列表（PDF/Word/Excel）。
-    norm_status = status if status else "未知"
+    """
+    url = f"{BASE_URL}/front/index/v_notice_view?id={notice_id}"
+    resp = requests.get(url, headers=HEADERS, timeout=timeout)
+    resp.raise_for_status()
+    html_text = resp.text
+    cleaned = re.sub(r"<script[\s\S]*?</script>|<style[\s\S]*?</style>", " ", html_text)
+    body_text = html_lib.unescape(re.sub(r"<[^>]+>", " ", cleaned))
+    body_text = re.sub(r"[ \t\r]+", " ", body_text)
+    attach_list = re.findall(r'href="(https?://[^"]+\.(?:pdf|docx?|xlsx?)[^"]*)"', cleaned)
+    return html_text, body_text, attach_list
 
-    return {
+
-        "id": notice_id,
+def download_attachment(url, save_path, timeout=60):
-        "title": item.get("noticename", ""),
+    """下载附件文件（PDF/Word/Excel 直链）"""
-        "org": item.get("orgname", ""),
+    resp = requests.get(url, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=timeout, stream=True)
-        "org_number": item.get("orgnumber", ""),
+    resp.raise_for_status()
-        "type": notice_type_label,
+    with open(save_path, "wb") as f:
-        "type_code": item.get("type", ""),
+        for chunk in resp.iter_content(chunk_size=65536):
-        "status": norm_status,
+            f.write(chunk)
-        "publish_time": pub_time,
+
-        "detail_url": item.get("url", ""),
+
-        "mobile_url": item.get("mobileurl", ""),
+# ============ 业务过滤与字段提取（沿用旧版） ============
-        "source": "新矿集团",
-    }
 
 
 def match_date_range(publish_time_str, date_from, date_to):
@@ -200,21 +236,22 @@ def generate_markdown(notices, output_dir, date_range):
         f"# 新矿集团供应链平台 公告清单\n",
         f"**抓取时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')}  |  **日期范围**: {date_range}",
         f"**公告总数**: {len(notices)} 条\n",
-        f"> 数据来源：{BASE_URL} | 平台：新矿集团供应链平台\n",
+        f"> 数据来源：{BASE_URL} | 平台：新矿集团供应链平台（新站）\n",
         "---\n",
     ]
 
     for i, n in enumerate(notices, 1):
-        lines.extend([
+        lines.extend(
+            [
                 f"## {i}. {n['title']}",
-            f"- **发布组织**: {n['org']}",
+                f"- **发布单位**: {n['org']}",
                 f"- **类型**: {n['type']}",
-            f"- **状态**: {n['status']}",
                 f"- **发布时间**: {n['publish_time']}",
                 f"- **详情**: [查看]({n['detail_url']})",
                 "",
                 "---\n",
-        ])
+            ]
+        )
 
     content = "\n".join(lines)
     date_str = date_range.replace("~", "-")
@@ -228,167 +265,6 @@ def generate_markdown(notices, output_dir, date_range):
     return filepath
 
 
-def run():
-    parser = argparse.ArgumentParser(description="新矿集团供应链平台 公告抓取")
-    parser.add_argument("--from", dest="from_date", help="起始日期 YYYY-MM-DD")
-    parser.add_argument("--to", dest="to_date", help="截止日期 YYYY-MM-DD")
-    parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
-    parser.add_argument("--force", action="store_true", help="强制重新抓取")
-    parser.add_argument("--keyword", default="", help="关键词过滤")
-    args = parser.parse_args()
-
-    today = date.today()
-
-    # 加载历史已下载记录（跨天去重）
-    today_str = today.strftime("%Y-%m-%d")
-    accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
-
-    if args.from_date:
-        date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
-    elif not args.force and last_run_date:
-        date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
-    else:
-        date_from = today - timedelta(days=7)
-
-    if args.to_date:
-        date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
-    else:
-        date_to = today
-
-    date_range = f"{date_from}~{date_to}"
-    days_span = (date_to - date_from).days + 1
-
-    # 公告类型
-    notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
-
-    # 输出目录
-    output_dir = os.path.join(OUTPUT_BASE, today_str)
-    os.makedirs(output_dir, exist_ok=True)
-
-    # 子目录：html（源码） + pdf（转存）
-    html_dir = os.path.join(output_dir, "html")
-    pdf_dir = os.path.join(output_dir, "pdf")
-    os.makedirs(html_dir, exist_ok=True)
-    os.makedirs(pdf_dir, exist_ok=True)
-
-    # 状态文件（使用累积状态做去重）
-    state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
-    state = accumulated_state
-
-    all_notices = []
-    new_count = 0
-    skip_count = 0
-    filtered_count = 0
-
-    print(f"===== 新矿集团供应链平台 公告抓取 =====")
-    print(f"日期范围: {date_from} ~ {date_to} ({days_span}天)")
-    print(f"公告类型: {', '.join([NOTICE_TYPES.get(t, t) for t in notice_types])}")
-    print()
-
-    for ntype in notice_types:
-        label = NOTICE_TYPES.get(ntype, ntype)
-        print(f"[{label}] 正在获取...")
-
-        items = fetch_all_notices(ntype)
-        print(f"  ✅ API 返回 {len(items)} 条")
-
-        type_new = 0
-        type_skip = 0
-        for item in items:
-            notice = normalize_notice(item, label)
-
-            # 日期过滤
-            if not match_date_range(notice["publish_time"], date_from, date_to):
-                filtered_count += 1
-                continue
-
-            # 关键词过滤
-            if args.keyword:
-                title = notice["title"]
-                org = notice["org"]
-                if args.keyword not in title and args.keyword not in org:
-                    filtered_count += 1
-                    continue
-
-            # 多次询价标题过滤 (二次/三次/四次...)
-            if not RETRY_PATTERN.search(notice["title"]):
-                filtered_count += 1
-                continue
-
-            # 发布组织白名单
-            if not any(org_name in notice["org"] for org_name in ORG_WHITELIST):
-                filtered_count += 1
-                continue
-
-            notice_id = notice["id"]
-
-            # 去重
-            if not args.force and notice_id in state:
-                skip_count += 1
-                type_skip += 1
-                continue
-
-            # 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
-            title_region = extract_region_from_text(notice["title"], notice["org"])
-            all_notices.append(notice)
-            state[notice_id] = {
-                "title": notice["title"],
-                "org": notice["org"],
-                "publish_time": notice["publish_time"],
-                "type": notice["type"],
-                "detail_url": notice["detail_url"],
-                "downloaded_at": datetime.now().isoformat(),
-                "region": title_region,
-            }
-            new_count += 1
-            type_new += 1
-
-        print(f"      新增: {type_new} | 跳过: {type_skip}")
-        print()
-
-    # 保存状态（含 lastRun 时间戳，用于下次增量爬取）
-    state["lastRun"] = datetime.now().isoformat()
-    save_state(state_file, state)
-
-    # 为缺少金额的记录从详情页提取
-    extract_amounts_for_existing(state, output_dir)
-
-    # 生成清单
-    if all_notices:
-        generate_markdown(all_notices, output_dir, date_range)
-
-    # 详情页转存 PDF + HTML（新公告 + 已有状态中缺文件的）
-    all_pending = list(all_notices)
-    if not args.force:
-        for notice_id, info in list(state.items()):
-            if not isinstance(info, dict):
-                continue
-            pdf_key = info.get("pdf", "")
-            html_key = info.get("html", "")
-            pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
-            html_ok = html_key and os.path.exists(os.path.join(output_dir, html_key))
-            if pdf_ok and html_ok:
-                continue
-            all_pending.append({
-                "id": notice_id,
-                "title": info.get("title", ""),
-                "org": info.get("org", ""),
-                "publish_time": info.get("publish_time", ""),
-                "type": info.get("type", ""),
-                "mobile_url": f"https://xkgroup.kdcloud.com/mobile.html?form=mobsp_quonotice_view&billid={notice_id}&userId=guest&accountId=2066845847806639104",
-                "detail_url": f"https://xkgroup.kdcloud.com/index.html?formId=quo_notice&noticeId={notice_id}&userId=guest&accountId=2066845847806639104",
-            })
-
-    if all_pending:
-        download_pdfs(all_pending, output_dir, html_dir, pdf_dir, state, state_file, force=args.force)
-    else:
-        print("  ℹ️ 指定日期范围内无新公告")
-
-    print(f"\n========== 完成 ==========")
-    print(f"✅ 新增: {new_count} | ⏩ 跳过: {skip_count} | 🔍 日期过滤: {filtered_count}")
-    print(f"📁 保存目录: {output_dir}")
-
-
 def extract_deadline_from_text(text):
     """从公告正文中提取截止日期（采购文件获取截止时间）"""
     if not text:
@@ -542,205 +418,219 @@ def extract_amount_from_text(text):
     return ""
 
 
-def extract_amounts_for_existing(state, output_dir):
+def process_details(notices, output_dir, html_dir, pdf_dir, state, state_file):
-    """为已抓取但缺少金额的公告，访问详情页提取金额"""
+    """对新公告做详情页增强：存 HTML 源码 + 提取金额/截止/地区 + 下载附件 PDF
-    try:
+
-        import asyncio
+    新站详情页是纯 SSR HTML，requests 直接可取（不再依赖 Playwright）。
-        from playwright.async_api import async_playwright
+    单条失败仅告警，不阻断。
-    except ImportError:
+    """
-        print("  ⚠️ Playwright 未安装，跳过金额提取")
+    if not notices:
         return
-
+    print(f"\n[详情增强] {len(notices)} 条新公告，提取金额/截止/地区 + 存档...")
-    # 筛选缺少金额的记录
-    pending = []
-    for notice_id, info in state.items():
-        if notice_id == "lastRun":
-            continue
-        if not isinstance(info, dict):
-            continue
-        if info.get("amount"):
-            continue
-        pending.append(notice_id)
-
-    if not pending:
-        print("  ✅ 所有记录已有金额信息")
-        return
-
-    print(f"\n[金额提取] {len(pending)} 条记录缺少金额，正在从详情页提取...")
-
-    async def _run():
-        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
     success = 0
-
-            for i, notice_id in enumerate(pending):
-                info = state[notice_id]
-                url = f"https://xkgroup.kdcloud.com/mobile.html?form=mobsp_quonotice_view&billid={notice_id}&userId=guest&accountId=2066845847806639104"
-                print(f"  [{i+1}/{len(pending)}] {info.get('title', '')[:40]}...", end=" ", flush=True)
-
-                try:
-                    page = await browser.new_page()
-                    page.set_default_timeout(20000)
-                    await page.goto(url, wait_until="domcontentloaded", timeout=20000)
-                    await page.wait_for_timeout(2000)
-                    text = await page.evaluate("() => document.body.innerText")
-                    await page.close()
-
-                    amount = extract_amount_from_text(text)
-                    deadline = extract_deadline_from_text(text)
-                    region = extract_region_from_text(text, info.get("org", ""))
-                    region_detail = extract_region_detail_from_text(text)
-                    updated = False
-                    if amount:
-                        state[notice_id]["amount"] = amount
-                        updated = True
-                    if deadline:
-                        state[notice_id]["deadline"] = deadline
-                        updated = True
-                    if region:
-                        state[notice_id]["region"] = region
-                        updated = True
-                    if region_detail:
-                        state[notice_id]["region_detail"] = region_detail
-                        updated = True
-                    if updated:
-                        success += 1
-                        print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
-                    else:
-                        print("⏭️ 未找到金额/截止日期/地区")
-                except Exception as e:
-                    print(f"❌ {e}")
-
-            await browser.close()
-            return success
-
-    import asyncio
-    try:
-        success = asyncio.run(_run())
-    except RuntimeError:
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        success = loop.run_until_complete(_run())
-        loop.close()
-
-    if success:
-        save_state(os.path.join(output_dir, "xkgroup_downloaded.json"), state)
-        print(f"  💰 金额提取完成: {success} 条")
-
-
-def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file=None, force=False):
-    """将新矿集团公告详情页转存为 PDF"""
-    try:
-        import asyncio
-        from playwright.async_api import async_playwright
-    except ImportError:
-        print("  ⚠️ Playwright 未安装，跳过 PDF 下载")
-        return
-
-    print("\n[PDF] 正在下载详情页...")
-
-    async def _run():
-        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
-            success = 0
-            fail = 0
-
     for i, notice in enumerate(notices):
-                notice_id = notice.get("id", "")
+        notice_id = notice["id"]
-                # 手机版页面含完整内容，桌面版只有摘要
+        print(f"  [{i+1}/{len(notices)}] {notice['title'][:40]}...", end=" ", flush=True)
-                pdf_url = notice.get("mobile_url", "") or notice.get("detail_url", "")
+        try:
-
+            html_text, body_text, attach_list = fetch_detail(notice_id)
-                if not notice_id or not pdf_url:
-                    continue
 
             pub_date = notice.get("publish_time", "")[:10]
             safe_title = re.sub(r"[\\/:*?\"<>|]", "_", notice["title"][:40])
             base_name = f"{pub_date}_{notice_id}_{safe_title}"
             html_name = f"{base_name}.html"
-                pdf_name = f"{base_name}.pdf"
+            with open(os.path.join(html_dir, html_name), "w", encoding="utf-8") as f:
-                html_path = os.path.join(html_dir, html_name)
+                f.write(html_text)
-                pdf_path = os.path.join(pdf_dir, pdf_name)
+            state[notice_id]["html"] = f"html/{html_name}"
-                rel_pdf = f"pdf/{pdf_name}"
 
-                if not force and os.path.exists(pdf_path):
+            # 字段提取
-                    continue
+            amount = extract_amount_from_text(body_text)
-
+            deadline = extract_deadline_from_text(body_text)
-                print(f"  [{i+1}/{len(notices)}] {notice['title'][:40]}...", end=" ", flush=True)
+            region = extract_region_from_text(body_text, notice.get("org", "")) or state[notice_id].get("region", "")
-
+            region_detail = extract_region_detail_from_text(body_text)
-                try:
+            if amount:
-                    page = await browser.new_page()
-                    page.set_default_timeout(30000)
-                    await page.goto(pdf_url, wait_until="domcontentloaded", timeout=25000)
-                    await page.wait_for_timeout(2000)
-
-                    # 提取页面文本，解析金额
-                    page_text = await page.evaluate("() => document.body.innerText")
-                    amount = extract_amount_from_text(page_text)
-                    if amount and state is not None and notice_id in state:
                 state[notice_id]["amount"] = amount
-                    deadline = extract_deadline_from_text(page_text)
+            if deadline:
-                    if deadline and state is not None and notice_id in state:
                 state[notice_id]["deadline"] = deadline
-                    region = extract_region_from_text(page_text, notice.get("org", ""))
+            if region:
-                    if region and state is not None and notice_id in state:
                 state[notice_id]["region"] = region
-                    region_detail = extract_region_detail_from_text(page_text)
+            if region_detail:
-                    if region_detail and state is not None and notice_id in state:
                 state[notice_id]["region_detail"] = region_detail
 
-                    # 保存 HTML 源码到 html/ 目录（供 AI 阅读）
+            # 附件下载（取第一个 PDF/文档）
-                    html_source = await page.content()
+            if attach_list:
-                    with open(html_path, "w", encoding="utf-8") as f:
+                ext = ".pdf" if ".pdf" in attach_list[0].lower() else os.path.splitext(attach_list[0])[1] or ".pdf"
-                        f.write(html_source)
+                pdf_name = f"{base_name}{ext}"
+                try:
+                    download_attachment(attach_list[0], os.path.join(pdf_dir, pdf_name))
+                    state[notice_id]["pdf"] = f"pdf/{pdf_name}"
+                except Exception as e:
+                    print(f"⚠️ 附件下载失败({e})", end=" ")
 
-                    # 获取实际内容高度（SPA 可能 overflow:hidden，scrollHeight 不准确）
+            print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'}")
-                    content_height = await page.evaluate(
-                        "() => Math.max("
-                        "  document.documentElement.scrollHeight,"
-                        "  document.body.scrollHeight,"
-                        "  ...Array.from(document.querySelectorAll('body *'))"
-                        "    .map(el => el.getBoundingClientRect().bottom)"
-                        "    .filter(v => v < 50000),"
-                        "  1000"
-                        ")"
-                    )
-                    await page.pdf(
-                        path=pdf_path,
-                        width="1440px",
-                        height=f"{content_height + 50}px",
-                        print_background=True,
-                    )
-                    await page.close()
-                    html_size = os.path.getsize(html_path)
-                    pdf_size = os.path.getsize(pdf_path)
-                    print(f"✅ PDF:{pdf_size//1024}KB HTML:{html_size//1024}KB")
             success += 1
-
-                    # 更新状态文件，记录路径
-                    if state is not None and state_file and notice_id in state:
-                        state[notice_id]["pdf"] = rel_pdf
-                        state[notice_id]["html"] = f"html/{html_name}"
-                        with open(state_file, "w", encoding="utf-8") as f:
-                            json.dump(state, f, ensure_ascii=False, indent=2)
         except Exception as e:
             print(f"❌ {e}")
-                    fail += 1
 
-            await browser.close()
+        # 边处理边落盘，中断不丢已处理的记录
-            return success, fail
+        with open(state_file, "w", encoding="utf-8") as f:
+            json.dump(state, f, ensure_ascii=False, indent=2)
 
-    import asyncio
+    if success:
+        print(f"  📦 详情增强完成: {success}/{len(notices)} 条")
+
+
+def run():
+    parser = argparse.ArgumentParser(description="新矿集团供应链平台 公告抓取（新站 SSR 版）")
+    parser.add_argument("--from", dest="from_date", help="起始日期 YYYY-MM-DD")
+    parser.add_argument("--to", dest="to_date", help="截止日期 YYYY-MM-DD")
+    parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型（含中标/结果/变更类）")
+    parser.add_argument("--force", action="store_true", help="强制重新抓取")
+    parser.add_argument("--keyword", default="", help="关键词过滤（标题/发布单位）")
+    parser.add_argument("--max-pages", dest="max_pages", type=int, default=MAX_PAGES_DEFAULT,
+                        help=f"每板块最大翻页数（默认 {MAX_PAGES_DEFAULT}）")
+    args = parser.parse_args()
+
+    today = date.today()
+
+    # 加载历史已下载记录（跨天去重）
+    today_str = today.strftime("%Y-%m-%d")
+    accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
+
+    if args.from_date:
+        date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
+    elif not args.force and last_run_date:
+        date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
+    else:
+        date_from = today - timedelta(days=7)
+
+    if args.to_date:
+        date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
+    else:
+        date_to = today
+
+    date_range = f"{date_from}~{date_to}"
+    days_span = (date_to - date_from).days + 1
+
+    # 允许的类型集合：默认商机类+流标类，--all-types 全部
+    allowed_tags = BUSINESS_TAGS | FLOWN_TAGS
+    if args.all_types:
+        allowed_tags = allowed_tags | RESULT_TAGS
+
+    # 输出目录
+    output_dir = os.path.join(OUTPUT_BASE, today_str)
+    os.makedirs(output_dir, exist_ok=True)
+
+    # 子目录：html（源码） + pdf（附件转存）
+    html_dir = os.path.join(output_dir, "html")
+    pdf_dir = os.path.join(output_dir, "pdf")
+    os.makedirs(html_dir, exist_ok=True)
+    os.makedirs(pdf_dir, exist_ok=True)
+
+    # 状态文件（使用累积状态做去重）
+    state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
+    state = accumulated_state
+
+    all_notices = []
+    new_count = 0
+    skip_count = 0
+    filtered_count = 0
+
+    print(f"===== 新矿集团供应链平台 公告抓取（新站 {BASE_URL}） =====")
+    print(f"日期范围: {date_from} ~ {date_to} ({days_span}天)")
+    print(f"类型范围: {'全部' if args.all_types else '商机类+流标类'}")
+    print()
+
+    for category in CATEGORIES:
+        print(f"[{category}] 正在获取...")
         try:
-        success, fail = asyncio.run(_run())
+            items = fetch_all_notices(category, date_from, args.max_pages)
-        if success:
+        except Exception as e:
-            print(f"  📄 PDF 下载完成: {success} 成功" + (f", {fail} 失败" if fail else ""))
+            print(f"  ❌ 板块 {category} 抓取失败: {e}")
-    except RuntimeError:
+            continue
-        loop = asyncio.new_event_loop()
+        print(f"  ✅ 列表解析 {len(items)} 条")
-        asyncio.set_event_loop(loop)
+
-        success, fail = loop.run_until_complete(_run())
+        type_new = 0
-        loop.close()
+        type_skip = 0
-        if success:
+        for item in items:
-            print(f"  📄 PDF 下载完成: {success} 成功" + (f", {fail} 失败" if fail else ""))
+            tag = item["tag"]
+            # 类型过滤
+            if tag and tag not in allowed_tags:
+                filtered_count += 1
+                continue
+
+            # 日期过滤
+            if not match_date_range(item["publish_date"], date_from, date_to):
+                filtered_count += 1
+                continue
+
+            # 关键词过滤
+            if args.keyword:
+                if args.keyword not in item["title"] and args.keyword not in item["org"]:
+                    filtered_count += 1
+                    continue
+
+            # 多次询价标题过滤 (二次/三次/四次...)
+            if not RETRY_PATTERN.search(item["title"]):
+                filtered_count += 1
+                continue
+
+            # 发布组织白名单（新站"发布单位"列常为代理机构，采购人名多在标题里，故标题+单位联合匹配）
+            org_text = item["org"] + item["title"]
+            if not any(org_name in org_text for org_name in ORG_WHITELIST):
+                filtered_count += 1
+                continue
+
+            notice_id = item["id"]
+
+            # 去重
+            if not args.force and notice_id in state:
+                skip_count += 1
+                type_skip += 1
+                continue
+
+            notice = {
+                "id": notice_id,
+                "title": item["title"],
+                "org": item["org"],
+                "type": tag or "公告",
+                "status": "",
+                "publish_time": item["publish_date"],
+                "detail_url": f"{BASE_URL}{item['url_path']}",
+                "mobile_url": "",
+                "source": "新矿集团",
+            }
+            # 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
+            title_region = extract_region_from_text(notice["title"], notice["org"])
+            all_notices.append(notice)
+            state[notice_id] = {
+                "title": notice["title"],
+                "org": notice["org"],
+                "publish_time": notice["publish_time"],
+                "type": notice["type"],
+                "detail_url": notice["detail_url"],
+                "downloaded_at": datetime.now().isoformat(),
+                "region": title_region,
+            }
+            new_count += 1
+            type_new += 1
+
+        print(f"      新增: {type_new} | 跳过: {type_skip}")
+        print()
+
+    # 保存状态（含 lastRun 时间戳，用于下次增量爬取）
+    state["lastRun"] = datetime.now().isoformat()
+    save_state(state_file, state)
+
+    # 生成清单
+    if all_notices:
+        generate_markdown(all_notices, output_dir, date_range)
+
+    # 详情页增强：HTML 存档 + 金额/截止/地区提取 + 附件 PDF 转存
+    process_details(all_notices, output_dir, html_dir, pdf_dir, state, state_file)
+
+    print(f"\n========== 完成 ==========")
+    print(f"✅ 新增: {new_count} | ⏩ 跳过: {skip_count} | 🔍 日期过滤: {filtered_count}")
+    print(f"📁 保存目录: {output_dir}")
 
 
 if __name__ == "__main__":