残留
+ link_m = re.search(
+ r']*title="([^"]+)"', row
+ )
+ if not link_m:
+ continue
+ tag_m = re.search(r'([^<]+)', row)
+ date_m = re.search(r"| \s*(20\d{2}-\d{2}-\d{2})\s* | ", row)
+ tds = re.findall(r"([^<]*) | ", row)
+ items.append(
+ {
+ "id": link_m.group(2),
+ "url_path": html_lib.unescape(link_m.group(1)),
+ "title": html_lib.unescape(link_m.group(3)).strip(),
+ "tag": tag_m.group(1).strip() if tag_m else "",
+ "publish_date": date_m.group(1) if date_m else "",
+ "org": tds[-1].strip() if tds else "",
+ }
+ )
+ return items
+
+
+def fetch_all_notices(category, date_from, max_pages):
+ """翻页抓取指定板块的公告列表（按发布时间倒序，翻到早于 date_from 即停）"""
all_items = []
- page = 1
-
- while True:
- data = fetch_notices(notice_type, page=page)
- if not data.get("success"):
- print(f" ❌ API 请求失败: {data.get('errorCode', 'unknown')}")
+ for page_no in range(max_pages):
+ html_text = fetch_list_page(category, page_no)
+ items = parse_list_html(html_text)
+ if not items:
break
-
- items = data.get("data", {}).get("noticedata", [])
- total = data.get("data", {}).get("total", 0)
-
all_items.extend(items)
-
- total_page = (total + PAGE_SIZE - 1) // PAGE_SIZE
- print(f" 第 {page}/{total_page} 页: {len(items)} 条")
-
- if len(items) < PAGE_SIZE or page >= total_page:
+ print(f" 第 {page_no + 1} 页: {len(items)} 条")
+ # 倒序排列：本页最旧一条都早于起始日期，无需继续翻页
+ dates = [it["publish_date"] for it in items if it["publish_date"]]
+ if dates and max(dates) < str(date_from):
break
-
- page += 1
-
return all_items
-def normalize_notice(item, notice_type_label):
- """统一公告数据格式"""
- notice_id = item.get("noticeId", "")
- pub_time = item.get("publishtime", "")
+def fetch_detail(notice_id, timeout=30):
+ """抓取详情页，返回 (html_text, body_text, attach_list)
- # 解析状态
- status = item.get("status", "")
- norm_status = status if status else "未知"
+ body_text 为去标签后的正文纯文本（供金额/截止/地区提取），
+ attach_list 为详情页附件直链列表（PDF/Word/Excel）。
+ """
+ url = f"{BASE_URL}/front/index/v_notice_view?id={notice_id}"
+ resp = requests.get(url, headers=HEADERS, timeout=timeout)
+ resp.raise_for_status()
+ html_text = resp.text
+ cleaned = re.sub(r"|", " ", html_text)
+ body_text = html_lib.unescape(re.sub(r"<[^>]+>", " ", cleaned))
+ body_text = re.sub(r"[ \t\r]+", " ", body_text)
+ attach_list = re.findall(r'href="(https?://[^"]+\.(?:pdf|docx?|xlsx?)[^"]*)"', cleaned)
+ return html_text, body_text, attach_list
- return {
- "id": notice_id,
- "title": item.get("noticename", ""),
- "org": item.get("orgname", ""),
- "org_number": item.get("orgnumber", ""),
- "type": notice_type_label,
- "type_code": item.get("type", ""),
- "status": norm_status,
- "publish_time": pub_time,
- "detail_url": item.get("url", ""),
- "mobile_url": item.get("mobileurl", ""),
- "source": "新矿集团",
- }
+
+def download_attachment(url, save_path, timeout=60):
+ """下载附件文件（PDF/Word/Excel 直链）"""
+ resp = requests.get(url, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=timeout, stream=True)
+ resp.raise_for_status()
+ with open(save_path, "wb") as f:
+ for chunk in resp.iter_content(chunk_size=65536):
+ f.write(chunk)
+
+
+# ============ 业务过滤与字段提取（沿用旧版） ============
def match_date_range(publish_time_str, date_from, date_to):
@@ -200,21 +236,22 @@ def generate_markdown(notices, output_dir, date_range):
f"# 新矿集团供应链平台 公告清单\n",
f"**抓取时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')} | **日期范围**: {date_range}",
f"**公告总数**: {len(notices)} 条\n",
- f"> 数据来源：{BASE_URL} | 平台：新矿集团供应链平台\n",
+ f"> 数据来源：{BASE_URL} | 平台：新矿集团供应链平台（新站）\n",
"---\n",
]
for i, n in enumerate(notices, 1):
- lines.extend([
- f"## {i}. {n['title']}",
- f"- **发布组织**: {n['org']}",
- f"- **类型**: {n['type']}",
- f"- **状态**: {n['status']}",
- f"- **发布时间**: {n['publish_time']}",
- f"- **详情**: [查看]({n['detail_url']})",
- "",
- "---\n",
- ])
+ lines.extend(
+ [
+ f"## {i}. {n['title']}",
+ f"- **发布单位**: {n['org']}",
+ f"- **类型**: {n['type']}",
+ f"- **发布时间**: {n['publish_time']}",
+ f"- **详情**: [查看]({n['detail_url']})",
+ "",
+ "---\n",
+ ]
+ )
content = "\n".join(lines)
date_str = date_range.replace("~", "-")
@@ -228,167 +265,6 @@ def generate_markdown(notices, output_dir, date_range):
return filepath
-def run():
- parser = argparse.ArgumentParser(description="新矿集团供应链平台 公告抓取")
- parser.add_argument("--from", dest="from_date", help="起始日期 YYYY-MM-DD")
- parser.add_argument("--to", dest="to_date", help="截止日期 YYYY-MM-DD")
- parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
- parser.add_argument("--force", action="store_true", help="强制重新抓取")
- parser.add_argument("--keyword", default="", help="关键词过滤")
- args = parser.parse_args()
-
- today = date.today()
-
- # 加载历史已下载记录（跨天去重）
- today_str = today.strftime("%Y-%m-%d")
- accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
-
- if args.from_date:
- date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
- elif not args.force and last_run_date:
- date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
- else:
- date_from = today - timedelta(days=7)
-
- if args.to_date:
- date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
- else:
- date_to = today
-
- date_range = f"{date_from}~{date_to}"
- days_span = (date_to - date_from).days + 1
-
- # 公告类型
- notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
-
- # 输出目录
- output_dir = os.path.join(OUTPUT_BASE, today_str)
- os.makedirs(output_dir, exist_ok=True)
-
- # 子目录：html（源码） + pdf（转存）
- html_dir = os.path.join(output_dir, "html")
- pdf_dir = os.path.join(output_dir, "pdf")
- os.makedirs(html_dir, exist_ok=True)
- os.makedirs(pdf_dir, exist_ok=True)
-
- # 状态文件（使用累积状态做去重）
- state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
- state = accumulated_state
-
- all_notices = []
- new_count = 0
- skip_count = 0
- filtered_count = 0
-
- print(f"===== 新矿集团供应链平台 公告抓取 =====")
- print(f"日期范围: {date_from} ~ {date_to} ({days_span}天)")
- print(f"公告类型: {', '.join([NOTICE_TYPES.get(t, t) for t in notice_types])}")
- print()
-
- for ntype in notice_types:
- label = NOTICE_TYPES.get(ntype, ntype)
- print(f"[{label}] 正在获取...")
-
- items = fetch_all_notices(ntype)
- print(f" ✅ API 返回 {len(items)} 条")
-
- type_new = 0
- type_skip = 0
- for item in items:
- notice = normalize_notice(item, label)
-
- # 日期过滤
- if not match_date_range(notice["publish_time"], date_from, date_to):
- filtered_count += 1
- continue
-
- # 关键词过滤
- if args.keyword:
- title = notice["title"]
- org = notice["org"]
- if args.keyword not in title and args.keyword not in org:
- filtered_count += 1
- continue
-
- # 多次询价标题过滤 (二次/三次/四次...)
- if not RETRY_PATTERN.search(notice["title"]):
- filtered_count += 1
- continue
-
- # 发布组织白名单
- if not any(org_name in notice["org"] for org_name in ORG_WHITELIST):
- filtered_count += 1
- continue
-
- notice_id = notice["id"]
-
- # 去重
- if not args.force and notice_id in state:
- skip_count += 1
- type_skip += 1
- continue
-
- # 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
- title_region = extract_region_from_text(notice["title"], notice["org"])
- all_notices.append(notice)
- state[notice_id] = {
- "title": notice["title"],
- "org": notice["org"],
- "publish_time": notice["publish_time"],
- "type": notice["type"],
- "detail_url": notice["detail_url"],
- "downloaded_at": datetime.now().isoformat(),
- "region": title_region,
- }
- new_count += 1
- type_new += 1
-
- print(f" 新增: {type_new} | 跳过: {type_skip}")
- print()
-
- # 保存状态（含 lastRun 时间戳，用于下次增量爬取）
- state["lastRun"] = datetime.now().isoformat()
- save_state(state_file, state)
-
- # 为缺少金额的记录从详情页提取
- extract_amounts_for_existing(state, output_dir)
-
- # 生成清单
- if all_notices:
- generate_markdown(all_notices, output_dir, date_range)
-
- # 详情页转存 PDF + HTML（新公告 + 已有状态中缺文件的）
- all_pending = list(all_notices)
- if not args.force:
- for notice_id, info in list(state.items()):
- if not isinstance(info, dict):
- continue
- pdf_key = info.get("pdf", "")
- html_key = info.get("html", "")
- pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
- html_ok = html_key and os.path.exists(os.path.join(output_dir, html_key))
- if pdf_ok and html_ok:
- continue
- all_pending.append({
- "id": notice_id,
- "title": info.get("title", ""),
- "org": info.get("org", ""),
- "publish_time": info.get("publish_time", ""),
- "type": info.get("type", ""),
- "mobile_url": f"https://xkgroup.kdcloud.com/mobile.html?form=mobsp_quonotice_view&billid={notice_id}&userId=guest&accountId=2066845847806639104",
- "detail_url": f"https://xkgroup.kdcloud.com/index.html?formId=quo_notice¬iceId={notice_id}&userId=guest&accountId=2066845847806639104",
- })
-
- if all_pending:
- download_pdfs(all_pending, output_dir, html_dir, pdf_dir, state, state_file, force=args.force)
- else:
- print(" ℹ️ 指定日期范围内无新公告")
-
- print(f"\n========== 完成 ==========")
- print(f"✅ 新增: {new_count} | ⏩ 跳过: {skip_count} | 🔍 日期过滤: {filtered_count}")
- print(f"📁 保存目录: {output_dir}")
-
-
def extract_deadline_from_text(text):
"""从公告正文中提取截止日期（采购文件获取截止时间）"""
if not text:
@@ -542,205 +418,219 @@ def extract_amount_from_text(text):
return ""
-def extract_amounts_for_existing(state, output_dir):
- """为已抓取但缺少金额的公告，访问详情页提取金额"""
- try:
- import asyncio
- from playwright.async_api import async_playwright
- except ImportError:
- print(" ⚠️ Playwright 未安装，跳过金额提取")
+def process_details(notices, output_dir, html_dir, pdf_dir, state, state_file):
+ """对新公告做详情页增强：存 HTML 源码 + 提取金额/截止/地区 + 下载附件 PDF
+
+ 新站详情页是纯 SSR HTML，requests 直接可取（不再依赖 Playwright）。
+ 单条失败仅告警，不阻断。
+ """
+ if not notices:
return
+ print(f"\n[详情增强] {len(notices)} 条新公告，提取金额/截止/地区 + 存档...")
+ success = 0
+ for i, notice in enumerate(notices):
+ notice_id = notice["id"]
+ print(f" [{i+1}/{len(notices)}] {notice['title'][:40]}...", end=" ", flush=True)
+ try:
+ html_text, body_text, attach_list = fetch_detail(notice_id)
- # 筛选缺少金额的记录
- pending = []
- for notice_id, info in state.items():
- if notice_id == "lastRun":
- continue
- if not isinstance(info, dict):
- continue
- if info.get("amount"):
- continue
- pending.append(notice_id)
+ pub_date = notice.get("publish_time", "")[:10]
+ safe_title = re.sub(r"[\\/:*?\"<>|]", "_", notice["title"][:40])
+ base_name = f"{pub_date}_{notice_id}_{safe_title}"
+ html_name = f"{base_name}.html"
+ with open(os.path.join(html_dir, html_name), "w", encoding="utf-8") as f:
+ f.write(html_text)
+ state[notice_id]["html"] = f"html/{html_name}"
- if not pending:
- print(" ✅ 所有记录已有金额信息")
- return
-
- print(f"\n[金额提取] {len(pending)} 条记录缺少金额，正在从详情页提取...")
-
- async def _run():
- async with async_playwright() as p:
- browser = await p.chromium.launch(headless=True)
- success = 0
-
- for i, notice_id in enumerate(pending):
- info = state[notice_id]
- url = f"https://xkgroup.kdcloud.com/mobile.html?form=mobsp_quonotice_view&billid={notice_id}&userId=guest&accountId=2066845847806639104"
- print(f" [{i+1}/{len(pending)}] {info.get('title', '')[:40]}...", end=" ", flush=True)
+ # 字段提取
+ amount = extract_amount_from_text(body_text)
+ deadline = extract_deadline_from_text(body_text)
+ region = extract_region_from_text(body_text, notice.get("org", "")) or state[notice_id].get("region", "")
+ region_detail = extract_region_detail_from_text(body_text)
+ if amount:
+ state[notice_id]["amount"] = amount
+ if deadline:
+ state[notice_id]["deadline"] = deadline
+ if region:
+ state[notice_id]["region"] = region
+ if region_detail:
+ state[notice_id]["region_detail"] = region_detail
+ # 附件下载（取第一个 PDF/文档）
+ if attach_list:
+ ext = ".pdf" if ".pdf" in attach_list[0].lower() else os.path.splitext(attach_list[0])[1] or ".pdf"
+ pdf_name = f"{base_name}{ext}"
try:
- page = await browser.new_page()
- page.set_default_timeout(20000)
- await page.goto(url, wait_until="domcontentloaded", timeout=20000)
- await page.wait_for_timeout(2000)
- text = await page.evaluate("() => document.body.innerText")
- await page.close()
-
- amount = extract_amount_from_text(text)
- deadline = extract_deadline_from_text(text)
- region = extract_region_from_text(text, info.get("org", ""))
- region_detail = extract_region_detail_from_text(text)
- updated = False
- if amount:
- state[notice_id]["amount"] = amount
- updated = True
- if deadline:
- state[notice_id]["deadline"] = deadline
- updated = True
- if region:
- state[notice_id]["region"] = region
- updated = True
- if region_detail:
- state[notice_id]["region_detail"] = region_detail
- updated = True
- if updated:
- success += 1
- print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
- else:
- print("⏭️ 未找到金额/截止日期/地区")
+ download_attachment(attach_list[0], os.path.join(pdf_dir, pdf_name))
+ state[notice_id]["pdf"] = f"pdf/{pdf_name}"
except Exception as e:
- print(f"❌ {e}")
+ print(f"⚠️ 附件下载失败({e})", end=" ")
- await browser.close()
- return success
+ print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'}")
+ success += 1
+ except Exception as e:
+ print(f"❌ {e}")
- import asyncio
- try:
- success = asyncio.run(_run())
- except RuntimeError:
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- success = loop.run_until_complete(_run())
- loop.close()
+ # 边处理边落盘，中断不丢已处理的记录
+ with open(state_file, "w", encoding="utf-8") as f:
+ json.dump(state, f, ensure_ascii=False, indent=2)
if success:
- save_state(os.path.join(output_dir, "xkgroup_downloaded.json"), state)
- print(f" 💰 金额提取完成: {success} 条")
+ print(f" 📦 详情增强完成: {success}/{len(notices)} 条")
-def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file=None, force=False):
- """将新矿集团公告详情页转存为 PDF"""
- try:
- import asyncio
- from playwright.async_api import async_playwright
- except ImportError:
- print(" ⚠️ Playwright 未安装，跳过 PDF 下载")
- return
+def run():
+ parser = argparse.ArgumentParser(description="新矿集团供应链平台 公告抓取（新站 SSR 版）")
+ parser.add_argument("--from", dest="from_date", help="起始日期 YYYY-MM-DD")
+ parser.add_argument("--to", dest="to_date", help="截止日期 YYYY-MM-DD")
+ parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型（含中标/结果/变更类）")
+ parser.add_argument("--force", action="store_true", help="强制重新抓取")
+ parser.add_argument("--keyword", default="", help="关键词过滤（标题/发布单位）")
+ parser.add_argument("--max-pages", dest="max_pages", type=int, default=MAX_PAGES_DEFAULT,
+ help=f"每板块最大翻页数（默认 {MAX_PAGES_DEFAULT}）")
+ args = parser.parse_args()
- print("\n[PDF] 正在下载详情页...")
+ today = date.today()
- async def _run():
- async with async_playwright() as p:
- browser = await p.chromium.launch(headless=True)
- success = 0
- fail = 0
+ # 加载历史已下载记录（跨天去重）
+ today_str = today.strftime("%Y-%m-%d")
+ accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
- for i, notice in enumerate(notices):
- notice_id = notice.get("id", "")
- # 手机版页面含完整内容，桌面版只有摘要
- pdf_url = notice.get("mobile_url", "") or notice.get("detail_url", "")
+ if args.from_date:
+ date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
+ elif not args.force and last_run_date:
+ date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
+ else:
+ date_from = today - timedelta(days=7)
- if not notice_id or not pdf_url:
+ if args.to_date:
+ date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
+ else:
+ date_to = today
+
+ date_range = f"{date_from}~{date_to}"
+ days_span = (date_to - date_from).days + 1
+
+ # 允许的类型集合：默认商机类+流标类，--all-types 全部
+ allowed_tags = BUSINESS_TAGS | FLOWN_TAGS
+ if args.all_types:
+ allowed_tags = allowed_tags | RESULT_TAGS
+
+ # 输出目录
+ output_dir = os.path.join(OUTPUT_BASE, today_str)
+ os.makedirs(output_dir, exist_ok=True)
+
+ # 子目录：html（源码） + pdf（附件转存）
+ html_dir = os.path.join(output_dir, "html")
+ pdf_dir = os.path.join(output_dir, "pdf")
+ os.makedirs(html_dir, exist_ok=True)
+ os.makedirs(pdf_dir, exist_ok=True)
+
+ # 状态文件（使用累积状态做去重）
+ state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
+ state = accumulated_state
+
+ all_notices = []
+ new_count = 0
+ skip_count = 0
+ filtered_count = 0
+
+ print(f"===== 新矿集团供应链平台 公告抓取（新站 {BASE_URL}） =====")
+ print(f"日期范围: {date_from} ~ {date_to} ({days_span}天)")
+ print(f"类型范围: {'全部' if args.all_types else '商机类+流标类'}")
+ print()
+
+ for category in CATEGORIES:
+ print(f"[{category}] 正在获取...")
+ try:
+ items = fetch_all_notices(category, date_from, args.max_pages)
+ except Exception as e:
+ print(f" ❌ 板块 {category} 抓取失败: {e}")
+ continue
+ print(f" ✅ 列表解析 {len(items)} 条")
+
+ type_new = 0
+ type_skip = 0
+ for item in items:
+ tag = item["tag"]
+ # 类型过滤
+ if tag and tag not in allowed_tags:
+ filtered_count += 1
+ continue
+
+ # 日期过滤
+ if not match_date_range(item["publish_date"], date_from, date_to):
+ filtered_count += 1
+ continue
+
+ # 关键词过滤
+ if args.keyword:
+ if args.keyword not in item["title"] and args.keyword not in item["org"]:
+ filtered_count += 1
continue
- pub_date = notice.get("publish_time", "")[:10]
- safe_title = re.sub(r"[\\/:*?\"<>|]", "_", notice["title"][:40])
- base_name = f"{pub_date}_{notice_id}_{safe_title}"
- html_name = f"{base_name}.html"
- pdf_name = f"{base_name}.pdf"
- html_path = os.path.join(html_dir, html_name)
- pdf_path = os.path.join(pdf_dir, pdf_name)
- rel_pdf = f"pdf/{pdf_name}"
+ # 多次询价标题过滤 (二次/三次/四次...)
+ if not RETRY_PATTERN.search(item["title"]):
+ filtered_count += 1
+ continue
- if not force and os.path.exists(pdf_path):
- continue
+ # 发布组织白名单（新站"发布单位"列常为代理机构，采购人名多在标题里，故标题+单位联合匹配）
+ org_text = item["org"] + item["title"]
+ if not any(org_name in org_text for org_name in ORG_WHITELIST):
+ filtered_count += 1
+ continue
- print(f" [{i+1}/{len(notices)}] {notice['title'][:40]}...", end=" ", flush=True)
+ notice_id = item["id"]
- try:
- page = await browser.new_page()
- page.set_default_timeout(30000)
- await page.goto(pdf_url, wait_until="domcontentloaded", timeout=25000)
- await page.wait_for_timeout(2000)
+ # 去重
+ if not args.force and notice_id in state:
+ skip_count += 1
+ type_skip += 1
+ continue
- # 提取页面文本，解析金额
- page_text = await page.evaluate("() => document.body.innerText")
- amount = extract_amount_from_text(page_text)
- if amount and state is not None and notice_id in state:
- state[notice_id]["amount"] = amount
- deadline = extract_deadline_from_text(page_text)
- if deadline and state is not None and notice_id in state:
- state[notice_id]["deadline"] = deadline
- region = extract_region_from_text(page_text, notice.get("org", ""))
- if region and state is not None and notice_id in state:
- state[notice_id]["region"] = region
- region_detail = extract_region_detail_from_text(page_text)
- if region_detail and state is not None and notice_id in state:
- state[notice_id]["region_detail"] = region_detail
+ notice = {
+ "id": notice_id,
+ "title": item["title"],
+ "org": item["org"],
+ "type": tag or "公告",
+ "status": "",
+ "publish_time": item["publish_date"],
+ "detail_url": f"{BASE_URL}{item['url_path']}",
+ "mobile_url": "",
+ "source": "新矿集团",
+ }
+ # 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
+ title_region = extract_region_from_text(notice["title"], notice["org"])
+ all_notices.append(notice)
+ state[notice_id] = {
+ "title": notice["title"],
+ "org": notice["org"],
+ "publish_time": notice["publish_time"],
+ "type": notice["type"],
+ "detail_url": notice["detail_url"],
+ "downloaded_at": datetime.now().isoformat(),
+ "region": title_region,
+ }
+ new_count += 1
+ type_new += 1
- # 保存 HTML 源码到 html/ 目录（供 AI 阅读）
- html_source = await page.content()
- with open(html_path, "w", encoding="utf-8") as f:
- f.write(html_source)
+ print(f" 新增: {type_new} | 跳过: {type_skip}")
+ print()
- # 获取实际内容高度（SPA 可能 overflow:hidden，scrollHeight 不准确）
- content_height = await page.evaluate(
- "() => Math.max("
- " document.documentElement.scrollHeight,"
- " document.body.scrollHeight,"
- " ...Array.from(document.querySelectorAll('body *'))"
- " .map(el => el.getBoundingClientRect().bottom)"
- " .filter(v => v < 50000),"
- " 1000"
- ")"
- )
- await page.pdf(
- path=pdf_path,
- width="1440px",
- height=f"{content_height + 50}px",
- print_background=True,
- )
- await page.close()
- html_size = os.path.getsize(html_path)
- pdf_size = os.path.getsize(pdf_path)
- print(f"✅ PDF:{pdf_size//1024}KB HTML:{html_size//1024}KB")
- success += 1
+ # 保存状态（含 lastRun 时间戳，用于下次增量爬取）
+ state["lastRun"] = datetime.now().isoformat()
+ save_state(state_file, state)
- # 更新状态文件，记录路径
- if state is not None and state_file and notice_id in state:
- state[notice_id]["pdf"] = rel_pdf
- state[notice_id]["html"] = f"html/{html_name}"
- with open(state_file, "w", encoding="utf-8") as f:
- json.dump(state, f, ensure_ascii=False, indent=2)
- except Exception as e:
- print(f"❌ {e}")
- fail += 1
+ # 生成清单
+ if all_notices:
+ generate_markdown(all_notices, output_dir, date_range)
- await browser.close()
- return success, fail
+ # 详情页增强：HTML 存档 + 金额/截止/地区提取 + 附件 PDF 转存
+ process_details(all_notices, output_dir, html_dir, pdf_dir, state, state_file)
- import asyncio
- try:
- success, fail = asyncio.run(_run())
- if success:
- print(f" 📄 PDF 下载完成: {success} 成功" + (f", {fail} 失败" if fail else ""))
- except RuntimeError:
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- success, fail = loop.run_until_complete(_run())
- loop.close()
- if success:
- print(f" 📄 PDF 下载完成: {success} 成功" + (f", {fail} 失败" if fail else ""))
+ print(f"\n========== 完成 ==========")
+ print(f"✅ 新增: {new_count} | ⏩ 跳过: {skip_count} | 🔍 日期过滤: {filtered_count}")
+ print(f"📁 保存目录: {output_dir}")
if __name__ == "__main__":