direv/platform-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(xkgroup): 采集器切换到新站 120.70.100.128:8096

Browse Source 
- 新站为 layui 服务端渲染(无 JSON API), 抓取层重写为 HTML 列表页解析
  (GET /front/notice/cgxx?category=bid|procure|auction 分页, pageNo 从 0 起)
- 旧站 kdcloud getPortalNoticeData API 已失效(302 到登录页)
- 类型标签映射: 默认抓商机类(招标/采购/竞价公告)+流标类(废标/竞价流拍),
  --all-types 额外抓结果/变更/其他类, 对应旧版 C+D / +B 语义
- 白名单过滤改为标题+发布单位联合匹配(新站发布单位多为代理机构,
  按单位匹配会漏掉白名单组织自己的公告)
- 详情增强改用 requests(新站纯 SSR 无需 Playwright): 存 HTML 源码 +
  提取金额/截止/地区 + 下载附件 PDF 直链
- 输出状态文件格式不变, CrawlerService 入库链路零改动(本地+生产端到端验证通过)
- 新增 --max-pages 参数(默认每板块 5 页)
This commit is contained in:
Sisyphus 2026-10-13 10:20:09 +08:00
parent d6ad7e02c5
commit 855d4c3f00
1 changed files with 308 additions and 418 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

726
scripts/tender-fetcher/fetch-xkgroup.py
View File

@ -1,23 +1,35 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
新矿集团供应链平台 公告抓取工具 新矿集团供应链平台 公告抓取工具
数据源：https://xkgroup.kdcloud.com/isv/q5m9/dist/srmportal.html 数据源：http://120.70.100.128:8096/（新站，2026-10 切换）
API：getPortalNoticeData (REST, 无需登录即可访问 guest 数据) - 纯服务端渲染（layui），无 JSON API，抓 HTML 列表页解析
- 列表页：/front/notice/cgxx?category=bid|procure|auction（GET 表单分页，pageNo 从 0 起，每页 10 条）
- 详情页：/front/index/v_notice_view?id=<32hex>（含正文 + 附件 PDF 直链）
抓取类型： 板块类型标签全集：
C - 寻源公告（核心采购公告） bid 招标公告 / 变更&澄清公告 / 资格预审公告 / 中标候选人公示 / 中标公告 / 废标公告
D - 流标公告 procure 采购公告 / 变更公告 / 结果公示 / 其他公告
B - 结果公示 auction 竞价公告 / 竞价流拍
默认抓取"商机类 + 流标类"（对应旧站 C 寻源 + D 流标语义）：
商机类 = 招标公告/采购公告/竞价公告/资格预审公告
流标类 = 废标公告/竞价流拍
--all-types 额外抓结果/其他类（中标候选人公示/中标公告/变更&澄清/结果公示/其他公告）
用法： 用法：
python3 fetch-xkgroup.py # 差量更新（默认抓取昨天的寻源公告） python3 fetch-xkgroup.py # 差量更新（默认回溯到上次运行/7 天内）
python3 fetch-xkgroup.py --all-types # 抓取所有公告类型 python3 fetch-xkgroup.py --all-types # 抓取所有公告类型
python3 fetch-xkgroup.py --from 2026-05-01 # 指定起始日期 python3 fetch-xkgroup.py --from 2026-10-01 # 指定起始日期
python3 fetch-xkgroup.py --force # 强制重新抓取（不跳过已下载） python3 fetch-xkgroup.py --force # 强制重新抓取（不跳过已下载）
python3 fetch-xkgroup.py --keyword 水泥 # 按关键词过滤 python3 fetch-xkgroup.py --keyword 水泥 # 按关键词过滤（标题/发布单位）
python3 fetch-xkgroup.py --max-pages 10 # 每板块最大翻页数（默认 5 页 = 50 条）
状态文件 xkgroup_downloaded.json 格式与旧版一致，CrawlerService.runGenericScript 依赖字段：
id/title/org/publish_time/detail_url/amount/deadline/region/region_detail/downloaded_at
""" """
import argparse import argparse
import html as html_lib
import json import json
import os import os
import re import re
@ -31,17 +43,19 @@ except ImportError:
sys.exit(1) sys.exit(1)
# ============ 配置区 ============ # ============ 配置区 ============
BASE_URL = "https://xkgroup.kdcloud.com/kapi/app/srm/srmwebapi" BASE_URL = "http://120.70.100.128:8096"
ACCOUNT_ID = "2066845847806639104" LIST_URL = f"{BASE_URL}/front/notice/cgxx"
PAGE_SIZE = 50 PAGE_SIZE = 10 # 新站固定每页 10 条
MAX_PAGES_DEFAULT = 5 # 每板块默认最多翻 5 页（日增量远小于此，历史回填可加大）
# 公告类型映射 # 板块与类型标签
NOTICE_TYPES = { CATEGORIES = ["bid", "procure", "auction"]
"C": "寻源公告",
"D": "流标公告", # 商机类 + 流标类（默认抓取，对应旧版 C+D）
"B": "结果公示", BUSINESS_TAGS = {"招标公告", "采购公告", "竞价公告", "资格预审公告"}
} FLOWN_TAGS = {"废标公告", "竞价流拍"}
DEFAULT_TYPES = ["C", "D"] # 抓取寻源公告 + 流标公告 # 结果/其他类（--all-types 才抓）
RESULT_TAGS = {"中标候选人公示", "中标公告", "变更&澄清公告", "变更公告", "结果公示", "其他公告"}
# 多次询价标题正则 (匹配 二次/三次/四次/.../九次) # 多次询价标题正则 (匹配 二次/三次/四次/.../九次)
RETRY_PATTERN = re.compile(r"[二三四五六七八九]次") RETRY_PATTERN = re.compile(r"[二三四五六七八九]次")
@ -56,77 +70,99 @@ OUTPUT_BASE = os.path.expanduser("~/desktop/招标公告")
HEADERS = { HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
"Accept": "application/json, text/plain, */*", "Accept": "text/html,application/xhtml+xml,*/*",
"Referer": "https://xkgroup.kdcloud.com/isv/q5m9/dist/srmportal.html", "Referer": f"{BASE_URL}/",
} }
# ============ 核心函数 ============ # ============ 抓取层（新站 HTML 解析） ============
def fetch_notices(notice_type, page=1, size=PAGE_SIZE): def fetch_list_page(category, page_no, timeout=30):
"""调用 API 获取公告列表""" """抓取列表页 HTML（GET 表单，pageNo 从 0 起）"""
params = { resp = requests.get(
"page": page, LIST_URL,
"size": size, params={"category": category, "pageNo": page_no},
"type": notice_type, headers=HEADERS,
"apiname": "getPortalNoticeData", timeout=timeout,
"methodname": "getPortalNoticeData", )
"accountId": ACCOUNT_ID,
}
resp = requests.get(BASE_URL, params=params, headers=HEADERS, timeout=30)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.text
def fetch_all_notices(notice_type): def parse_list_html(html_text):
"""翻页获取指定类型的所有公告""" """解析列表页 tbody#listTbody 中的公告条目
返回 [{"id", "url_path", "title", "tag", "publish_date", "org"}]
"""
items = []
tbody_m = re.search(r'<tbody id="listTbody">([\s\S]*?)</tbody>', html_text)
rows = re.split(r"<tr>", tbody_m.group(1)) if tbody_m else []
for row in rows[1:]: # 第一段是 <thead> 残留
link_m = re.search(
r'<a href="(/front/index/v_notice_view\?id=([a-f0-9]{32})[^"]*)"[^>]*title="([^"]+)"', row
)
if not link_m:
continue
tag_m = re.search(r'<span class="tag[^"]*">([^<]+)</span>', row)
date_m = re.search(r"<td>\s*(20\d{2}-\d{2}-\d{2})\s*</td>", row)
tds = re.findall(r"<td>([^<]*)</td>", row)
items.append(
{
"id": link_m.group(2),
"url_path": html_lib.unescape(link_m.group(1)),
"title": html_lib.unescape(link_m.group(3)).strip(),
"tag": tag_m.group(1).strip() if tag_m else "",
"publish_date": date_m.group(1) if date_m else "",
"org": tds[-1].strip() if tds else "",
}
)
return items
def fetch_all_notices(category, date_from, max_pages):
"""翻页抓取指定板块的公告列表（按发布时间倒序，翻到早于 date_from 即停）"""
all_items = [] all_items = []
page = 1 for page_no in range(max_pages):
html_text = fetch_list_page(category, page_no)
while True: items = parse_list_html(html_text)
data = fetch_notices(notice_type, page=page) if not items:
if not data.get("success"):
print(f" ❌ API 请求失败: {data.get('errorCode', 'unknown')}")
break break
items = data.get("data", {}).get("noticedata", [])
total = data.get("data", {}).get("total", 0)
all_items.extend(items) all_items.extend(items)
print(f" 第 {page_no + 1} 页: {len(items)} 条")
total_page = (total + PAGE_SIZE - 1) // PAGE_SIZE # 倒序排列：本页最旧一条都早于起始日期，无需继续翻页
print(f" 第 {page}/{total_page} 页: {len(items)} 条") dates = [it["publish_date"] for it in items if it["publish_date"]]
if dates and max(dates) < str(date_from):
if len(items) < PAGE_SIZE or page >= total_page:
break break
page += 1
return all_items return all_items
def normalize_notice(item, notice_type_label): def fetch_detail(notice_id, timeout=30):
"""统一公告数据格式""" """抓取详情页，返回 (html_text, body_text, attach_list)
notice_id = item.get("noticeId", "")
pub_time = item.get("publishtime", "")
# 解析状态 body_text 为去标签后的正文纯文本（供金额/截止/地区提取），
status = item.get("status", "") attach_list 为详情页附件直链列表（PDF/Word/Excel）。
norm_status = status if status else "未知" """
url = f"{BASE_URL}/front/index/v_notice_view?id={notice_id}"
resp = requests.get(url, headers=HEADERS, timeout=timeout)
resp.raise_for_status()
html_text = resp.text
cleaned = re.sub(r"<script[\s\S]*?</script>|<style[\s\S]*?</style>", " ", html_text)
body_text = html_lib.unescape(re.sub(r"<[^>]+>", " ", cleaned))
body_text = re.sub(r"[ \t\r]+", " ", body_text)
attach_list = re.findall(r'href="(https?://[^"]+\.(?:pdf|docx?|xlsx?)[^"]*)"', cleaned)
return html_text, body_text, attach_list
return {
"id": notice_id, def download_attachment(url, save_path, timeout=60):
"title": item.get("noticename", ""), """下载附件文件（PDF/Word/Excel 直链）"""
"org": item.get("orgname", ""), resp = requests.get(url, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=timeout, stream=True)
"org_number": item.get("orgnumber", ""), resp.raise_for_status()
"type": notice_type_label, with open(save_path, "wb") as f:
"type_code": item.get("type", ""), for chunk in resp.iter_content(chunk_size=65536):
"status": norm_status, f.write(chunk)
"publish_time": pub_time,
"detail_url": item.get("url", ""),
"mobile_url": item.get("mobileurl", ""), # ============ 业务过滤与字段提取（沿用旧版） ============
"source": "新矿集团",
}
def match_date_range(publish_time_str, date_from, date_to): def match_date_range(publish_time_str, date_from, date_to):
@ -200,21 +236,22 @@ def generate_markdown(notices, output_dir, date_range):
f"# 新矿集团供应链平台 公告清单\n", f"# 新矿集团供应链平台 公告清单\n",
f"**抓取时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')} | **日期范围**: {date_range}", f"**抓取时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')} | **日期范围**: {date_range}",
f"**公告总数**: {len(notices)} 条\n", f"**公告总数**: {len(notices)} 条\n",
f"> 数据来源：{BASE_URL} | 平台：新矿集团供应链平台\n", f"> 数据来源：{BASE_URL} | 平台：新矿集团供应链平台（新站）\n",
"---\n", "---\n",
] ]
for i, n in enumerate(notices, 1): for i, n in enumerate(notices, 1):
lines.extend([ lines.extend(
f"## {i}. {n['title']}", [
f"- **发布组织**: {n['org']}", f"## {i}. {n['title']}",
f"- **类型**: {n['type']}", f"- **发布单位**: {n['org']}",
f"- **状态**: {n['status']}", f"- **类型**: {n['type']}",
f"- **发布时间**: {n['publish_time']}", f"- **发布时间**: {n['publish_time']}",
f"- **详情**: [查看]({n['detail_url']})", f"- **详情**: [查看]({n['detail_url']})",
"", "",
"---\n", "---\n",
]) ]
)
content = "\n".join(lines) content = "\n".join(lines)
date_str = date_range.replace("~", "-") date_str = date_range.replace("~", "-")
@ -228,167 +265,6 @@ def generate_markdown(notices, output_dir, date_range):
return filepath return filepath
def run():
parser = argparse.ArgumentParser(description="新矿集团供应链平台 公告抓取")
parser.add_argument("--from", dest="from_date", help="起始日期 YYYY-MM-DD")
parser.add_argument("--to", dest="to_date", help="截止日期 YYYY-MM-DD")
parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
parser.add_argument("--force", action="store_true", help="强制重新抓取")
parser.add_argument("--keyword", default="", help="关键词过滤")
args = parser.parse_args()
today = date.today()
# 加载历史已下载记录（跨天去重）
today_str = today.strftime("%Y-%m-%d")
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
if args.from_date:
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
elif not args.force and last_run_date:
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
else:
date_from = today - timedelta(days=7)
if args.to_date:
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
else:
date_to = today
date_range = f"{date_from}~{date_to}"
days_span = (date_to - date_from).days + 1
# 公告类型
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
# 输出目录
output_dir = os.path.join(OUTPUT_BASE, today_str)
os.makedirs(output_dir, exist_ok=True)
# 子目录：html（源码） + pdf（转存）
html_dir = os.path.join(output_dir, "html")
pdf_dir = os.path.join(output_dir, "pdf")
os.makedirs(html_dir, exist_ok=True)
os.makedirs(pdf_dir, exist_ok=True)
# 状态文件（使用累积状态做去重）
state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
state = accumulated_state
all_notices = []
new_count = 0
skip_count = 0
filtered_count = 0
print(f"===== 新矿集团供应链平台 公告抓取 =====")
print(f"日期范围: {date_from} ~ {date_to} ({days_span}天)")
print(f"公告类型: {', '.join([NOTICE_TYPES.get(t, t) for t in notice_types])}")
print()
for ntype in notice_types:
label = NOTICE_TYPES.get(ntype, ntype)
print(f"[{label}] 正在获取...")
items = fetch_all_notices(ntype)
print(f" ✅ API 返回 {len(items)} 条")
type_new = 0
type_skip = 0
for item in items:
notice = normalize_notice(item, label)
# 日期过滤
if not match_date_range(notice["publish_time"], date_from, date_to):
filtered_count += 1
continue
# 关键词过滤
if args.keyword:
title = notice["title"]
org = notice["org"]
if args.keyword not in title and args.keyword not in org:
filtered_count += 1
continue
# 多次询价标题过滤 (二次/三次/四次...)
if not RETRY_PATTERN.search(notice["title"]):
filtered_count += 1
continue
# 发布组织白名单
if not any(org_name in notice["org"] for org_name in ORG_WHITELIST):
filtered_count += 1
continue
notice_id = notice["id"]
# 去重
if not args.force and notice_id in state:
skip_count += 1
type_skip += 1
continue
# 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
title_region = extract_region_from_text(notice["title"], notice["org"])
all_notices.append(notice)
state[notice_id] = {
"title": notice["title"],
"org": notice["org"],
"publish_time": notice["publish_time"],
"type": notice["type"],
"detail_url": notice["detail_url"],
"downloaded_at": datetime.now().isoformat(),
"region": title_region,
}
new_count += 1
type_new += 1
print(f" 新增: {type_new} | 跳过: {type_skip}")
print()
# 保存状态（含 lastRun 时间戳，用于下次增量爬取）
state["lastRun"] = datetime.now().isoformat()
save_state(state_file, state)
# 为缺少金额的记录从详情页提取
extract_amounts_for_existing(state, output_dir)
# 生成清单
if all_notices:
generate_markdown(all_notices, output_dir, date_range)
# 详情页转存 PDF + HTML（新公告 + 已有状态中缺文件的）
all_pending = list(all_notices)
if not args.force:
for notice_id, info in list(state.items()):
if not isinstance(info, dict):
continue
pdf_key = info.get("pdf", "")
html_key = info.get("html", "")
pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
html_ok = html_key and os.path.exists(os.path.join(output_dir, html_key))
if pdf_ok and html_ok:
continue
all_pending.append({
"id": notice_id,
"title": info.get("title", ""),
"org": info.get("org", ""),
"publish_time": info.get("publish_time", ""),
"type": info.get("type", ""),
"mobile_url": f"https://xkgroup.kdcloud.com/mobile.html?form=mobsp_quonotice_view&billid={notice_id}&userId=guest&accountId=2066845847806639104",
"detail_url": f"https://xkgroup.kdcloud.com/index.html?formId=quo_notice&noticeId={notice_id}&userId=guest&accountId=2066845847806639104",
})
if all_pending:
download_pdfs(all_pending, output_dir, html_dir, pdf_dir, state, state_file, force=args.force)
else:
print(" ℹ️ 指定日期范围内无新公告")
print(f"\n========== 完成 ==========")
print(f"✅ 新增: {new_count} | ⏩ 跳过: {skip_count} | 🔍 日期过滤: {filtered_count}")
print(f"📁 保存目录: {output_dir}")
def extract_deadline_from_text(text): def extract_deadline_from_text(text):
"""从公告正文中提取截止日期（采购文件获取截止时间）""" """从公告正文中提取截止日期（采购文件获取截止时间）"""
if not text: if not text:
@ -542,205 +418,219 @@ def extract_amount_from_text(text):
return "" return ""
def extract_amounts_for_existing(state, output_dir): def process_details(notices, output_dir, html_dir, pdf_dir, state, state_file):
"""为已抓取但缺少金额的公告，访问详情页提取金额""" """对新公告做详情页增强：存 HTML 源码 + 提取金额/截止/地区 + 下载附件 PDF
try:
import asyncio 新站详情页是纯 SSR HTML，requests 直接可取（不再依赖 Playwright）。
from playwright.async_api import async_playwright 单条失败仅告警，不阻断。
except ImportError: """
print(" ⚠️ Playwright 未安装，跳过金额提取") if not notices:
return return
print(f"\n[详情增强] {len(notices)} 条新公告，提取金额/截止/地区 + 存档...")
success = 0
for i, notice in enumerate(notices):
notice_id = notice["id"]
print(f" [{i+1}/{len(notices)}] {notice['title'][:40]}...", end=" ", flush=True)
try:
html_text, body_text, attach_list = fetch_detail(notice_id)
# 筛选缺少金额的记录 pub_date = notice.get("publish_time", "")[:10]
pending = [] safe_title = re.sub(r"[\\/:*?\"<>|]", "_", notice["title"][:40])
for notice_id, info in state.items(): base_name = f"{pub_date}_{notice_id}_{safe_title}"
if notice_id == "lastRun": html_name = f"{base_name}.html"
continue with open(os.path.join(html_dir, html_name), "w", encoding="utf-8") as f:
if not isinstance(info, dict): f.write(html_text)
continue state[notice_id]["html"] = f"html/{html_name}"
if info.get("amount"):
continue
pending.append(notice_id)
if not pending: # 字段提取
print(" ✅ 所有记录已有金额信息") amount = extract_amount_from_text(body_text)
return deadline = extract_deadline_from_text(body_text)
region = extract_region_from_text(body_text, notice.get("org", "")) or state[notice_id].get("region", "")
print(f"\n[金额提取] {len(pending)} 条记录缺少金额，正在从详情页提取...") region_detail = extract_region_detail_from_text(body_text)
if amount:
async def _run(): state[notice_id]["amount"] = amount
async with async_playwright() as p: if deadline:
browser = await p.chromium.launch(headless=True) state[notice_id]["deadline"] = deadline
success = 0 if region:
state[notice_id]["region"] = region
for i, notice_id in enumerate(pending): if region_detail:
info = state[notice_id] state[notice_id]["region_detail"] = region_detail
url = f"https://xkgroup.kdcloud.com/mobile.html?form=mobsp_quonotice_view&billid={notice_id}&userId=guest&accountId=2066845847806639104"
print(f" [{i+1}/{len(pending)}] {info.get('title', '')[:40]}...", end=" ", flush=True)
# 附件下载（取第一个 PDF/文档）
if attach_list:
ext = ".pdf" if ".pdf" in attach_list[0].lower() else os.path.splitext(attach_list[0])[1] or ".pdf"
pdf_name = f"{base_name}{ext}"
try: try:
page = await browser.new_page() download_attachment(attach_list[0], os.path.join(pdf_dir, pdf_name))
page.set_default_timeout(20000) state[notice_id]["pdf"] = f"pdf/{pdf_name}"
await page.goto(url, wait_until="domcontentloaded", timeout=20000)
await page.wait_for_timeout(2000)
text = await page.evaluate("() => document.body.innerText")
await page.close()
amount = extract_amount_from_text(text)
deadline = extract_deadline_from_text(text)
region = extract_region_from_text(text, info.get("org", ""))
region_detail = extract_region_detail_from_text(text)
updated = False
if amount:
state[notice_id]["amount"] = amount
updated = True
if deadline:
state[notice_id]["deadline"] = deadline
updated = True
if region:
state[notice_id]["region"] = region
updated = True
if region_detail:
state[notice_id]["region_detail"] = region_detail
updated = True
if updated:
success += 1
print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
else:
print("⏭️ 未找到金额/截止日期/地区")
except Exception as e: except Exception as e:
print(f"❌ {e}") print(f"⚠️ 附件下载失败({e})", end=" ")
await browser.close() print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'}")
return success success += 1
except Exception as e:
print(f"❌ {e}")
import asyncio # 边处理边落盘，中断不丢已处理的记录
try: with open(state_file, "w", encoding="utf-8") as f:
success = asyncio.run(_run()) json.dump(state, f, ensure_ascii=False, indent=2)
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
success = loop.run_until_complete(_run())
loop.close()
if success: if success:
save_state(os.path.join(output_dir, "xkgroup_downloaded.json"), state) print(f" 📦 详情增强完成: {success}/{len(notices)} 条")
print(f" 💰 金额提取完成: {success} 条")
def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file=None, force=False): def run():
"""将新矿集团公告详情页转存为 PDF""" parser = argparse.ArgumentParser(description="新矿集团供应链平台 公告抓取（新站 SSR 版）")
try: parser.add_argument("--from", dest="from_date", help="起始日期 YYYY-MM-DD")
import asyncio parser.add_argument("--to", dest="to_date", help="截止日期 YYYY-MM-DD")
from playwright.async_api import async_playwright parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型（含中标/结果/变更类）")
except ImportError: parser.add_argument("--force", action="store_true", help="强制重新抓取")
print(" ⚠️ Playwright 未安装，跳过 PDF 下载") parser.add_argument("--keyword", default="", help="关键词过滤（标题/发布单位）")
return parser.add_argument("--max-pages", dest="max_pages", type=int, default=MAX_PAGES_DEFAULT,
help=f"每板块最大翻页数（默认 {MAX_PAGES_DEFAULT}）")
args = parser.parse_args()
print("\n[PDF] 正在下载详情页...") today = date.today()
async def _run(): # 加载历史已下载记录（跨天去重）
async with async_playwright() as p: today_str = today.strftime("%Y-%m-%d")
browser = await p.chromium.launch(headless=True) accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
success = 0
fail = 0
for i, notice in enumerate(notices): if args.from_date:
notice_id = notice.get("id", "") date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
# 手机版页面含完整内容，桌面版只有摘要 elif not args.force and last_run_date:
pdf_url = notice.get("mobile_url", "") or notice.get("detail_url", "") date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
else:
date_from = today - timedelta(days=7)
if not notice_id or not pdf_url: if args.to_date:
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
else:
date_to = today
date_range = f"{date_from}~{date_to}"
days_span = (date_to - date_from).days + 1
# 允许的类型集合：默认商机类+流标类，--all-types 全部
allowed_tags = BUSINESS_TAGS | FLOWN_TAGS
if args.all_types:
allowed_tags = allowed_tags | RESULT_TAGS
# 输出目录
output_dir = os.path.join(OUTPUT_BASE, today_str)
os.makedirs(output_dir, exist_ok=True)
# 子目录：html（源码） + pdf（附件转存）
html_dir = os.path.join(output_dir, "html")
pdf_dir = os.path.join(output_dir, "pdf")
os.makedirs(html_dir, exist_ok=True)
os.makedirs(pdf_dir, exist_ok=True)
# 状态文件（使用累积状态做去重）
state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
state = accumulated_state
all_notices = []
new_count = 0
skip_count = 0
filtered_count = 0
print(f"===== 新矿集团供应链平台 公告抓取（新站 {BASE_URL}） =====")
print(f"日期范围: {date_from} ~ {date_to} ({days_span}天)")
print(f"类型范围: {'全部' if args.all_types else '商机类+流标类'}")
print()
for category in CATEGORIES:
print(f"[{category}] 正在获取...")
try:
items = fetch_all_notices(category, date_from, args.max_pages)
except Exception as e:
print(f" ❌ 板块 {category} 抓取失败: {e}")
continue
print(f" ✅ 列表解析 {len(items)} 条")
type_new = 0
type_skip = 0
for item in items:
tag = item["tag"]
# 类型过滤
if tag and tag not in allowed_tags:
filtered_count += 1
continue
# 日期过滤
if not match_date_range(item["publish_date"], date_from, date_to):
filtered_count += 1
continue
# 关键词过滤
if args.keyword:
if args.keyword not in item["title"] and args.keyword not in item["org"]:
filtered_count += 1
continue continue
pub_date = notice.get("publish_time", "")[:10] # 多次询价标题过滤 (二次/三次/四次...)
safe_title = re.sub(r"[\\/:*?\"<>|]", "_", notice["title"][:40]) if not RETRY_PATTERN.search(item["title"]):
base_name = f"{pub_date}_{notice_id}_{safe_title}" filtered_count += 1
html_name = f"{base_name}.html" continue
pdf_name = f"{base_name}.pdf"
html_path = os.path.join(html_dir, html_name)
pdf_path = os.path.join(pdf_dir, pdf_name)
rel_pdf = f"pdf/{pdf_name}"
if not force and os.path.exists(pdf_path): # 发布组织白名单（新站"发布单位"列常为代理机构，采购人名多在标题里，故标题+单位联合匹配）
continue org_text = item["org"] + item["title"]
if not any(org_name in org_text for org_name in ORG_WHITELIST):
filtered_count += 1
continue
print(f" [{i+1}/{len(notices)}] {notice['title'][:40]}...", end=" ", flush=True) notice_id = item["id"]
try: # 去重
page = await browser.new_page() if not args.force and notice_id in state:
page.set_default_timeout(30000) skip_count += 1
await page.goto(pdf_url, wait_until="domcontentloaded", timeout=25000) type_skip += 1
await page.wait_for_timeout(2000) continue
# 提取页面文本，解析金额 notice = {
page_text = await page.evaluate("() => document.body.innerText") "id": notice_id,
amount = extract_amount_from_text(page_text) "title": item["title"],
if amount and state is not None and notice_id in state: "org": item["org"],
state[notice_id]["amount"] = amount "type": tag or "公告",
deadline = extract_deadline_from_text(page_text) "status": "",
if deadline and state is not None and notice_id in state: "publish_time": item["publish_date"],
state[notice_id]["deadline"] = deadline "detail_url": f"{BASE_URL}{item['url_path']}",
region = extract_region_from_text(page_text, notice.get("org", "")) "mobile_url": "",
if region and state is not None and notice_id in state: "source": "新矿集团",
state[notice_id]["region"] = region }
region_detail = extract_region_detail_from_text(page_text) # 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
if region_detail and state is not None and notice_id in state: title_region = extract_region_from_text(notice["title"], notice["org"])
state[notice_id]["region_detail"] = region_detail all_notices.append(notice)
state[notice_id] = {
"title": notice["title"],
"org": notice["org"],
"publish_time": notice["publish_time"],
"type": notice["type"],
"detail_url": notice["detail_url"],
"downloaded_at": datetime.now().isoformat(),
"region": title_region,
}
new_count += 1
type_new += 1
# 保存 HTML 源码到 html/ 目录（供 AI 阅读） print(f" 新增: {type_new} | 跳过: {type_skip}")
html_source = await page.content() print()
with open(html_path, "w", encoding="utf-8") as f:
f.write(html_source)
# 获取实际内容高度（SPA 可能 overflow:hidden，scrollHeight 不准确） # 保存状态（含 lastRun 时间戳，用于下次增量爬取）
content_height = await page.evaluate( state["lastRun"] = datetime.now().isoformat()
"() => Math.max(" save_state(state_file, state)
" document.documentElement.scrollHeight,"
" document.body.scrollHeight,"
" ...Array.from(document.querySelectorAll('body *'))"
" .map(el => el.getBoundingClientRect().bottom)"
" .filter(v => v < 50000),"
" 1000"
")"
)
await page.pdf(
path=pdf_path,
width="1440px",
height=f"{content_height + 50}px",
print_background=True,
)
await page.close()
html_size = os.path.getsize(html_path)
pdf_size = os.path.getsize(pdf_path)
print(f"✅ PDF:{pdf_size//1024}KB HTML:{html_size//1024}KB")
success += 1
# 更新状态文件，记录路径 # 生成清单
if state is not None and state_file and notice_id in state: if all_notices:
state[notice_id]["pdf"] = rel_pdf generate_markdown(all_notices, output_dir, date_range)
state[notice_id]["html"] = f"html/{html_name}"
with open(state_file, "w", encoding="utf-8") as f:
json.dump(state, f, ensure_ascii=False, indent=2)
except Exception as e:
print(f"❌ {e}")
fail += 1
await browser.close() # 详情页增强：HTML 存档 + 金额/截止/地区提取 + 附件 PDF 转存
return success, fail process_details(all_notices, output_dir, html_dir, pdf_dir, state, state_file)
import asyncio print(f"\n========== 完成 ==========")
try: print(f"✅ 新增: {new_count} | ⏩ 跳过: {skip_count} | 🔍 日期过滤: {filtered_count}")
success, fail = asyncio.run(_run()) print(f"📁 保存目录: {output_dir}")
if success:
print(f" 📄 PDF 下载完成: {success} 成功" + (f", {fail} 失败" if fail else ""))
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
success, fail = loop.run_until_complete(_run())
loop.close()
if success:
print(f" 📄 PDF 下载完成: {success} 成功" + (f", {fail} 失败" if fail else ""))
if __name__ == "__main__": if __name__ == "__main__":