From bd213eb5eca02fcd58c3261244fd47e5c7227ecb Mon Sep 17 00:00:00 2001 From: Sisyphus Date: Fri, 26 Jun 2026 16:24:32 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=87=87=E9=9B=86=E5=99=A8=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=E6=8F=90=E5=8F=96=E4=BC=98=E5=8C=96=20-=20=E7=9C=81?= =?UTF-8?q?=E4=BB=BD=E5=85=B3=E9=94=AE=E8=AF=8D=E5=8C=B9=E9=85=8D=E3=80=81?= =?UTF-8?q?=E6=A0=87=E9=A2=98=E5=85=9C=E5=BA=95=E3=80=81=E8=B7=A8=E5=A4=A9?= =?UTF-8?q?=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- apps/api/src/crawler/crawler.service.ts | 121 ++++++++++---- scripts/extract_address_from_pdf.py | 132 +++++++++++++++ scripts/tender-fetcher/fetch-cnncecp.py | 64 +++++-- scripts/tender-fetcher/fetch-v1.2.js | 78 ++++++++- scripts/tender-fetcher/fetch-xkgroup.py | 212 +++++++++++++++++++++++- 5 files changed, 544 insertions(+), 63 deletions(-) create mode 100644 scripts/extract_address_from_pdf.py diff --git a/apps/api/src/crawler/crawler.service.ts b/apps/api/src/crawler/crawler.service.ts index 1401e26..67ff925 100644 --- a/apps/api/src/crawler/crawler.service.ts +++ b/apps/api/src/crawler/crawler.service.ts @@ -18,6 +18,8 @@ interface FetchResult { noticeId: string detailUrl: string amount?: string + region?: string + regionDetail?: string } @Injectable() @@ -109,7 +111,7 @@ export class CrawlerService implements OnModuleInit { let results: FetchResult[] = [] if (config.type === 'bid_powerchina') { - results = await this.runPowerChina() + results = await this.runPowerChina(config) } else { results = await this.runGenericScript(config) } @@ -129,6 +131,8 @@ export class CrawlerService implements OnModuleInit { detailUrl: r.detailUrl, publishTime: r.publishTime || null, ...(r.amount ? { amount: r.amount } : {}), + ...(r.region ? { region: r.region } : {}), + ...(r.regionDetail ? { regionDetail: r.regionDetail } : {}), summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`, }, create: { @@ -143,7 +147,8 @@ export class CrawlerService implements OnModuleInit { status: '待复核', level: 'C', category: '', - region: '', + region: r.region || '', + regionDetail: r.regionDetail || '', summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`, }, }) @@ -172,49 +177,92 @@ export class CrawlerService implements OnModuleInit { } } - private async runPowerChina(): Promise { + private async runPowerChina(config: { keywords?: string | null }): Promise { const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js') if (!fs.existsSync(scriptPath)) { throw new Error('中电建抓取脚本不存在') } - execSync(`node "${scriptPath}"`, { - encoding: 'utf-8', - maxBuffer: 10 * 1024 * 1024, - timeout: 180000, - }) + let keywords: string[] + try { + keywords = config.keywords ? JSON.parse(config.keywords) : [] + } catch { + keywords = [] + } + if (keywords.length === 0) keywords = [''] const today = new Date().toISOString().split('T')[0] - const statePath = path.join( - process.env.HOME || '', - 'desktop/招标公告', - today, - 'downloaded.json', - ) + const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today) + const seen = new Set() + const allResults: FetchResult[] = [] - if (!fs.existsSync(statePath)) return [] + for (const kw of keywords) { + const kwFlag = kw ? ` --keyword "${kw}"` : '' + try { + execSync(`node "${scriptPath}"${kwFlag}`, { + encoding: 'utf-8', + maxBuffer: 10 * 1024 * 1024, + timeout: 180000, + }) + } catch { + // 单个 keyword 失败不影响其他 keyword + continue + } - const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) - const downloaded = state.downloaded || {} - const results: FetchResult[] = [] + const statePath = path.join(baseDir, 'downloaded.json') + if (!fs.existsSync(statePath)) continue - for (const [key, v] of Object.entries(downloaded)) { - const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string } - if (info.skipped || info.failed) continue - const [systemId] = key.split('_') - const realId = info.id || systemId - results.push({ - source: '中电建', - title: info.title || '', - organization: '中电建', - publishTime: state.lastRun || '', - deadline: info.deadline || '', - noticeId: systemId, - detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`, - }) + try { + const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) + const downloaded = state.downloaded || {} + + const outputDir = path.dirname(statePath) + const pdfDir = path.join(outputDir, 'pdf') + + for (const [key, v] of Object.entries(downloaded)) { + const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string; publishTime?: string; path?: string } + if (info.skipped || info.failed) continue + const [systemId] = key.split('_') + if (seen.has(systemId)) continue + seen.add(systemId) + + const realId = info.id || systemId + + let region = '' + let regionDetail = '' + if (info.path && fs.existsSync(pdfDir)) { + const pdfPath = path.join(pdfDir, path.basename(info.path)) + if (fs.existsSync(pdfPath)) { + try { + const extractor = path.join(SCRIPTS_DIR, 'extract_address_from_pdf.py') + const title = info.title || '' + const cmd = `python3 "${extractor}" "${pdfPath}" "${title}"` + const out = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim() + const parsed = JSON.parse(out) + region = parsed.region || '' + regionDetail = parsed.regionDetail || '' + } catch {} + } + } + + allResults.push({ + source: '中电建', + title: info.title || '', + organization: '中电建', + publishTime: info.publishTime || state.lastRun || '', + deadline: info.deadline || '', + noticeId: systemId, + detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`, + region, + regionDetail, + }) + } + } catch { + continue + } } - return results + return allResults } private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise { @@ -248,10 +296,13 @@ export class CrawlerService implements OnModuleInit { const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) const results: FetchResult[] = [] + const todayStr = new Date().toISOString().substring(0, 10) for (const [noticeId, v] of Object.entries(state)) { - const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string } + const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string } if (noticeId === 'lastRun') continue + // 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理 + if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue const leadAmount = info.amount || '' results.push({ source, @@ -262,6 +313,8 @@ export class CrawlerService implements OnModuleInit { noticeId: info.id || noticeId, detailUrl: info.detail_url || '', amount: leadAmount, + region: info.region || '', + regionDetail: info.region_detail || '', }) } diff --git a/scripts/extract_address_from_pdf.py b/scripts/extract_address_from_pdf.py new file mode 100644 index 0000000..19341d3 --- /dev/null +++ b/scripts/extract_address_from_pdf.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +从招标公告 PDF 中提取地址信息。 +用法: python3 extract_address_from_pdf.py [公告标题] +输出: JSON {"region": "省/市", "regionDetail": "详细地址"} +""" +import fitz +import re +import json +import sys + +# 全国省份/直辖市/自治区名称（含简称、别称） +PROVINCES = [ + '北京', '天津', '上海', '重庆', + '河北', '山西', '辽宁', '吉林', '黑龙江', + '江苏', '浙江', '安徽', '福建', '江西', '山东', + '河南', '湖北', '湖南', '广东', '海南', + '四川', '贵州', '云南', '陕西', '甘肃', '青海', + '台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆', +] +TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2] +SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏'] + + +def extract_province_best(text): + """从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）""" + if not text: + return '' + text_clean = text.replace(' ', '') + # 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截） + for prov in SPECIAL_PROVS: + if prov in text_clean: + return prov + # 2. 直辖市 + for prov in ['北京', '天津', '上海', '重庆']: + if prov in text_clean: + return prov + # 3. "XX省"（如"陕西省"、"广东省"） + m = re.search(r'([\u4e00-\u9fff]{2}省)', text) + if m: + return m.group(1)[:2] + # 4. 两字省份关键词 + for prov in TWO_CHAR_PROVS: + if prov in text_clean: + return prov + return '' + + +def extract_city_from_text(text): + """从文本中提取城市名（XX市）""" + if not text: + return '' + m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text) + if m: + city = m.group(1) + if city not in ['北京市', '天津市', '上海市', '重庆市']: + return city + return '' + + +def extract_address(pdf_path: str, title: str = '') -> dict: + pdf_text = '' + try: + doc = fitz.open(pdf_path) + for page in doc: + pdf_text += page.get_text() + doc.close() + except Exception: + pdf_text = '' + + result = {'region': '', 'regionDetail': ''} + + # ── 1. 从PDF正文中提取详细地址 ── + addr_patterns = [ + r'建设地点[：:]\s*(.+?)(?:\n|$)', + r'项目地点[：:]\s*(.+?)(?:\n|$)', + r'交货地点[：:]\s*(.+?)(?:\n|$)', + r'施工地点[：:]\s*(.+?)(?:\n|$)', + r'项目地址[：:]\s*(.+?)(?:\n|$)', + r'供货地点[：:]\s*(.+?)(?:\n|$)', + r'到货地点[：:]\s*(.+?)(?:\n|$)', + r'送货地点[：:]\s*(.+?)(?:\n|$)', + r'工程地点[：:]\s*(.+?)(?:\n|$)', + r'实施地点[：:]\s*(.+?)(?:\n|$)', + r'服务地点[：:]\s*(.+?)(?:\n|$)', + r'收货地址[：:]\s*(.+?)(?:\n|$)', + r'所在地[：:]\s*(.+?)(?:\n|$)', + ] + + addr = '' + for p in addr_patterns: + m = re.search(p, pdf_text) + if m: + addr = m.group(1).strip() + if len(addr) >= 4: + break + addr = '' + + if addr: + result['regionDetail'] = addr + prov = extract_province_best(addr) + if prov: + result['region'] = prov + else: + city = extract_city_from_text(addr) + if city: + result['region'] = city + + # ── 2. 正文没提取到region，从整个PDF文本中搜索 ── + if not result['region'] and pdf_text: + prov = extract_province_best(pdf_text) + if prov: + result['region'] = prov + + # ── 3. 还没提取到，从标题中搜索 ── + if not result['region'] and title: + prov = extract_province_best(title) + if prov: + result['region'] = prov + + return result + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print(json.dumps({'region': '', 'regionDetail': ''})) + sys.exit(0) + + pdf_path = sys.argv[1] + title = sys.argv[2] if len(sys.argv) > 2 else '' + result = extract_address(pdf_path, title) + print(json.dumps(result, ensure_ascii=False)) diff --git a/scripts/tender-fetcher/fetch-cnncecp.py b/scripts/tender-fetcher/fetch-cnncecp.py index bf86309..d9877ec 100755 --- a/scripts/tender-fetcher/fetch-cnncecp.py +++ b/scripts/tender-fetcher/fetch-cnncecp.py @@ -107,7 +107,7 @@ def parse_list_page(html, type_label): "detail_url": BASE_URL + a_tag.get("href", ""), "notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "", "type": type_label, - "publish_date": "", + "publish_time": "", "deadline": "", "status": "", } @@ -120,7 +120,7 @@ def parse_list_page(html, type_label): if "Gray" in cls: # 发布日期 if re.match(r"\d{4}-\d{2}-\d{2}", text): - notice["publish_date"] = text + notice["publish_time"] = text elif "Green" in cls: # 正在报名 notice["status"] = text.strip("[]") @@ -132,7 +132,7 @@ def parse_list_page(html, type_label): em_tag = span.find("em", class_="Red") if em_tag: em_text = em_tag.get_text(strip=True) - dl_match = re.search(r"文件下载截止[：:](.+)", em_text) + dl_match = re.search(r"文件(?:发售|下载)截止[：:](.+)", em_text) if dl_match: notice["deadline"] = dl_match.group(1).strip() @@ -173,6 +173,39 @@ def save_state(state_path, state): json.dump(state, f, ensure_ascii=False, indent=2) +def load_accumulated_state(output_base, state_filename, today_str): + """加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录""" + accumulated = {} + last_run_date = None + state_dir = os.path.join(output_base, today_str) + today_state = load_state(os.path.join(state_dir, state_filename)) + if today_state: + for k, v in today_state.items(): + if k != "lastRun": + accumulated[k] = v + last_run = today_state.get("lastRun") + if last_run: + last_run_date = last_run[:10] + try: + for i in range(1, 31): + d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d") + path = os.path.join(output_base, d, state_filename) + if os.path.exists(path): + try: + with open(path, "r", encoding="utf-8") as f: + prev = json.load(f) + except: + continue + for k, v in prev.items(): + if k != "lastRun" and k not in accumulated: + accumulated[k] = v + if k == "lastRun" and not last_run_date: + last_run_date = str(v)[:10] + except: + pass + return accumulated, last_run_date + + def generate_markdown(notices, output_dir, date_range, keyword=""): """生成公告清单 Markdown""" lines = [ @@ -195,7 +228,7 @@ def generate_markdown(notices, output_dir, date_range, keyword=""): deadline_display = n['deadline'] if n['deadline'] else '-' lines.extend([ f"## {i}. {n['title']}", - f"- **发布日期**: {n['publish_date']}", + f"- **发布日期**: {n['publish_time']}", f"- **公告类型**: {n['type']}", f"- **状态**: {status_display}", f"- **文件下载截止**: {deadline_display}", @@ -225,15 +258,21 @@ def run(): parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型") parser.add_argument("--force", action="store_true", help="强制重新抓取") parser.add_argument("--keyword", default="", help="关键词过滤（匹配标题）") - parser.add_argument("--max-pages", type=int, default=0, help="最大翻页数（0=不限制）") + parser.add_argument("--max-pages", type=int, default=10, help="最大翻页数（0=不限制，默认10页≈160条）") args = parser.parse_args() today = date.today() + today_str = today.strftime("%Y-%m-%d") + + # 加载历史已下载记录（跨天去重） + accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "cnncecp_downloaded.json", today_str) if args.from_date: date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date() + elif not args.force and last_run_date: + date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date() else: - date_from = today - timedelta(days=7) + date_from = today if args.to_date: date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date() @@ -247,12 +286,12 @@ def run(): notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES # 输出目录 - output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d")) + output_dir = os.path.join(OUTPUT_BASE, today_str) os.makedirs(output_dir, exist_ok=True) - # 状态文件 + # 状态文件（使用累积状态做去重） state_file = os.path.join(output_dir, "cnncecp_downloaded.json") - state = load_state(state_file) + state = accumulated_state all_notices = [] new_count = 0 @@ -306,7 +345,7 @@ def run(): type_total += 1 # 日期过滤 - if not match_date_range(notice["publish_date"], date_from, date_to): + if not match_date_range(notice.get("publish_time", ""), date_from, date_to): type_filtered += 1 filtered_count += 1 continue @@ -329,7 +368,7 @@ def run(): state[notice_id] = { "title": notice["title"], "type": notice["type"], - "publish_date": notice["publish_date"], + "publish_time": notice["publish_time"], "deadline": notice["deadline"], "status": notice["status"], "detail_url": notice["detail_url"], @@ -352,7 +391,8 @@ def run(): print(f" ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}") print() - # 保存状态 + # 保存状态（含 lastRun 时间戳，用于下次增量爬取） + state["lastRun"] = datetime.now().isoformat() save_state(state_file, state) # 生成清单 diff --git a/scripts/tender-fetcher/fetch-v1.2.js b/scripts/tender-fetcher/fetch-v1.2.js index 0c11e41..2c272e4 100755 --- a/scripts/tender-fetcher/fetch-v1.2.js +++ b/scripts/tender-fetcher/fetch-v1.2.js @@ -25,9 +25,14 @@ const http = require('http'); const crypto = require('crypto'); // ========== 配置区 ========== -const KEYWORD = '水电十一局'; + +// 支持命令行传入 --keyword（例如: node fetch-v1.2.js --keyword 电缆） +const args = process.argv.slice(2) +const keywordIdx = args.indexOf('--keyword') +const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : '' + // 公告类型：可按需增减。支持的类型值见 API 返回的 announcementType 字段 -const TYPES = ['招采公告', '资格预审公告', '采购预告', '中标结果公示']; +const TYPES = ['招采公告']; // 可选：留空则从上次抓取日期继续；有值则从指定日期开始 const START_DATE = ''; const END_DATE = new Date().toISOString().split('T')[0]; // 今天 @@ -220,6 +225,54 @@ async function runWithConcurrency(tasks, limit) { return results; } +// ---------- 地址提取 ---------- + +const PROVINCES = [ + '北京', '天津', '上海', '重庆', + '河北', '山西', '辽宁', '吉林', '黑龙江', + '江苏', '浙江', '安徽', '福建', '江西', '山东', + '河南', '湖北', '湖南', '广东', '海南', + '四川', '贵州', '云南', '陕西', '甘肃', '青海', + '台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆', +] +const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2) +const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏'] + +function extractRegionFromTitle(title) { + if (!title) return '' + // 匹配 "XX省"、"XX市"（直辖市）、"XX自治区" + const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/) + if (m) { + const region = m[1] + if (region.includes('新疆')) return '新疆' + if (region.includes('广西')) return '广西' + if (region.includes('内蒙古')) return '内蒙古' + if (region.includes('宁夏')) return '宁夏' + if (region.includes('西藏')) return '西藏' + if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2) + if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2) + return '' + } + // 直接匹配省份关键词 + for (const prov of SPECIAL_PROVS) { + if (title.includes(prov)) return prov + } + for (const prov of TWO_CHAR_PROVS) { + if (title.includes(prov)) return prov + } + return '' +} + +function extractCityFromTitle(title) { + if (!title) return '' + const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/) + if (m) { + const city = m[1] + if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city + } + return '' +} + // ---------- 生成公告清单 ---------- function buildAttachmentName(r, index) { @@ -332,9 +385,11 @@ async function main() { log(`共获取 ${stats.total} 条公告（分 ${page} 页）`); - // 过滤目标公告类型（由 TYPES 配置定义） - const caigou = allAnnouncements.filter(r => TYPES.includes(r.announcementType)); - log(`其中 ${TYPES.join('、')}: ${caigou.length} 条`); + // 过滤目标公告类型（由 TYPES 配置定义），且仅取货物类 + const caigou = allAnnouncements.filter(r => + TYPES.includes(r.announcementType) && r.titleTypeName === '货物类' + ); + log(`其中 ${TYPES.join('、')}（货物类）: ${caigou.length} 条`); // 生成公告清单（所有本次抓到的招采公告，无论是否已下载） const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE); @@ -363,7 +418,7 @@ async function main() { if (!picUrl) { stats.skipNoAttachment++; log(`跳过（无附件）: ${r.title}`, 'WARN'); - state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', time: new Date().toISOString() }; + state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() }; continue; } @@ -376,15 +431,20 @@ async function main() { tasks.push(async () => { log(`下载 [${shortName}]: ${fname}`); + + // 从标题中提取省份（详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取） + const region = extractRegionFromTitle(r.title) || '' + const regionDetail = extractCityFromTitle(r.title) || '' + try { await downloadFile(picUrl, fpath); const size = fs.statSync(fpath).size; - state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', path: `pdf/${fname}`, size, time: new Date().toISOString() }; + state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail }; stats.success++; - log(` ✅ ${fname} (${size} bytes)`); + log(` ✅ ${fname} (${size} bytes) region=${region || '-'} addr=${regionDetail || '-'}`); } catch (err) { stats.failed++; - state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', failed: true, error: err.message, time: new Date().toISOString() }; + state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail }; log(` ❌ ${fname}: ${err.message}`, 'ERROR'); } }); diff --git a/scripts/tender-fetcher/fetch-xkgroup.py b/scripts/tender-fetcher/fetch-xkgroup.py index 383637f..fe376a9 100755 --- a/scripts/tender-fetcher/fetch-xkgroup.py +++ b/scripts/tender-fetcher/fetch-xkgroup.py @@ -150,6 +150,41 @@ def save_state(state_path, state): json.dump(state, f, ensure_ascii=False, indent=2) +def load_accumulated_state(output_base, state_filename, today_str): + """加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录""" + accumulated = {} + last_run_date = None + state_dir = os.path.join(output_base, today_str) + # 先加载今日状态 + today_state = load_state(os.path.join(state_dir, state_filename)) + if today_state: + for k, v in today_state.items(): + if k != "lastRun": + accumulated[k] = v + last_run = today_state.get("lastRun") + if last_run: + last_run_date = last_run[:10] + # 扫描历史目录（最近 30 天），合并之前下载的记录 + try: + for i in range(1, 31): + d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d") + path = os.path.join(output_base, d, state_filename) + if os.path.exists(path): + try: + with open(path, "r", encoding="utf-8") as f: + prev = json.load(f) + except: + continue + for k, v in prev.items(): + if k != "lastRun" and k not in accumulated: + accumulated[k] = v + if k == "lastRun" and not last_run_date: + last_run_date = str(v)[:10] + except: + pass + return accumulated, last_run_date + + def generate_markdown(notices, output_dir, date_range): """生成公告清单 Markdown""" lines = [ @@ -195,10 +230,16 @@ def run(): today = date.today() + # 加载历史已下载记录（跨天去重） + today_str = today.strftime("%Y-%m-%d") + accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str) + if args.from_date: date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date() + elif not args.force and last_run_date: + date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date() else: - date_from = today - timedelta(days=7) # 默认抓取最近 7 天 + date_from = today - timedelta(days=7) if args.to_date: date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date() @@ -212,7 +253,7 @@ def run(): notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES # 输出目录 - output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d")) + output_dir = os.path.join(OUTPUT_BASE, today_str) os.makedirs(output_dir, exist_ok=True) # 子目录：html（源码） + pdf（转存） @@ -221,9 +262,9 @@ def run(): os.makedirs(html_dir, exist_ok=True) os.makedirs(pdf_dir, exist_ok=True) - # 状态文件 + # 状态文件（使用累积状态做去重） state_file = os.path.join(output_dir, "xkgroup_downloaded.json") - state = load_state(state_file) + state = accumulated_state all_notices = [] new_count = 0 @@ -268,6 +309,8 @@ def run(): type_skip += 1 continue + # 从标题/组织名初步提取地区（详情页提取时会补充/覆盖） + title_region = extract_region_from_text(notice["title"], notice["org"]) all_notices.append(notice) state[notice_id] = { "title": notice["title"], @@ -276,6 +319,7 @@ def run(): "type": notice["type"], "detail_url": notice["detail_url"], "downloaded_at": datetime.now().isoformat(), + "region": title_region, } new_count += 1 type_new += 1 @@ -283,7 +327,8 @@ def run(): print(f" 新增: {type_new} | 跳过: {type_skip}") print() - # 保存状态 + # 保存状态（含 lastRun 时间戳，用于下次增量爬取） + state["lastRun"] = datetime.now().isoformat() save_state(state_file, state) # 为缺少金额的记录从详情页提取 @@ -296,7 +341,9 @@ def run(): # 详情页转存 PDF + HTML（新公告 + 已有状态中缺文件的） all_pending = list(all_notices) if not args.force: - for notice_id, info in state.items(): + for notice_id, info in list(state.items()): + if not isinstance(info, dict): + continue pdf_key = info.get("pdf", "") html_key = info.get("html", "") pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key)) @@ -323,6 +370,129 @@ def run(): print(f"📁 保存目录: {output_dir}") +def extract_deadline_from_text(text): + """从公告正文中提取截止日期（采购文件获取截止时间）""" + if not text: + return "" + # 模式1: "起至2026年6月22日19:00（北京时间）止" → 取截止日期 + m = re.search(r"起至(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2}):(\d{2})", text) + if m: + return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}" + # 模式2: "获取时间：...至2026-06-22 19:00" + m = re.search(r"至\s*(\d{4})-(\d{1,2})-(\d{1,2})\s+\d{1,2}:\d{2}", text) + if m: + return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}" + # 模式3: "截止时间：2026年6月22日" 或 "截止日期：2026-06-22" + for pat in [ + r"截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日", + r"截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})", + r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日", + r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})", + ]: + m = re.search(pat, text) + if m: + return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}" + return "" + + +# 全国省份/直辖市/自治区 +PROVINCES = [ + '北京', '天津', '上海', '重庆', + '河北', '山西', '辽宁', '吉林', '黑龙江', + '江苏', '浙江', '安徽', '福建', '江西', '山东', + '河南', '湖北', '湖南', '广东', '海南', + '四川', '贵州', '云南', '陕西', '甘肃', '青海', + '台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆', +] +TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2] +SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏'] + + +def extract_province_best(text): + """从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）""" + if not text: + return '' + text_clean = text.replace(' ', '') + # 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截） + for sp in SPECIAL_PROVS: + if sp in text_clean: + return sp + # 2. 直辖市 + for dc in ['北京', '天津', '上海', '重庆']: + if dc in text_clean: + return dc + # 3. "XX省"（如"陕西省"、"广东省"） + m = re.search(r'([\u4e00-\u9fff]{2}省)', text) + if m: + return m.group(1)[:2] + # 4. 两字省份关键词 + for p in TWO_CHAR_PROVS: + if p in text_clean: + return p + return '' + + +def extract_city_from_text(text): + """从文本中提取城市名（XX市）""" + if not text: + return '' + m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text) + if m: + city = m.group(1) + if city not in ['北京市', '天津市', '上海市', '重庆市']: + return city + return '' + + +def extract_region_from_text(text, org=""): + """从公告正文或公司名中提取地区（省份）""" + if not text and not org: + return "" + # 1. 从正文找项目/交货/施工地点 + if text: + for pat in [ + r"(?:项目|交货|送货|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))", + r"(?:地[址点]|所在地区)[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))", + r"位\s*于\s*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))", + r"(?:项目|工程)\s*(?:位于|所在地|所在)[：:：\s]*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))", + ]: + m = re.search(pat, text) + if m: + region = m.group(1) + for sp in SPECIAL_PROVS: + if sp in region: + return sp + return region[:2] # 取前两个字 + # 2. 从全文扫描省份关键词 + prov = extract_province_best(text) + if prov: + return prov + # 3. 从公司名推断 + if org: + org_prov = extract_province_best(org) + if org_prov: + return org_prov + return "" + + +def extract_region_detail_from_text(text): + """从公告正文中提取详细收货/交货地址""" + if not text: + return "" + for pat in [ + r"(?:交货|收货|送货|项目|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)", + r"(?:地[址点])[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)", + r"建设地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)", + r"项目地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)", + ]: + m = re.search(pat, text) + if m: + addr = m.group(1).strip() + if len(addr) >= 6: + return addr + return "" + + def extract_amount_from_text(text): """从公告正文中提取金额（万元）""" if not text: @@ -367,6 +537,8 @@ def extract_amounts_for_existing(state, output_dir): for notice_id, info in state.items(): if notice_id == "lastRun": continue + if not isinstance(info, dict): + continue if info.get("amount"): continue pending.append(notice_id) @@ -396,12 +568,27 @@ def extract_amounts_for_existing(state, output_dir): await page.close() amount = extract_amount_from_text(text) + deadline = extract_deadline_from_text(text) + region = extract_region_from_text(text, info.get("org", "")) + region_detail = extract_region_detail_from_text(text) + updated = False if amount: state[notice_id]["amount"] = amount + updated = True + if deadline: + state[notice_id]["deadline"] = deadline + updated = True + if region: + state[notice_id]["region"] = region + updated = True + if region_detail: + state[notice_id]["region_detail"] = region_detail + updated = True + if updated: success += 1 - print(f"✅ {amount}") + print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}") else: - print("⏭️ 未找到金额") + print("⏭️ 未找到金额/截止日期/地区") except Exception as e: print(f"❌ {e}") @@ -472,6 +659,15 @@ def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file amount = extract_amount_from_text(page_text) if amount and state is not None and notice_id in state: state[notice_id]["amount"] = amount + deadline = extract_deadline_from_text(page_text) + if deadline and state is not None and notice_id in state: + state[notice_id]["deadline"] = deadline + region = extract_region_from_text(page_text, notice.get("org", "")) + if region and state is not None and notice_id in state: + state[notice_id]["region"] = region + region_detail = extract_region_detail_from_text(page_text) + if region_detail and state is not None and notice_id in state: + state[notice_id]["region_detail"] = region_detail # 保存 HTML 源码到 html/ 目录（供 AI 阅读） html_source = await page.content()