feat: 采集器地址提取优化 - 省份关键词匹配、标题兜底、跨天去重
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
Sisyphus
parent
49f922444a
commit
bd213eb5ec
@ -18,6 +18,8 @@ interface FetchResult {
|
||||
noticeId: string
|
||||
detailUrl: string
|
||||
amount?: string
|
||||
region?: string
|
||||
regionDetail?: string
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
@ -109,7 +111,7 @@ export class CrawlerService implements OnModuleInit {
|
||||
let results: FetchResult[] = []
|
||||
|
||||
if (config.type === 'bid_powerchina') {
|
||||
results = await this.runPowerChina()
|
||||
results = await this.runPowerChina(config)
|
||||
} else {
|
||||
results = await this.runGenericScript(config)
|
||||
}
|
||||
@ -129,6 +131,8 @@ export class CrawlerService implements OnModuleInit {
|
||||
detailUrl: r.detailUrl,
|
||||
publishTime: r.publishTime || null,
|
||||
...(r.amount ? { amount: r.amount } : {}),
|
||||
...(r.region ? { region: r.region } : {}),
|
||||
...(r.regionDetail ? { regionDetail: r.regionDetail } : {}),
|
||||
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
||||
},
|
||||
create: {
|
||||
@ -143,7 +147,8 @@ export class CrawlerService implements OnModuleInit {
|
||||
status: '待复核',
|
||||
level: 'C',
|
||||
category: '',
|
||||
region: '',
|
||||
region: r.region || '',
|
||||
regionDetail: r.regionDetail || '',
|
||||
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
||||
},
|
||||
})
|
||||
@ -172,49 +177,92 @@ export class CrawlerService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
private async runPowerChina(): Promise<FetchResult[]> {
|
||||
private async runPowerChina(config: { keywords?: string | null }): Promise<FetchResult[]> {
|
||||
const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js')
|
||||
if (!fs.existsSync(scriptPath)) {
|
||||
throw new Error('中电建抓取脚本不存在')
|
||||
}
|
||||
|
||||
execSync(`node "${scriptPath}"`, {
|
||||
let keywords: string[]
|
||||
try {
|
||||
keywords = config.keywords ? JSON.parse(config.keywords) : []
|
||||
} catch {
|
||||
keywords = []
|
||||
}
|
||||
if (keywords.length === 0) keywords = ['']
|
||||
|
||||
const today = new Date().toISOString().split('T')[0]
|
||||
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
|
||||
const seen = new Set<string>()
|
||||
const allResults: FetchResult[] = []
|
||||
|
||||
for (const kw of keywords) {
|
||||
const kwFlag = kw ? ` --keyword "${kw}"` : ''
|
||||
try {
|
||||
execSync(`node "${scriptPath}"${kwFlag}`, {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
timeout: 180000,
|
||||
})
|
||||
} catch {
|
||||
// 单个 keyword 失败不影响其他 keyword
|
||||
continue
|
||||
}
|
||||
|
||||
const today = new Date().toISOString().split('T')[0]
|
||||
const statePath = path.join(
|
||||
process.env.HOME || '',
|
||||
'desktop/招标公告',
|
||||
today,
|
||||
'downloaded.json',
|
||||
)
|
||||
|
||||
if (!fs.existsSync(statePath)) return []
|
||||
const statePath = path.join(baseDir, 'downloaded.json')
|
||||
if (!fs.existsSync(statePath)) continue
|
||||
|
||||
try {
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||
const downloaded = state.downloaded || {}
|
||||
const results: FetchResult[] = []
|
||||
|
||||
const outputDir = path.dirname(statePath)
|
||||
const pdfDir = path.join(outputDir, 'pdf')
|
||||
|
||||
for (const [key, v] of Object.entries(downloaded)) {
|
||||
const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string }
|
||||
const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string; publishTime?: string; path?: string }
|
||||
if (info.skipped || info.failed) continue
|
||||
const [systemId] = key.split('_')
|
||||
if (seen.has(systemId)) continue
|
||||
seen.add(systemId)
|
||||
|
||||
const realId = info.id || systemId
|
||||
results.push({
|
||||
|
||||
let region = ''
|
||||
let regionDetail = ''
|
||||
if (info.path && fs.existsSync(pdfDir)) {
|
||||
const pdfPath = path.join(pdfDir, path.basename(info.path))
|
||||
if (fs.existsSync(pdfPath)) {
|
||||
try {
|
||||
const extractor = path.join(SCRIPTS_DIR, 'extract_address_from_pdf.py')
|
||||
const title = info.title || ''
|
||||
const cmd = `python3 "${extractor}" "${pdfPath}" "${title}"`
|
||||
const out = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim()
|
||||
const parsed = JSON.parse(out)
|
||||
region = parsed.region || ''
|
||||
regionDetail = parsed.regionDetail || ''
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
allResults.push({
|
||||
source: '中电建',
|
||||
title: info.title || '',
|
||||
organization: '中电建',
|
||||
publishTime: state.lastRun || '',
|
||||
publishTime: info.publishTime || state.lastRun || '',
|
||||
deadline: info.deadline || '',
|
||||
noticeId: systemId,
|
||||
detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`,
|
||||
region,
|
||||
regionDetail,
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
return allResults
|
||||
}
|
||||
|
||||
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
|
||||
@ -248,10 +296,13 @@ export class CrawlerService implements OnModuleInit {
|
||||
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||
const results: FetchResult[] = []
|
||||
const todayStr = new Date().toISOString().substring(0, 10)
|
||||
|
||||
for (const [noticeId, v] of Object.entries(state)) {
|
||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string }
|
||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
|
||||
if (noticeId === 'lastRun') continue
|
||||
// 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理
|
||||
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
|
||||
const leadAmount = info.amount || ''
|
||||
results.push({
|
||||
source,
|
||||
@ -262,6 +313,8 @@ export class CrawlerService implements OnModuleInit {
|
||||
noticeId: info.id || noticeId,
|
||||
detailUrl: info.detail_url || '',
|
||||
amount: leadAmount,
|
||||
region: info.region || '',
|
||||
regionDetail: info.region_detail || '',
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
132
scripts/extract_address_from_pdf.py
Normal file
132
scripts/extract_address_from_pdf.py
Normal file
@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
从招标公告 PDF 中提取地址信息。
|
||||
用法: python3 extract_address_from_pdf.py <pdf文件路径> [公告标题]
|
||||
输出: JSON {"region": "省/市", "regionDetail": "详细地址"}
|
||||
"""
|
||||
import fitz
|
||||
import re
|
||||
import json
|
||||
import sys
|
||||
|
||||
# 全国省份/直辖市/自治区名称（含简称、别称）
|
||||
PROVINCES = [
|
||||
'北京', '天津', '上海', '重庆',
|
||||
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||
'河南', '湖北', '湖南', '广东', '海南',
|
||||
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||
]
|
||||
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
|
||||
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||
|
||||
|
||||
def extract_province_best(text):
|
||||
"""从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）"""
|
||||
if not text:
|
||||
return ''
|
||||
text_clean = text.replace(' ', '')
|
||||
# 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截）
|
||||
for prov in SPECIAL_PROVS:
|
||||
if prov in text_clean:
|
||||
return prov
|
||||
# 2. 直辖市
|
||||
for prov in ['北京', '天津', '上海', '重庆']:
|
||||
if prov in text_clean:
|
||||
return prov
|
||||
# 3. "XX省"（如"陕西省"、"广东省"）
|
||||
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
|
||||
if m:
|
||||
return m.group(1)[:2]
|
||||
# 4. 两字省份关键词
|
||||
for prov in TWO_CHAR_PROVS:
|
||||
if prov in text_clean:
|
||||
return prov
|
||||
return ''
|
||||
|
||||
|
||||
def extract_city_from_text(text):
|
||||
"""从文本中提取城市名（XX市）"""
|
||||
if not text:
|
||||
return ''
|
||||
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
|
||||
if m:
|
||||
city = m.group(1)
|
||||
if city not in ['北京市', '天津市', '上海市', '重庆市']:
|
||||
return city
|
||||
return ''
|
||||
|
||||
|
||||
def extract_address(pdf_path: str, title: str = '') -> dict:
|
||||
pdf_text = ''
|
||||
try:
|
||||
doc = fitz.open(pdf_path)
|
||||
for page in doc:
|
||||
pdf_text += page.get_text()
|
||||
doc.close()
|
||||
except Exception:
|
||||
pdf_text = ''
|
||||
|
||||
result = {'region': '', 'regionDetail': ''}
|
||||
|
||||
# ── 1. 从PDF正文中提取详细地址 ──
|
||||
addr_patterns = [
|
||||
r'建设地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'项目地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'交货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'施工地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'项目地址[：:]\s*(.+?)(?:\n|$)',
|
||||
r'供货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'到货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'送货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'工程地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'实施地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'服务地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'收货地址[：:]\s*(.+?)(?:\n|$)',
|
||||
r'所在地[：:]\s*(.+?)(?:\n|$)',
|
||||
]
|
||||
|
||||
addr = ''
|
||||
for p in addr_patterns:
|
||||
m = re.search(p, pdf_text)
|
||||
if m:
|
||||
addr = m.group(1).strip()
|
||||
if len(addr) >= 4:
|
||||
break
|
||||
addr = ''
|
||||
|
||||
if addr:
|
||||
result['regionDetail'] = addr
|
||||
prov = extract_province_best(addr)
|
||||
if prov:
|
||||
result['region'] = prov
|
||||
else:
|
||||
city = extract_city_from_text(addr)
|
||||
if city:
|
||||
result['region'] = city
|
||||
|
||||
# ── 2. 正文没提取到region，从整个PDF文本中搜索 ──
|
||||
if not result['region'] and pdf_text:
|
||||
prov = extract_province_best(pdf_text)
|
||||
if prov:
|
||||
result['region'] = prov
|
||||
|
||||
# ── 3. 还没提取到，从标题中搜索 ──
|
||||
if not result['region'] and title:
|
||||
prov = extract_province_best(title)
|
||||
if prov:
|
||||
result['region'] = prov
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print(json.dumps({'region': '', 'regionDetail': ''}))
|
||||
sys.exit(0)
|
||||
|
||||
pdf_path = sys.argv[1]
|
||||
title = sys.argv[2] if len(sys.argv) > 2 else ''
|
||||
result = extract_address(pdf_path, title)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
@ -107,7 +107,7 @@ def parse_list_page(html, type_label):
|
||||
"detail_url": BASE_URL + a_tag.get("href", ""),
|
||||
"notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "",
|
||||
"type": type_label,
|
||||
"publish_date": "",
|
||||
"publish_time": "",
|
||||
"deadline": "",
|
||||
"status": "",
|
||||
}
|
||||
@ -120,7 +120,7 @@ def parse_list_page(html, type_label):
|
||||
if "Gray" in cls:
|
||||
# 发布日期
|
||||
if re.match(r"\d{4}-\d{2}-\d{2}", text):
|
||||
notice["publish_date"] = text
|
||||
notice["publish_time"] = text
|
||||
elif "Green" in cls:
|
||||
# 正在报名
|
||||
notice["status"] = text.strip("[]")
|
||||
@ -132,7 +132,7 @@ def parse_list_page(html, type_label):
|
||||
em_tag = span.find("em", class_="Red")
|
||||
if em_tag:
|
||||
em_text = em_tag.get_text(strip=True)
|
||||
dl_match = re.search(r"文件下载截止[：:](.+)", em_text)
|
||||
dl_match = re.search(r"文件(?:发售|下载)截止[：:](.+)", em_text)
|
||||
if dl_match:
|
||||
notice["deadline"] = dl_match.group(1).strip()
|
||||
|
||||
@ -173,6 +173,39 @@ def save_state(state_path, state):
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def load_accumulated_state(output_base, state_filename, today_str):
|
||||
"""加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录"""
|
||||
accumulated = {}
|
||||
last_run_date = None
|
||||
state_dir = os.path.join(output_base, today_str)
|
||||
today_state = load_state(os.path.join(state_dir, state_filename))
|
||||
if today_state:
|
||||
for k, v in today_state.items():
|
||||
if k != "lastRun":
|
||||
accumulated[k] = v
|
||||
last_run = today_state.get("lastRun")
|
||||
if last_run:
|
||||
last_run_date = last_run[:10]
|
||||
try:
|
||||
for i in range(1, 31):
|
||||
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||
path = os.path.join(output_base, d, state_filename)
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
prev = json.load(f)
|
||||
except:
|
||||
continue
|
||||
for k, v in prev.items():
|
||||
if k != "lastRun" and k not in accumulated:
|
||||
accumulated[k] = v
|
||||
if k == "lastRun" and not last_run_date:
|
||||
last_run_date = str(v)[:10]
|
||||
except:
|
||||
pass
|
||||
return accumulated, last_run_date
|
||||
|
||||
|
||||
def generate_markdown(notices, output_dir, date_range, keyword=""):
|
||||
"""生成公告清单 Markdown"""
|
||||
lines = [
|
||||
@ -195,7 +228,7 @@ def generate_markdown(notices, output_dir, date_range, keyword=""):
|
||||
deadline_display = n['deadline'] if n['deadline'] else '-'
|
||||
lines.extend([
|
||||
f"## {i}. {n['title']}",
|
||||
f"- **发布日期**: {n['publish_date']}",
|
||||
f"- **发布日期**: {n['publish_time']}",
|
||||
f"- **公告类型**: {n['type']}",
|
||||
f"- **状态**: {status_display}",
|
||||
f"- **文件下载截止**: {deadline_display}",
|
||||
@ -225,15 +258,21 @@ def run():
|
||||
parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
|
||||
parser.add_argument("--force", action="store_true", help="强制重新抓取")
|
||||
parser.add_argument("--keyword", default="", help="关键词过滤（匹配标题）")
|
||||
parser.add_argument("--max-pages", type=int, default=0, help="最大翻页数（0=不限制）")
|
||||
parser.add_argument("--max-pages", type=int, default=10, help="最大翻页数（0=不限制，默认10页≈160条）")
|
||||
args = parser.parse_args()
|
||||
|
||||
today = date.today()
|
||||
today_str = today.strftime("%Y-%m-%d")
|
||||
|
||||
# 加载历史已下载记录（跨天去重）
|
||||
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "cnncecp_downloaded.json", today_str)
|
||||
|
||||
if args.from_date:
|
||||
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
||||
elif not args.force and last_run_date:
|
||||
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
|
||||
else:
|
||||
date_from = today - timedelta(days=7)
|
||||
date_from = today
|
||||
|
||||
if args.to_date:
|
||||
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
||||
@ -247,12 +286,12 @@ def run():
|
||||
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
||||
|
||||
# 输出目录
|
||||
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
|
||||
output_dir = os.path.join(OUTPUT_BASE, today_str)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# 状态文件
|
||||
# 状态文件（使用累积状态做去重）
|
||||
state_file = os.path.join(output_dir, "cnncecp_downloaded.json")
|
||||
state = load_state(state_file)
|
||||
state = accumulated_state
|
||||
|
||||
all_notices = []
|
||||
new_count = 0
|
||||
@ -306,7 +345,7 @@ def run():
|
||||
type_total += 1
|
||||
|
||||
# 日期过滤
|
||||
if not match_date_range(notice["publish_date"], date_from, date_to):
|
||||
if not match_date_range(notice.get("publish_time", ""), date_from, date_to):
|
||||
type_filtered += 1
|
||||
filtered_count += 1
|
||||
continue
|
||||
@ -329,7 +368,7 @@ def run():
|
||||
state[notice_id] = {
|
||||
"title": notice["title"],
|
||||
"type": notice["type"],
|
||||
"publish_date": notice["publish_date"],
|
||||
"publish_time": notice["publish_time"],
|
||||
"deadline": notice["deadline"],
|
||||
"status": notice["status"],
|
||||
"detail_url": notice["detail_url"],
|
||||
@ -352,7 +391,8 @@ def run():
|
||||
print(f" ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}")
|
||||
print()
|
||||
|
||||
# 保存状态
|
||||
# 保存状态（含 lastRun 时间戳，用于下次增量爬取）
|
||||
state["lastRun"] = datetime.now().isoformat()
|
||||
save_state(state_file, state)
|
||||
|
||||
# 生成清单
|
||||
|
||||
@ -25,9 +25,14 @@ const http = require('http');
|
||||
const crypto = require('crypto');
|
||||
|
||||
// ========== 配置区 ==========
|
||||
const KEYWORD = '水电十一局';
|
||||
|
||||
// 支持命令行传入 --keyword（例如: node fetch-v1.2.js --keyword 电缆）
|
||||
const args = process.argv.slice(2)
|
||||
const keywordIdx = args.indexOf('--keyword')
|
||||
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
|
||||
|
||||
// 公告类型：可按需增减。支持的类型值见 API 返回的 announcementType 字段
|
||||
const TYPES = ['招采公告', '资格预审公告', '采购预告', '中标结果公示'];
|
||||
const TYPES = ['招采公告'];
|
||||
// 可选：留空则从上次抓取日期继续；有值则从指定日期开始
|
||||
const START_DATE = '';
|
||||
const END_DATE = new Date().toISOString().split('T')[0]; // 今天
|
||||
@ -220,6 +225,54 @@ async function runWithConcurrency(tasks, limit) {
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------- 地址提取 ----------
|
||||
|
||||
const PROVINCES = [
|
||||
'北京', '天津', '上海', '重庆',
|
||||
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||
'河南', '湖北', '湖南', '广东', '海南',
|
||||
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||
]
|
||||
const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
|
||||
const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||
|
||||
function extractRegionFromTitle(title) {
|
||||
if (!title) return ''
|
||||
// 匹配 "XX省"、"XX市"（直辖市）、"XX自治区"
|
||||
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
|
||||
if (m) {
|
||||
const region = m[1]
|
||||
if (region.includes('新疆')) return '新疆'
|
||||
if (region.includes('广西')) return '广西'
|
||||
if (region.includes('内蒙古')) return '内蒙古'
|
||||
if (region.includes('宁夏')) return '宁夏'
|
||||
if (region.includes('西藏')) return '西藏'
|
||||
if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
|
||||
if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
|
||||
return ''
|
||||
}
|
||||
// 直接匹配省份关键词
|
||||
for (const prov of SPECIAL_PROVS) {
|
||||
if (title.includes(prov)) return prov
|
||||
}
|
||||
for (const prov of TWO_CHAR_PROVS) {
|
||||
if (title.includes(prov)) return prov
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function extractCityFromTitle(title) {
|
||||
if (!title) return ''
|
||||
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
|
||||
if (m) {
|
||||
const city = m[1]
|
||||
if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
// ---------- 生成公告清单 ----------
|
||||
|
||||
function buildAttachmentName(r, index) {
|
||||
@ -332,9 +385,11 @@ async function main() {
|
||||
|
||||
log(`共获取 ${stats.total} 条公告（分 ${page} 页）`);
|
||||
|
||||
// 过滤目标公告类型（由 TYPES 配置定义）
|
||||
const caigou = allAnnouncements.filter(r => TYPES.includes(r.announcementType));
|
||||
log(`其中 ${TYPES.join('、')}: ${caigou.length} 条`);
|
||||
// 过滤目标公告类型（由 TYPES 配置定义），且仅取货物类
|
||||
const caigou = allAnnouncements.filter(r =>
|
||||
TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
|
||||
);
|
||||
log(`其中 ${TYPES.join('、')}（货物类）: ${caigou.length} 条`);
|
||||
|
||||
// 生成公告清单（所有本次抓到的招采公告，无论是否已下载）
|
||||
const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
|
||||
@ -363,7 +418,7 @@ async function main() {
|
||||
if (!picUrl) {
|
||||
stats.skipNoAttachment++;
|
||||
log(`跳过（无附件）: ${r.title}`, 'WARN');
|
||||
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', time: new Date().toISOString() };
|
||||
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -376,15 +431,20 @@ async function main() {
|
||||
|
||||
tasks.push(async () => {
|
||||
log(`下载 [${shortName}]: ${fname}`);
|
||||
|
||||
// 从标题中提取省份（详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取）
|
||||
const region = extractRegionFromTitle(r.title) || ''
|
||||
const regionDetail = extractCityFromTitle(r.title) || ''
|
||||
|
||||
try {
|
||||
await downloadFile(picUrl, fpath);
|
||||
const size = fs.statSync(fpath).size;
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', path: `pdf/${fname}`, size, time: new Date().toISOString() };
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
|
||||
stats.success++;
|
||||
log(` ✅ ${fname} (${size} bytes)`);
|
||||
log(` ✅ ${fname} (${size} bytes) region=${region || '-'} addr=${regionDetail || '-'}`);
|
||||
} catch (err) {
|
||||
stats.failed++;
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', failed: true, error: err.message, time: new Date().toISOString() };
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
|
||||
log(` ❌ ${fname}: ${err.message}`, 'ERROR');
|
||||
}
|
||||
});
|
||||
|
||||
@ -150,6 +150,41 @@ def save_state(state_path, state):
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def load_accumulated_state(output_base, state_filename, today_str):
|
||||
"""加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录"""
|
||||
accumulated = {}
|
||||
last_run_date = None
|
||||
state_dir = os.path.join(output_base, today_str)
|
||||
# 先加载今日状态
|
||||
today_state = load_state(os.path.join(state_dir, state_filename))
|
||||
if today_state:
|
||||
for k, v in today_state.items():
|
||||
if k != "lastRun":
|
||||
accumulated[k] = v
|
||||
last_run = today_state.get("lastRun")
|
||||
if last_run:
|
||||
last_run_date = last_run[:10]
|
||||
# 扫描历史目录（最近 30 天），合并之前下载的记录
|
||||
try:
|
||||
for i in range(1, 31):
|
||||
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||
path = os.path.join(output_base, d, state_filename)
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
prev = json.load(f)
|
||||
except:
|
||||
continue
|
||||
for k, v in prev.items():
|
||||
if k != "lastRun" and k not in accumulated:
|
||||
accumulated[k] = v
|
||||
if k == "lastRun" and not last_run_date:
|
||||
last_run_date = str(v)[:10]
|
||||
except:
|
||||
pass
|
||||
return accumulated, last_run_date
|
||||
|
||||
|
||||
def generate_markdown(notices, output_dir, date_range):
|
||||
"""生成公告清单 Markdown"""
|
||||
lines = [
|
||||
@ -195,10 +230,16 @@ def run():
|
||||
|
||||
today = date.today()
|
||||
|
||||
# 加载历史已下载记录（跨天去重）
|
||||
today_str = today.strftime("%Y-%m-%d")
|
||||
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
|
||||
|
||||
if args.from_date:
|
||||
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
||||
elif not args.force and last_run_date:
|
||||
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
|
||||
else:
|
||||
date_from = today - timedelta(days=7) # 默认抓取最近 7 天
|
||||
date_from = today - timedelta(days=7)
|
||||
|
||||
if args.to_date:
|
||||
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
||||
@ -212,7 +253,7 @@ def run():
|
||||
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
||||
|
||||
# 输出目录
|
||||
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
|
||||
output_dir = os.path.join(OUTPUT_BASE, today_str)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# 子目录：html（源码） + pdf（转存）
|
||||
@ -221,9 +262,9 @@ def run():
|
||||
os.makedirs(html_dir, exist_ok=True)
|
||||
os.makedirs(pdf_dir, exist_ok=True)
|
||||
|
||||
# 状态文件
|
||||
# 状态文件（使用累积状态做去重）
|
||||
state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
|
||||
state = load_state(state_file)
|
||||
state = accumulated_state
|
||||
|
||||
all_notices = []
|
||||
new_count = 0
|
||||
@ -268,6 +309,8 @@ def run():
|
||||
type_skip += 1
|
||||
continue
|
||||
|
||||
# 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
|
||||
title_region = extract_region_from_text(notice["title"], notice["org"])
|
||||
all_notices.append(notice)
|
||||
state[notice_id] = {
|
||||
"title": notice["title"],
|
||||
@ -276,6 +319,7 @@ def run():
|
||||
"type": notice["type"],
|
||||
"detail_url": notice["detail_url"],
|
||||
"downloaded_at": datetime.now().isoformat(),
|
||||
"region": title_region,
|
||||
}
|
||||
new_count += 1
|
||||
type_new += 1
|
||||
@ -283,7 +327,8 @@ def run():
|
||||
print(f" 新增: {type_new} | 跳过: {type_skip}")
|
||||
print()
|
||||
|
||||
# 保存状态
|
||||
# 保存状态（含 lastRun 时间戳，用于下次增量爬取）
|
||||
state["lastRun"] = datetime.now().isoformat()
|
||||
save_state(state_file, state)
|
||||
|
||||
# 为缺少金额的记录从详情页提取
|
||||
@ -296,7 +341,9 @@ def run():
|
||||
# 详情页转存 PDF + HTML（新公告 + 已有状态中缺文件的）
|
||||
all_pending = list(all_notices)
|
||||
if not args.force:
|
||||
for notice_id, info in state.items():
|
||||
for notice_id, info in list(state.items()):
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
pdf_key = info.get("pdf", "")
|
||||
html_key = info.get("html", "")
|
||||
pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
|
||||
@ -323,6 +370,129 @@ def run():
|
||||
print(f"📁 保存目录: {output_dir}")
|
||||
|
||||
|
||||
def extract_deadline_from_text(text):
|
||||
"""从公告正文中提取截止日期（采购文件获取截止时间）"""
|
||||
if not text:
|
||||
return ""
|
||||
# 模式1: "起至2026年6月22日19:00（北京时间）止" → 取截止日期
|
||||
m = re.search(r"起至(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2}):(\d{2})", text)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||
# 模式2: "获取时间：...至2026-06-22 19:00"
|
||||
m = re.search(r"至\s*(\d{4})-(\d{1,2})-(\d{1,2})\s+\d{1,2}:\d{2}", text)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||
# 模式3: "截止时间：2026年6月22日" 或 "截止日期：2026-06-22"
|
||||
for pat in [
|
||||
r"截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
|
||||
r"截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
|
||||
r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
|
||||
r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
|
||||
]:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||
return ""
|
||||
|
||||
|
||||
# 全国省份/直辖市/自治区
|
||||
PROVINCES = [
|
||||
'北京', '天津', '上海', '重庆',
|
||||
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||
'河南', '湖北', '湖南', '广东', '海南',
|
||||
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||
]
|
||||
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
|
||||
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||
|
||||
|
||||
def extract_province_best(text):
|
||||
"""从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）"""
|
||||
if not text:
|
||||
return ''
|
||||
text_clean = text.replace(' ', '')
|
||||
# 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截）
|
||||
for sp in SPECIAL_PROVS:
|
||||
if sp in text_clean:
|
||||
return sp
|
||||
# 2. 直辖市
|
||||
for dc in ['北京', '天津', '上海', '重庆']:
|
||||
if dc in text_clean:
|
||||
return dc
|
||||
# 3. "XX省"（如"陕西省"、"广东省"）
|
||||
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
|
||||
if m:
|
||||
return m.group(1)[:2]
|
||||
# 4. 两字省份关键词
|
||||
for p in TWO_CHAR_PROVS:
|
||||
if p in text_clean:
|
||||
return p
|
||||
return ''
|
||||
|
||||
|
||||
def extract_city_from_text(text):
|
||||
"""从文本中提取城市名（XX市）"""
|
||||
if not text:
|
||||
return ''
|
||||
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
|
||||
if m:
|
||||
city = m.group(1)
|
||||
if city not in ['北京市', '天津市', '上海市', '重庆市']:
|
||||
return city
|
||||
return ''
|
||||
|
||||
|
||||
def extract_region_from_text(text, org=""):
|
||||
"""从公告正文或公司名中提取地区（省份）"""
|
||||
if not text and not org:
|
||||
return ""
|
||||
# 1. 从正文找项目/交货/施工地点
|
||||
if text:
|
||||
for pat in [
|
||||
r"(?:项目|交货|送货|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
|
||||
r"(?:地[址点]|所在地区)[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
|
||||
r"位\s*于\s*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
|
||||
r"(?:项目|工程)\s*(?:位于|所在地|所在)[：:：\s]*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
|
||||
]:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
region = m.group(1)
|
||||
for sp in SPECIAL_PROVS:
|
||||
if sp in region:
|
||||
return sp
|
||||
return region[:2] # 取前两个字
|
||||
# 2. 从全文扫描省份关键词
|
||||
prov = extract_province_best(text)
|
||||
if prov:
|
||||
return prov
|
||||
# 3. 从公司名推断
|
||||
if org:
|
||||
org_prov = extract_province_best(org)
|
||||
if org_prov:
|
||||
return org_prov
|
||||
return ""
|
||||
|
||||
|
||||
def extract_region_detail_from_text(text):
|
||||
"""从公告正文中提取详细收货/交货地址"""
|
||||
if not text:
|
||||
return ""
|
||||
for pat in [
|
||||
r"(?:交货|收货|送货|项目|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
|
||||
r"(?:地[址点])[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
|
||||
r"建设地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
|
||||
r"项目地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
|
||||
]:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
addr = m.group(1).strip()
|
||||
if len(addr) >= 6:
|
||||
return addr
|
||||
return ""
|
||||
|
||||
|
||||
def extract_amount_from_text(text):
|
||||
"""从公告正文中提取金额（万元）"""
|
||||
if not text:
|
||||
@ -367,6 +537,8 @@ def extract_amounts_for_existing(state, output_dir):
|
||||
for notice_id, info in state.items():
|
||||
if notice_id == "lastRun":
|
||||
continue
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
if info.get("amount"):
|
||||
continue
|
||||
pending.append(notice_id)
|
||||
@ -396,12 +568,27 @@ def extract_amounts_for_existing(state, output_dir):
|
||||
await page.close()
|
||||
|
||||
amount = extract_amount_from_text(text)
|
||||
deadline = extract_deadline_from_text(text)
|
||||
region = extract_region_from_text(text, info.get("org", ""))
|
||||
region_detail = extract_region_detail_from_text(text)
|
||||
updated = False
|
||||
if amount:
|
||||
state[notice_id]["amount"] = amount
|
||||
updated = True
|
||||
if deadline:
|
||||
state[notice_id]["deadline"] = deadline
|
||||
updated = True
|
||||
if region:
|
||||
state[notice_id]["region"] = region
|
||||
updated = True
|
||||
if region_detail:
|
||||
state[notice_id]["region_detail"] = region_detail
|
||||
updated = True
|
||||
if updated:
|
||||
success += 1
|
||||
print(f"✅ {amount}")
|
||||
print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
|
||||
else:
|
||||
print("⏭️ 未找到金额")
|
||||
print("⏭️ 未找到金额/截止日期/地区")
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
|
||||
@ -472,6 +659,15 @@ def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file
|
||||
amount = extract_amount_from_text(page_text)
|
||||
if amount and state is not None and notice_id in state:
|
||||
state[notice_id]["amount"] = amount
|
||||
deadline = extract_deadline_from_text(page_text)
|
||||
if deadline and state is not None and notice_id in state:
|
||||
state[notice_id]["deadline"] = deadline
|
||||
region = extract_region_from_text(page_text, notice.get("org", ""))
|
||||
if region and state is not None and notice_id in state:
|
||||
state[notice_id]["region"] = region
|
||||
region_detail = extract_region_detail_from_text(page_text)
|
||||
if region_detail and state is not None and notice_id in state:
|
||||
state[notice_id]["region_detail"] = region_detail
|
||||
|
||||
# 保存 HTML 源码到 html/ 目录（供 AI 阅读）
|
||||
html_source = await page.content()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user