fix(crawler): runGenericScript 支持 DB keywords, 跟 runPowerChina 对齐

【问题】
runGenericScript() 直接 execSync 没拼 --keyword, 导致 DB CrawlerConfig.keywords
对新矿/中核等 generic python 脚本完全失效. 之前 7 月份排查中电建入库少时确认过:
- 中电建: runPowerChina() 会逐 keyword 调用 → DB 关键词生效
- 新矿/中核: runGenericScript() 只调用一次不传 keyword → DB 关键词沦为摆设,
  新矿实际是全量抓"寻源公告", 入库量看起来正常但跟 DB 配置不一致

【修复】
照搬 runPowerChina() 的 keyword 循环+去重模式:
- 解析 config.keywords (JSON), 为空默认 ['']
- 逐个 keyword 调用 execSync, 失败继续下一个
- 用 seen Set 跨 keyword 去重
- 只处理 downloaded_at 是今天的记录 (避免历史重复入库)

【验证(生产)】
- 部署后 health 200 OK
- 手动跑 fetch-xkgroup.py --keyword 电缆/钢材/阀门/管材, 4 个全部 0 命中
  (新矿寻源公告确实没有电缆/钢材等关键词)
- 关键词确认生效, 新矿后续入库量会大幅下降 (按 DB 关键词配置真实反映)
This commit is contained in:
Sisyphus 2026-07-22 12:31:23 +08:00
parent fc4a4e81d4
commit 587a09774e

View File

@ -273,7 +273,7 @@ export class CrawlerService implements OnModuleInit {
return allResults return allResults
} }
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> { private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
const slug = config.type.replace(/^bid_/, '') const slug = config.type.replace(/^bid_/, '')
const scriptFile = config.scriptPath || `fetch-${slug}.py` const scriptFile = config.scriptPath || `fetch-${slug}.py`
const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile) const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile)
@ -284,46 +284,65 @@ export class CrawlerService implements OnModuleInit {
throw new Error(`抓取脚本不存在: ${scriptFile}`) throw new Error(`抓取脚本不存在: ${scriptFile}`)
} }
const isJs = scriptFile.endsWith('.js') let keywords: string[]
const cmd = isJs ? `node "${scriptPath}"` : `python3 "${scriptPath}"` try {
execSync(cmd, { keywords = config.keywords ? JSON.parse(config.keywords) : []
encoding: 'utf-8', } catch {
maxBuffer: 10 * 1024 * 1024, keywords = []
timeout: 180000, }
}) if (keywords.length === 0) keywords = ['']
const today = new Date().toISOString().split('T')[0] const today = new Date().toISOString().split('T')[0]
const statePath = path.join( const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
process.env.HOME || '', const statePath = path.join(baseDir, jsonFile)
'desktop/招标公告',
today,
jsonFile,
)
if (!fs.existsSync(statePath)) return [] const isJs = scriptFile.endsWith('.js')
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
const results: FetchResult[] = [] const results: FetchResult[] = []
const seen = new Set<string>()
const todayStr = new Date().toISOString().substring(0, 10) const todayStr = new Date().toISOString().substring(0, 10)
for (const [noticeId, v] of Object.entries(state)) { for (const kw of keywords) {
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string } const kwFlag = kw ? ` --keyword "${kw}"` : ''
if (noticeId === 'lastRun') continue const cmd = isJs ? `node "${scriptPath}"${kwFlag}` : `python3 "${scriptPath}"${kwFlag}`
// 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理 try {
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue execSync(cmd, {
const leadAmount = info.amount || '' encoding: 'utf-8',
results.push({ maxBuffer: 10 * 1024 * 1024,
source, timeout: 180000,
title: info.title || '', })
organization: info.org || source, } catch {
publishTime: info.publish_time || '', // 单个 keyword 失败不影响其他 keyword
deadline: info.deadline || '', continue
noticeId: info.id || noticeId, }
detailUrl: info.detail_url || '',
amount: leadAmount, if (!fs.existsSync(statePath)) continue
region: info.region || '',
regionDetail: info.region_detail || '', try {
}) const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
for (const [noticeId, v] of Object.entries(state)) {
if (noticeId === 'lastRun') continue
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
// 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
const dedupeKey = info.id || noticeId
if (seen.has(dedupeKey)) continue
seen.add(dedupeKey)
results.push({
source,
title: info.title || '',
organization: info.org || source,
publishTime: info.publish_time || '',
deadline: info.deadline || '',
noticeId: dedupeKey,
detailUrl: info.detail_url || '',
amount: info.amount || '',
region: info.region || '',
regionDetail: info.region_detail || '',
})
}
} catch {
continue
}
} }
return results return results