fix(crawler): runGenericScript 支持 DB keywords, 跟 runPowerChina 对齐

【问题】
runGenericScript() 直接 execSync 没拼 --keyword, 导致 DB CrawlerConfig.keywords
对新矿/中核等 generic python 脚本完全失效. 之前 7 月份排查中电建入库少时确认过:
- 中电建: runPowerChina() 会逐 keyword 调用 → DB 关键词生效
- 新矿/中核: runGenericScript() 只调用一次不传 keyword → DB 关键词沦为摆设,
  新矿实际是全量抓"寻源公告", 入库量看起来正常但跟 DB 配置不一致

【修复】
照搬 runPowerChina() 的 keyword 循环+去重模式:
- 解析 config.keywords (JSON), 为空默认 ['']
- 逐个 keyword 调用 execSync, 失败继续下一个
- 用 seen Set 跨 keyword 去重
- 只处理 downloaded_at 是今天的记录 (避免历史重复入库)

【验证(生产)】
- 部署后 health 200 OK
- 手动跑 fetch-xkgroup.py --keyword 电缆/钢材/阀门/管材, 4 个全部 0 命中
  (新矿寻源公告确实没有电缆/钢材等关键词)
- 关键词确认生效, 新矿后续入库量会大幅下降 (按 DB 关键词配置真实反映)
This commit is contained in:
Sisyphus 2026-07-22 12:31:23 +08:00
parent fc4a4e81d4
commit 587a09774e

View File

@ -273,7 +273,7 @@ export class CrawlerService implements OnModuleInit {
return allResults return allResults
} }
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> { private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
const slug = config.type.replace(/^bid_/, '') const slug = config.type.replace(/^bid_/, '')
const scriptFile = config.scriptPath || `fetch-${slug}.py` const scriptFile = config.scriptPath || `fetch-${slug}.py`
const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile) const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile)
@ -284,47 +284,66 @@ export class CrawlerService implements OnModuleInit {
throw new Error(`抓取脚本不存在: ${scriptFile}`) throw new Error(`抓取脚本不存在: ${scriptFile}`)
} }
let keywords: string[]
try {
keywords = config.keywords ? JSON.parse(config.keywords) : []
} catch {
keywords = []
}
if (keywords.length === 0) keywords = ['']
const today = new Date().toISOString().split('T')[0]
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
const statePath = path.join(baseDir, jsonFile)
const isJs = scriptFile.endsWith('.js') const isJs = scriptFile.endsWith('.js')
const cmd = isJs ? `node "${scriptPath}"` : `python3 "${scriptPath}"` const results: FetchResult[] = []
const seen = new Set<string>()
const todayStr = new Date().toISOString().substring(0, 10)
for (const kw of keywords) {
const kwFlag = kw ? ` --keyword "${kw}"` : ''
const cmd = isJs ? `node "${scriptPath}"${kwFlag}` : `python3 "${scriptPath}"${kwFlag}`
try {
execSync(cmd, { execSync(cmd, {
encoding: 'utf-8', encoding: 'utf-8',
maxBuffer: 10 * 1024 * 1024, maxBuffer: 10 * 1024 * 1024,
timeout: 180000, timeout: 180000,
}) })
} catch {
// 单个 keyword 失败不影响其他 keyword
continue
}
const today = new Date().toISOString().split('T')[0] if (!fs.existsSync(statePath)) continue
const statePath = path.join(
process.env.HOME || '',
'desktop/招标公告',
today,
jsonFile,
)
if (!fs.existsSync(statePath)) return []
try {
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
const results: FetchResult[] = []
const todayStr = new Date().toISOString().substring(0, 10)
for (const [noticeId, v] of Object.entries(state)) { for (const [noticeId, v] of Object.entries(state)) {
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
if (noticeId === 'lastRun') continue if (noticeId === 'lastRun') continue
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
// 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理 // 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
const leadAmount = info.amount || '' const dedupeKey = info.id || noticeId
if (seen.has(dedupeKey)) continue
seen.add(dedupeKey)
results.push({ results.push({
source, source,
title: info.title || '', title: info.title || '',
organization: info.org || source, organization: info.org || source,
publishTime: info.publish_time || '', publishTime: info.publish_time || '',
deadline: info.deadline || '', deadline: info.deadline || '',
noticeId: info.id || noticeId, noticeId: dedupeKey,
detailUrl: info.detail_url || '', detailUrl: info.detail_url || '',
amount: leadAmount, amount: info.amount || '',
region: info.region || '', region: info.region || '',
regionDetail: info.region_detail || '', regionDetail: info.region_detail || '',
}) })
} }
} catch {
continue
}
}
return results return results
} }