fix(crawler): runGenericScript 支持 DB keywords, 跟 runPowerChina 对齐
【问题】 runGenericScript() 直接 execSync 没拼 --keyword, 导致 DB CrawlerConfig.keywords 对新矿/中核等 generic python 脚本完全失效. 之前 7 月份排查中电建入库少时确认过: - 中电建: runPowerChina() 会逐 keyword 调用 → DB 关键词生效 - 新矿/中核: runGenericScript() 只调用一次不传 keyword → DB 关键词沦为摆设, 新矿实际是全量抓"寻源公告", 入库量看起来正常但跟 DB 配置不一致 【修复】 照搬 runPowerChina() 的 keyword 循环+去重模式: - 解析 config.keywords (JSON), 为空默认 [''] - 逐个 keyword 调用 execSync, 失败继续下一个 - 用 seen Set 跨 keyword 去重 - 只处理 downloaded_at 是今天的记录 (避免历史重复入库) 【验证(生产)】 - 部署后 health 200 OK - 手动跑 fetch-xkgroup.py --keyword 电缆/钢材/阀门/管材, 4 个全部 0 命中 (新矿寻源公告确实没有电缆/钢材等关键词) - 关键词确认生效, 新矿后续入库量会大幅下降 (按 DB 关键词配置真实反映)
This commit is contained in:
Sisyphus
parent
fc4a4e81d4
commit
587a09774e
@ -273,7 +273,7 @@ export class CrawlerService implements OnModuleInit {
|
||||
return allResults
|
||||
}
|
||||
|
||||
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
|
||||
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
|
||||
const slug = config.type.replace(/^bid_/, '')
|
||||
const scriptFile = config.scriptPath || `fetch-${slug}.py`
|
||||
const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile)
|
||||
@ -284,46 +284,65 @@ export class CrawlerService implements OnModuleInit {
|
||||
throw new Error(`抓取脚本不存在: ${scriptFile}`)
|
||||
}
|
||||
|
||||
const isJs = scriptFile.endsWith('.js')
|
||||
const cmd = isJs ? `node "${scriptPath}"` : `python3 "${scriptPath}"`
|
||||
execSync(cmd, {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
timeout: 180000,
|
||||
})
|
||||
let keywords: string[]
|
||||
try {
|
||||
keywords = config.keywords ? JSON.parse(config.keywords) : []
|
||||
} catch {
|
||||
keywords = []
|
||||
}
|
||||
if (keywords.length === 0) keywords = ['']
|
||||
|
||||
const today = new Date().toISOString().split('T')[0]
|
||||
const statePath = path.join(
|
||||
process.env.HOME || '',
|
||||
'desktop/招标公告',
|
||||
today,
|
||||
jsonFile,
|
||||
)
|
||||
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
|
||||
const statePath = path.join(baseDir, jsonFile)
|
||||
|
||||
if (!fs.existsSync(statePath)) return []
|
||||
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||
const isJs = scriptFile.endsWith('.js')
|
||||
const results: FetchResult[] = []
|
||||
const seen = new Set<string>()
|
||||
const todayStr = new Date().toISOString().substring(0, 10)
|
||||
|
||||
for (const [noticeId, v] of Object.entries(state)) {
|
||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
|
||||
if (noticeId === 'lastRun') continue
|
||||
// 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理
|
||||
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
|
||||
const leadAmount = info.amount || ''
|
||||
results.push({
|
||||
source,
|
||||
title: info.title || '',
|
||||
organization: info.org || source,
|
||||
publishTime: info.publish_time || '',
|
||||
deadline: info.deadline || '',
|
||||
noticeId: info.id || noticeId,
|
||||
detailUrl: info.detail_url || '',
|
||||
amount: leadAmount,
|
||||
region: info.region || '',
|
||||
regionDetail: info.region_detail || '',
|
||||
})
|
||||
for (const kw of keywords) {
|
||||
const kwFlag = kw ? ` --keyword "${kw}"` : ''
|
||||
const cmd = isJs ? `node "${scriptPath}"${kwFlag}` : `python3 "${scriptPath}"${kwFlag}`
|
||||
try {
|
||||
execSync(cmd, {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
timeout: 180000,
|
||||
})
|
||||
} catch {
|
||||
// 单个 keyword 失败不影响其他 keyword
|
||||
continue
|
||||
}
|
||||
|
||||
if (!fs.existsSync(statePath)) continue
|
||||
|
||||
try {
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||
for (const [noticeId, v] of Object.entries(state)) {
|
||||
if (noticeId === 'lastRun') continue
|
||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
|
||||
// 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理
|
||||
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
|
||||
const dedupeKey = info.id || noticeId
|
||||
if (seen.has(dedupeKey)) continue
|
||||
seen.add(dedupeKey)
|
||||
results.push({
|
||||
source,
|
||||
title: info.title || '',
|
||||
organization: info.org || source,
|
||||
publishTime: info.publish_time || '',
|
||||
deadline: info.deadline || '',
|
||||
noticeId: dedupeKey,
|
||||
detailUrl: info.detail_url || '',
|
||||
amount: info.amount || '',
|
||||
region: info.region || '',
|
||||
regionDetail: info.region_detail || '',
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
Loading…
Reference in New Issue
Block a user