fix(crawler): runGenericScript 支持 DB keywords, 跟 runPowerChina 对齐
【问题】 runGenericScript() 直接 execSync 没拼 --keyword, 导致 DB CrawlerConfig.keywords 对新矿/中核等 generic python 脚本完全失效. 之前 7 月份排查中电建入库少时确认过: - 中电建: runPowerChina() 会逐 keyword 调用 → DB 关键词生效 - 新矿/中核: runGenericScript() 只调用一次不传 keyword → DB 关键词沦为摆设, 新矿实际是全量抓"寻源公告", 入库量看起来正常但跟 DB 配置不一致 【修复】 照搬 runPowerChina() 的 keyword 循环+去重模式: - 解析 config.keywords (JSON), 为空默认 [''] - 逐个 keyword 调用 execSync, 失败继续下一个 - 用 seen Set 跨 keyword 去重 - 只处理 downloaded_at 是今天的记录 (避免历史重复入库) 【验证(生产)】 - 部署后 health 200 OK - 手动跑 fetch-xkgroup.py --keyword 电缆/钢材/阀门/管材, 4 个全部 0 命中 (新矿寻源公告确实没有电缆/钢材等关键词) - 关键词确认生效, 新矿后续入库量会大幅下降 (按 DB 关键词配置真实反映)
This commit is contained in:
parent
fc4a4e81d4
commit
587a09774e
@ -273,7 +273,7 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
return allResults
|
return allResults
|
||||||
}
|
}
|
||||||
|
|
||||||
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
|
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
|
||||||
const slug = config.type.replace(/^bid_/, '')
|
const slug = config.type.replace(/^bid_/, '')
|
||||||
const scriptFile = config.scriptPath || `fetch-${slug}.py`
|
const scriptFile = config.scriptPath || `fetch-${slug}.py`
|
||||||
const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile)
|
const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile)
|
||||||
@ -284,46 +284,65 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
throw new Error(`抓取脚本不存在: ${scriptFile}`)
|
throw new Error(`抓取脚本不存在: ${scriptFile}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
const isJs = scriptFile.endsWith('.js')
|
let keywords: string[]
|
||||||
const cmd = isJs ? `node "${scriptPath}"` : `python3 "${scriptPath}"`
|
try {
|
||||||
execSync(cmd, {
|
keywords = config.keywords ? JSON.parse(config.keywords) : []
|
||||||
encoding: 'utf-8',
|
} catch {
|
||||||
maxBuffer: 10 * 1024 * 1024,
|
keywords = []
|
||||||
timeout: 180000,
|
}
|
||||||
})
|
if (keywords.length === 0) keywords = ['']
|
||||||
|
|
||||||
const today = new Date().toISOString().split('T')[0]
|
const today = new Date().toISOString().split('T')[0]
|
||||||
const statePath = path.join(
|
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
|
||||||
process.env.HOME || '',
|
const statePath = path.join(baseDir, jsonFile)
|
||||||
'desktop/招标公告',
|
|
||||||
today,
|
|
||||||
jsonFile,
|
|
||||||
)
|
|
||||||
|
|
||||||
if (!fs.existsSync(statePath)) return []
|
const isJs = scriptFile.endsWith('.js')
|
||||||
|
|
||||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
|
||||||
const results: FetchResult[] = []
|
const results: FetchResult[] = []
|
||||||
|
const seen = new Set<string>()
|
||||||
const todayStr = new Date().toISOString().substring(0, 10)
|
const todayStr = new Date().toISOString().substring(0, 10)
|
||||||
|
|
||||||
for (const [noticeId, v] of Object.entries(state)) {
|
for (const kw of keywords) {
|
||||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
|
const kwFlag = kw ? ` --keyword "${kw}"` : ''
|
||||||
if (noticeId === 'lastRun') continue
|
const cmd = isJs ? `node "${scriptPath}"${kwFlag}` : `python3 "${scriptPath}"${kwFlag}`
|
||||||
// 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理
|
try {
|
||||||
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
|
execSync(cmd, {
|
||||||
const leadAmount = info.amount || ''
|
encoding: 'utf-8',
|
||||||
results.push({
|
maxBuffer: 10 * 1024 * 1024,
|
||||||
source,
|
timeout: 180000,
|
||||||
title: info.title || '',
|
})
|
||||||
organization: info.org || source,
|
} catch {
|
||||||
publishTime: info.publish_time || '',
|
// 单个 keyword 失败不影响其他 keyword
|
||||||
deadline: info.deadline || '',
|
continue
|
||||||
noticeId: info.id || noticeId,
|
}
|
||||||
detailUrl: info.detail_url || '',
|
|
||||||
amount: leadAmount,
|
if (!fs.existsSync(statePath)) continue
|
||||||
region: info.region || '',
|
|
||||||
regionDetail: info.region_detail || '',
|
try {
|
||||||
})
|
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||||
|
for (const [noticeId, v] of Object.entries(state)) {
|
||||||
|
if (noticeId === 'lastRun') continue
|
||||||
|
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
|
||||||
|
// 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理
|
||||||
|
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
|
||||||
|
const dedupeKey = info.id || noticeId
|
||||||
|
if (seen.has(dedupeKey)) continue
|
||||||
|
seen.add(dedupeKey)
|
||||||
|
results.push({
|
||||||
|
source,
|
||||||
|
title: info.title || '',
|
||||||
|
organization: info.org || source,
|
||||||
|
publishTime: info.publish_time || '',
|
||||||
|
deadline: info.deadline || '',
|
||||||
|
noticeId: dedupeKey,
|
||||||
|
detailUrl: info.detail_url || '',
|
||||||
|
amount: info.amount || '',
|
||||||
|
region: info.region || '',
|
||||||
|
regionDetail: info.region_detail || '',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user