diff --git a/apps/api/src/crawler/crawler.service.ts b/apps/api/src/crawler/crawler.service.ts index 69cccb5..4814a8a 100644 --- a/apps/api/src/crawler/crawler.service.ts +++ b/apps/api/src/crawler/crawler.service.ts @@ -273,7 +273,7 @@ export class CrawlerService implements OnModuleInit { return allResults } - private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise { + private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise { const slug = config.type.replace(/^bid_/, '') const scriptFile = config.scriptPath || `fetch-${slug}.py` const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile) @@ -284,46 +284,65 @@ export class CrawlerService implements OnModuleInit { throw new Error(`抓取脚本不存在: ${scriptFile}`) } - const isJs = scriptFile.endsWith('.js') - const cmd = isJs ? `node "${scriptPath}"` : `python3 "${scriptPath}"` - execSync(cmd, { - encoding: 'utf-8', - maxBuffer: 10 * 1024 * 1024, - timeout: 180000, - }) + let keywords: string[] + try { + keywords = config.keywords ? JSON.parse(config.keywords) : [] + } catch { + keywords = [] + } + if (keywords.length === 0) keywords = [''] const today = new Date().toISOString().split('T')[0] - const statePath = path.join( - process.env.HOME || '', - 'desktop/招标公告', - today, - jsonFile, - ) + const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today) + const statePath = path.join(baseDir, jsonFile) - if (!fs.existsSync(statePath)) return [] - - const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) + const isJs = scriptFile.endsWith('.js') const results: FetchResult[] = [] + const seen = new Set() const todayStr = new Date().toISOString().substring(0, 10) - for (const [noticeId, v] of Object.entries(state)) { - const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string } - if (noticeId === 'lastRun') continue - // 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理 - if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue - const leadAmount = info.amount || '' - results.push({ - source, - title: info.title || '', - organization: info.org || source, - publishTime: info.publish_time || '', - deadline: info.deadline || '', - noticeId: info.id || noticeId, - detailUrl: info.detail_url || '', - amount: leadAmount, - region: info.region || '', - regionDetail: info.region_detail || '', - }) + for (const kw of keywords) { + const kwFlag = kw ? ` --keyword "${kw}"` : '' + const cmd = isJs ? `node "${scriptPath}"${kwFlag}` : `python3 "${scriptPath}"${kwFlag}` + try { + execSync(cmd, { + encoding: 'utf-8', + maxBuffer: 10 * 1024 * 1024, + timeout: 180000, + }) + } catch { + // 单个 keyword 失败不影响其他 keyword + continue + } + + if (!fs.existsSync(statePath)) continue + + try { + const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) + for (const [noticeId, v] of Object.entries(state)) { + if (noticeId === 'lastRun') continue + const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string } + // 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理 + if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue + const dedupeKey = info.id || noticeId + if (seen.has(dedupeKey)) continue + seen.add(dedupeKey) + results.push({ + source, + title: info.title || '', + organization: info.org || source, + publishTime: info.publish_time || '', + deadline: info.deadline || '', + noticeId: dedupeKey, + detailUrl: info.detail_url || '', + amount: info.amount || '', + region: info.region || '', + regionDetail: info.region_detail || '', + }) + } + } catch { + continue + } } return results