From 587a09774e0f541b89307c08b14c769cc72880ea Mon Sep 17 00:00:00 2001
From: Sisyphus <sisyphus@direv.local>
Date: Wed, 22 Jul 2026 12:31:23 +0800
Subject: [PATCH] =?UTF-8?q?fix(crawler):=20runGenericScript=20=E6=94=AF?=
 =?UTF-8?q?=E6=8C=81=20DB=20keywords,=20=E8=B7=9F=20runPowerChina=20?=
 =?UTF-8?q?=E5=AF=B9=E9=BD=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

【问题】
runGenericScript() 直接 execSync 没拼 --keyword, 导致 DB CrawlerConfig.keywords
对新矿/中核等 generic python 脚本完全失效. 之前 7 月份排查中电建入库少时确认过:
- 中电建: runPowerChina() 会逐 keyword 调用 → DB 关键词生效
- 新矿/中核: runGenericScript() 只调用一次不传 keyword → DB 关键词沦为摆设,
  新矿实际是全量抓"寻源公告", 入库量看起来正常但跟 DB 配置不一致

【修复】
照搬 runPowerChina() 的 keyword 循环+去重模式:
- 解析 config.keywords (JSON), 为空默认 ['']
- 逐个 keyword 调用 execSync, 失败继续下一个
- 用 seen Set 跨 keyword 去重
- 只处理 downloaded_at 是今天的记录 (避免历史重复入库)

【验证(生产)】
- 部署后 health 200 OK
- 手动跑 fetch-xkgroup.py --keyword 电缆/钢材/阀门/管材, 4 个全部 0 命中
  (新矿寻源公告确实没有电缆/钢材等关键词)
- 关键词确认生效, 新矿后续入库量会大幅下降 (按 DB 关键词配置真实反映)
---
 apps/api/src/crawler/crawler.service.ts | 89 +++++++++++++++----------
 1 file changed, 54 insertions(+), 35 deletions(-)

diff --git a/apps/api/src/crawler/crawler.service.ts b/apps/api/src/crawler/crawler.service.ts
index 69cccb5..4814a8a 100644
--- a/apps/api/src/crawler/crawler.service.ts
+++ b/apps/api/src/crawler/crawler.service.ts
@@ -273,7 +273,7 @@ export class CrawlerService implements OnModuleInit {
     return allResults
   }
 
-  private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
+  private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
     const slug = config.type.replace(/^bid_/, '')
     const scriptFile = config.scriptPath || `fetch-${slug}.py`
     const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile)
@@ -284,46 +284,65 @@ export class CrawlerService implements OnModuleInit {
       throw new Error(`抓取脚本不存在: ${scriptFile}`)
     }
 
-    const isJs = scriptFile.endsWith('.js')
-    const cmd = isJs ? `node "${scriptPath}"` : `python3 "${scriptPath}"`
-    execSync(cmd, {
-      encoding: 'utf-8',
-      maxBuffer: 10 * 1024 * 1024,
-      timeout: 180000,
-    })
+    let keywords: string[]
+    try {
+      keywords = config.keywords ? JSON.parse(config.keywords) : []
+    } catch {
+      keywords = []
+    }
+    if (keywords.length === 0) keywords = ['']
 
     const today = new Date().toISOString().split('T')[0]
-    const statePath = path.join(
-      process.env.HOME || '',
-      'desktop/招标公告',
-      today,
-      jsonFile,
-    )
+    const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
+    const statePath = path.join(baseDir, jsonFile)
 
-    if (!fs.existsSync(statePath)) return []
-
-    const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
+    const isJs = scriptFile.endsWith('.js')
     const results: FetchResult[] = []
+    const seen = new Set<string>()
     const todayStr = new Date().toISOString().substring(0, 10)
 
-    for (const [noticeId, v] of Object.entries(state)) {
-      const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
-      if (noticeId === 'lastRun') continue
-      // 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理
-      if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
-      const leadAmount = info.amount || ''
-      results.push({
-        source,
-        title: info.title || '',
-        organization: info.org || source,
-        publishTime: info.publish_time || '',
-        deadline: info.deadline || '',
-        noticeId: info.id || noticeId,
-        detailUrl: info.detail_url || '',
-        amount: leadAmount,
-        region: info.region || '',
-        regionDetail: info.region_detail || '',
-      })
+    for (const kw of keywords) {
+      const kwFlag = kw ? ` --keyword "${kw}"` : ''
+      const cmd = isJs ? `node "${scriptPath}"${kwFlag}` : `python3 "${scriptPath}"${kwFlag}`
+      try {
+        execSync(cmd, {
+          encoding: 'utf-8',
+          maxBuffer: 10 * 1024 * 1024,
+          timeout: 180000,
+        })
+      } catch {
+        // 单个 keyword 失败不影响其他 keyword
+        continue
+      }
+
+      if (!fs.existsSync(statePath)) continue
+
+      try {
+        const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
+        for (const [noticeId, v] of Object.entries(state)) {
+          if (noticeId === 'lastRun') continue
+          const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
+          // 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理
+          if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
+          const dedupeKey = info.id || noticeId
+          if (seen.has(dedupeKey)) continue
+          seen.add(dedupeKey)
+          results.push({
+            source,
+            title: info.title || '',
+            organization: info.org || source,
+            publishTime: info.publish_time || '',
+            deadline: info.deadline || '',
+            noticeId: dedupeKey,
+            detailUrl: info.detail_url || '',
+            amount: info.amount || '',
+            region: info.region || '',
+            regionDetail: info.region_detail || '',
+          })
+        }
+      } catch {
+        continue
+      }
     }
 
     return results