feat: 采集器地址提取优化 - 省份关键词匹配、标题兜底、跨天去重

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

apps/api/src/crawler/crawler.service.ts

@@ -18,6 +18,8 @@ interface FetchResult {
   noticeId: string
   detailUrl: string
   amount?: string
+  region?: string
+  regionDetail?: string
 }
 
 @Injectable()
@@ -109,7 +111,7 @@ export class CrawlerService implements OnModuleInit {
       let results: FetchResult[] = []
 
       if (config.type === 'bid_powerchina') {
-        results = await this.runPowerChina()
+        results = await this.runPowerChina(config)
       } else {
         results = await this.runGenericScript(config)
       }
@@ -129,6 +131,8 @@ export class CrawlerService implements OnModuleInit {
               detailUrl: r.detailUrl,
               publishTime: r.publishTime || null,
               ...(r.amount ? { amount: r.amount } : {}),
+              ...(r.region ? { region: r.region } : {}),
+              ...(r.regionDetail ? { regionDetail: r.regionDetail } : {}),
               summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
             },
             create: {
@@ -143,7 +147,8 @@ export class CrawlerService implements OnModuleInit {
               status: '待复核',
               level: 'C',
               category: '',
-              region: '',
+              region: r.region || '',
+              regionDetail: r.regionDetail || '',
               summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
             },
           })
@@ -172,49 +177,92 @@ export class CrawlerService implements OnModuleInit {
     }
   }
 
-  private async runPowerChina(): Promise<FetchResult[]> {
+  private async runPowerChina(config: { keywords?: string | null }): Promise<FetchResult[]> {
     const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js')
     if (!fs.existsSync(scriptPath)) {
       throw new Error('中电建抓取脚本不存在')
     }
 
-    execSync(`node "${scriptPath}"`, {
+    let keywords: string[]
-      encoding: 'utf-8',
+    try {
-      maxBuffer: 10 * 1024 * 1024,
+      keywords = config.keywords ? JSON.parse(config.keywords) : []
-      timeout: 180000,
+    } catch {
-    })
+      keywords = []
+    }
+    if (keywords.length === 0) keywords = ['']
 
     const today = new Date().toISOString().split('T')[0]
-    const statePath = path.join(
+    const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
-      process.env.HOME || '',
+    const seen = new Set<string>()
-      'desktop/招标公告',
+    const allResults: FetchResult[] = []
-      today,
-      'downloaded.json',
-    )
 
-    if (!fs.existsSync(statePath)) return []
+    for (const kw of keywords) {
+      const kwFlag = kw ? ` --keyword "${kw}"` : ''
+      try {
+        execSync(`node "${scriptPath}"${kwFlag}`, {
+          encoding: 'utf-8',
+          maxBuffer: 10 * 1024 * 1024,
+          timeout: 180000,
+        })
+      } catch {
+        // 单个 keyword 失败不影响其他 keyword
+        continue
+      }
 
-    const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
+      const statePath = path.join(baseDir, 'downloaded.json')
-    const downloaded = state.downloaded || {}
+      if (!fs.existsSync(statePath)) continue
-    const results: FetchResult[] = []
 
-    for (const [key, v] of Object.entries(downloaded)) {
+      try {
-      const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string }
+        const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
-      if (info.skipped || info.failed) continue
+        const downloaded = state.downloaded || {}
-      const [systemId] = key.split('_')
+
-      const realId = info.id || systemId
+        const outputDir = path.dirname(statePath)
-      results.push({
+        const pdfDir = path.join(outputDir, 'pdf')
-        source: '中电建',
+
-        title: info.title || '',
+        for (const [key, v] of Object.entries(downloaded)) {
-        organization: '中电建',
+          const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string; publishTime?: string; path?: string }
-        publishTime: state.lastRun || '',
+          if (info.skipped || info.failed) continue
-        deadline: info.deadline || '',
+          const [systemId] = key.split('_')
-        noticeId: systemId,
+          if (seen.has(systemId)) continue
-        detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`,
+          seen.add(systemId)
-      })
+
+          const realId = info.id || systemId
+
+          let region = ''
+          let regionDetail = ''
+          if (info.path && fs.existsSync(pdfDir)) {
+            const pdfPath = path.join(pdfDir, path.basename(info.path))
+            if (fs.existsSync(pdfPath)) {
+              try {
+                const extractor = path.join(SCRIPTS_DIR, 'extract_address_from_pdf.py')
+                const title = info.title || ''
+                const cmd = `python3 "${extractor}" "${pdfPath}" "${title}"`
+                const out = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim()
+                const parsed = JSON.parse(out)
+                region = parsed.region || ''
+                regionDetail = parsed.regionDetail || ''
+              } catch {}
+            }
+          }
+
+          allResults.push({
+            source: '中电建',
+            title: info.title || '',
+            organization: '中电建',
+            publishTime: info.publishTime || state.lastRun || '',
+            deadline: info.deadline || '',
+            noticeId: systemId,
+            detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`,
+            region,
+            regionDetail,
+          })
+        }
+      } catch {
+        continue
+      }
     }
 
-    return results
+    return allResults
   }
 
   private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
@@ -248,10 +296,13 @@ export class CrawlerService implements OnModuleInit {
 
     const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
     const results: FetchResult[] = []
+    const todayStr = new Date().toISOString().substring(0, 10)
 
     for (const [noticeId, v] of Object.entries(state)) {
-      const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string }
+      const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
       if (noticeId === 'lastRun') continue
+      // 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理
+      if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
       const leadAmount = info.amount || ''
       results.push({
         source,
@@ -262,6 +313,8 @@ export class CrawlerService implements OnModuleInit {
         noticeId: info.id || noticeId,
         detailUrl: info.detail_url || '',
         amount: leadAmount,
+        region: info.region || '',
+        regionDetail: info.region_detail || '',
       })
     }
 

scripts/extract_address_from_pdf.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+从招标公告 PDF 中提取地址信息。
+用法: python3 extract_address_from_pdf.py <pdf文件路径> [公告标题]
+输出: JSON {"region": "省/市", "regionDetail": "详细地址"}
+"""
+import fitz
+import re
+import json
+import sys
+
+# 全国省份/直辖市/自治区名称（含简称、别称）
+PROVINCES = [
+    '北京', '天津', '上海', '重庆',
+    '河北', '山西', '辽宁', '吉林', '黑龙江',
+    '江苏', '浙江', '安徽', '福建', '江西', '山东',
+    '河南', '湖北', '湖南', '广东', '海南',
+    '四川', '贵州', '云南', '陕西', '甘肃', '青海',
+    '台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
+]
+TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
+SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
+
+
+def extract_province_best(text):
+    """从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）"""
+    if not text:
+        return ''
+    text_clean = text.replace(' ', '')
+    # 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截）
+    for prov in SPECIAL_PROVS:
+        if prov in text_clean:
+            return prov
+    # 2. 直辖市
+    for prov in ['北京', '天津', '上海', '重庆']:
+        if prov in text_clean:
+            return prov
+    # 3. "XX省"（如"陕西省"、"广东省"）
+    m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
+    if m:
+        return m.group(1)[:2]
+    # 4. 两字省份关键词
+    for prov in TWO_CHAR_PROVS:
+        if prov in text_clean:
+            return prov
+    return ''
+
+
+def extract_city_from_text(text):
+    """从文本中提取城市名（XX市）"""
+    if not text:
+        return ''
+    m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
+    if m:
+        city = m.group(1)
+        if city not in ['北京市', '天津市', '上海市', '重庆市']:
+            return city
+    return ''
+
+
+def extract_address(pdf_path: str, title: str = '') -> dict:
+    pdf_text = ''
+    try:
+        doc = fitz.open(pdf_path)
+        for page in doc:
+            pdf_text += page.get_text()
+        doc.close()
+    except Exception:
+        pdf_text = ''
+
+    result = {'region': '', 'regionDetail': ''}
+
+    # ── 1. 从PDF正文中提取详细地址 ──
+    addr_patterns = [
+        r'建设地点[：:]\s*(.+?)(?:\n|$)',
+        r'项目地点[：:]\s*(.+?)(?:\n|$)',
+        r'交货地点[：:]\s*(.+?)(?:\n|$)',
+        r'施工地点[：:]\s*(.+?)(?:\n|$)',
+        r'项目地址[：:]\s*(.+?)(?:\n|$)',
+        r'供货地点[：:]\s*(.+?)(?:\n|$)',
+        r'到货地点[：:]\s*(.+?)(?:\n|$)',
+        r'送货地点[：:]\s*(.+?)(?:\n|$)',
+        r'工程地点[：:]\s*(.+?)(?:\n|$)',
+        r'实施地点[：:]\s*(.+?)(?:\n|$)',
+        r'服务地点[：:]\s*(.+?)(?:\n|$)',
+        r'收货地址[：:]\s*(.+?)(?:\n|$)',
+        r'所在地[：:]\s*(.+?)(?:\n|$)',
+    ]
+
+    addr = ''
+    for p in addr_patterns:
+        m = re.search(p, pdf_text)
+        if m:
+            addr = m.group(1).strip()
+            if len(addr) >= 4:
+                break
+            addr = ''
+
+    if addr:
+        result['regionDetail'] = addr
+        prov = extract_province_best(addr)
+        if prov:
+            result['region'] = prov
+        else:
+            city = extract_city_from_text(addr)
+            if city:
+                result['region'] = city
+
+    # ── 2. 正文没提取到region，从整个PDF文本中搜索 ──
+    if not result['region'] and pdf_text:
+        prov = extract_province_best(pdf_text)
+        if prov:
+            result['region'] = prov
+
+    # ── 3. 还没提取到，从标题中搜索 ──
+    if not result['region'] and title:
+        prov = extract_province_best(title)
+        if prov:
+            result['region'] = prov
+
+    return result
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print(json.dumps({'region': '', 'regionDetail': ''}))
+        sys.exit(0)
+
+    pdf_path = sys.argv[1]
+    title = sys.argv[2] if len(sys.argv) > 2 else ''
+    result = extract_address(pdf_path, title)
+    print(json.dumps(result, ensure_ascii=False))

scripts/tender-fetcher/fetch-cnncecp.py

@@ -107,7 +107,7 @@ def parse_list_page(html, type_label):
                 "detail_url": BASE_URL + a_tag.get("href", ""),
                 "notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "",
                 "type": type_label,
-                "publish_date": "",
+                "publish_time": "",
                 "deadline": "",
                 "status": "",
             }
@@ -120,7 +120,7 @@ def parse_list_page(html, type_label):
                 if "Gray" in cls:
                     # 发布日期
                     if re.match(r"\d{4}-\d{2}-\d{2}", text):
-                        notice["publish_date"] = text
+                        notice["publish_time"] = text
                 elif "Green" in cls:
                     # 正在报名
                     notice["status"] = text.strip("[]")
@@ -132,7 +132,7 @@ def parse_list_page(html, type_label):
                 em_tag = span.find("em", class_="Red")
                 if em_tag:
                     em_text = em_tag.get_text(strip=True)
-                    dl_match = re.search(r"文件下载截止[：:](.+)", em_text)
+                    dl_match = re.search(r"文件(?:发售|下载)截止[：:](.+)", em_text)
                     if dl_match:
                         notice["deadline"] = dl_match.group(1).strip()
 
@@ -173,6 +173,39 @@ def save_state(state_path, state):
         json.dump(state, f, ensure_ascii=False, indent=2)
 
 
+def load_accumulated_state(output_base, state_filename, today_str):
+    """加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录"""
+    accumulated = {}
+    last_run_date = None
+    state_dir = os.path.join(output_base, today_str)
+    today_state = load_state(os.path.join(state_dir, state_filename))
+    if today_state:
+        for k, v in today_state.items():
+            if k != "lastRun":
+                accumulated[k] = v
+        last_run = today_state.get("lastRun")
+        if last_run:
+            last_run_date = last_run[:10]
+    try:
+        for i in range(1, 31):
+            d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
+            path = os.path.join(output_base, d, state_filename)
+            if os.path.exists(path):
+                try:
+                    with open(path, "r", encoding="utf-8") as f:
+                        prev = json.load(f)
+                except:
+                    continue
+                for k, v in prev.items():
+                    if k != "lastRun" and k not in accumulated:
+                        accumulated[k] = v
+                    if k == "lastRun" and not last_run_date:
+                        last_run_date = str(v)[:10]
+    except:
+        pass
+    return accumulated, last_run_date
+
+
 def generate_markdown(notices, output_dir, date_range, keyword=""):
     """生成公告清单 Markdown"""
     lines = [
@@ -195,7 +228,7 @@ def generate_markdown(notices, output_dir, date_range, keyword=""):
         deadline_display = n['deadline'] if n['deadline'] else '-'
         lines.extend([
             f"## {i}. {n['title']}",
-            f"- **发布日期**: {n['publish_date']}",
+            f"- **发布日期**: {n['publish_time']}",
             f"- **公告类型**: {n['type']}",
             f"- **状态**: {status_display}",
             f"- **文件下载截止**: {deadline_display}",
@@ -225,15 +258,21 @@ def run():
     parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
     parser.add_argument("--force", action="store_true", help="强制重新抓取")
     parser.add_argument("--keyword", default="", help="关键词过滤（匹配标题）")
-    parser.add_argument("--max-pages", type=int, default=0, help="最大翻页数（0=不限制）")
+    parser.add_argument("--max-pages", type=int, default=10, help="最大翻页数（0=不限制，默认10页≈160条）")
     args = parser.parse_args()
 
     today = date.today()
+    today_str = today.strftime("%Y-%m-%d")
+
+    # 加载历史已下载记录（跨天去重）
+    accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "cnncecp_downloaded.json", today_str)
 
     if args.from_date:
         date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
+    elif not args.force and last_run_date:
+        date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
     else:
-        date_from = today - timedelta(days=7)
+        date_from = today
 
     if args.to_date:
         date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
@@ -247,12 +286,12 @@ def run():
     notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
 
     # 输出目录
-    output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
+    output_dir = os.path.join(OUTPUT_BASE, today_str)
     os.makedirs(output_dir, exist_ok=True)
 
-    # 状态文件
+    # 状态文件（使用累积状态做去重）
     state_file = os.path.join(output_dir, "cnncecp_downloaded.json")
-    state = load_state(state_file)
+    state = accumulated_state
 
     all_notices = []
     new_count = 0
@@ -306,7 +345,7 @@ def run():
                 type_total += 1
 
                 # 日期过滤
-                if not match_date_range(notice["publish_date"], date_from, date_to):
+                if not match_date_range(notice.get("publish_time", ""), date_from, date_to):
                     type_filtered += 1
                     filtered_count += 1
                     continue
@@ -329,7 +368,7 @@ def run():
                 state[notice_id] = {
                     "title": notice["title"],
                     "type": notice["type"],
-                    "publish_date": notice["publish_date"],
+                    "publish_time": notice["publish_time"],
                     "deadline": notice["deadline"],
                     "status": notice["status"],
                     "detail_url": notice["detail_url"],
@@ -352,7 +391,8 @@ def run():
         print(f"    ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}")
         print()
 
-    # 保存状态
+    # 保存状态（含 lastRun 时间戳，用于下次增量爬取）
+    state["lastRun"] = datetime.now().isoformat()
     save_state(state_file, state)
 
     # 生成清单

scripts/tender-fetcher/fetch-v1.2.js

@@ -25,9 +25,14 @@ const http = require('http');
 const crypto = require('crypto');
 
 // ========== 配置区 ==========
-const KEYWORD = '水电十一局';
+
+// 支持命令行传入 --keyword（例如: node fetch-v1.2.js --keyword 电缆）
+const args = process.argv.slice(2)
+const keywordIdx = args.indexOf('--keyword')
+const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
+
 // 公告类型：可按需增减。支持的类型值见 API 返回的 announcementType 字段
-const TYPES = ['招采公告', '资格预审公告', '采购预告', '中标结果公示'];
+const TYPES = ['招采公告'];
 // 可选：留空则从上次抓取日期继续；有值则从指定日期开始
 const START_DATE = '';
 const END_DATE = new Date().toISOString().split('T')[0];  // 今天
@@ -220,6 +225,54 @@ async function runWithConcurrency(tasks, limit) {
   return results;
 }
 
+// ---------- 地址提取 ----------
+
+const PROVINCES = [
+  '北京', '天津', '上海', '重庆',
+  '河北', '山西', '辽宁', '吉林', '黑龙江',
+  '江苏', '浙江', '安徽', '福建', '江西', '山东',
+  '河南', '湖北', '湖南', '广东', '海南',
+  '四川', '贵州', '云南', '陕西', '甘肃', '青海',
+  '台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
+]
+const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
+const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
+
+function extractRegionFromTitle(title) {
+  if (!title) return ''
+  // 匹配 "XX省"、"XX市"（直辖市）、"XX自治区"
+  const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
+  if (m) {
+    const region = m[1]
+    if (region.includes('新疆')) return '新疆'
+    if (region.includes('广西')) return '广西'
+    if (region.includes('内蒙古')) return '内蒙古'
+    if (region.includes('宁夏')) return '宁夏'
+    if (region.includes('西藏')) return '西藏'
+    if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
+    if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
+    return ''
+  }
+  // 直接匹配省份关键词
+  for (const prov of SPECIAL_PROVS) {
+    if (title.includes(prov)) return prov
+  }
+  for (const prov of TWO_CHAR_PROVS) {
+    if (title.includes(prov)) return prov
+  }
+  return ''
+}
+
+function extractCityFromTitle(title) {
+  if (!title) return ''
+  const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
+  if (m) {
+    const city = m[1]
+    if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
+  }
+  return ''
+}
+
 // ---------- 生成公告清单 ----------
 
 function buildAttachmentName(r, index) {
@@ -332,9 +385,11 @@ async function main() {
 
   log(`共获取 ${stats.total} 条公告（分 ${page} 页）`);
 
-  // 过滤目标公告类型（由 TYPES 配置定义）
+  // 过滤目标公告类型（由 TYPES 配置定义），且仅取货物类
-  const caigou = allAnnouncements.filter(r => TYPES.includes(r.announcementType));
+  const caigou = allAnnouncements.filter(r =>
-  log(`其中 ${TYPES.join('、')}: ${caigou.length} 条`);
+    TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
+  );
+  log(`其中 ${TYPES.join('、')}（货物类）: ${caigou.length} 条`);
 
   // 生成公告清单（所有本次抓到的招采公告，无论是否已下载）
   const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
@@ -363,7 +418,7 @@ async function main() {
     if (!picUrl) {
       stats.skipNoAttachment++;
       log(`跳过（无附件）: ${r.title}`, 'WARN');
-      state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', time: new Date().toISOString() };
+      state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
       continue;
     }
 
@@ -376,15 +431,20 @@ async function main() {
 
     tasks.push(async () => {
       log(`下载 [${shortName}]: ${fname}`);
+
+      // 从标题中提取省份（详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取）
+      const region = extractRegionFromTitle(r.title) || ''
+      const regionDetail = extractCityFromTitle(r.title) || ''
+
       try {
         await downloadFile(picUrl, fpath);
         const size = fs.statSync(fpath).size;
-        state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', path: `pdf/${fname}`, size, time: new Date().toISOString() };
+        state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
         stats.success++;
-        log(`  ✅ ${fname} (${size} bytes)`);
+        log(`  ✅ ${fname} (${size} bytes)  region=${region || '-'} addr=${regionDetail || '-'}`);
       } catch (err) {
         stats.failed++;
-        state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', failed: true, error: err.message, time: new Date().toISOString() };
+        state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
         log(`  ❌ ${fname}: ${err.message}`, 'ERROR');
       }
     });

scripts/tender-fetcher/fetch-xkgroup.py

@@ -150,6 +150,41 @@ def save_state(state_path, state):
         json.dump(state, f, ensure_ascii=False, indent=2)
 
 
+def load_accumulated_state(output_base, state_filename, today_str):
+    """加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录"""
+    accumulated = {}
+    last_run_date = None
+    state_dir = os.path.join(output_base, today_str)
+    # 先加载今日状态
+    today_state = load_state(os.path.join(state_dir, state_filename))
+    if today_state:
+        for k, v in today_state.items():
+            if k != "lastRun":
+                accumulated[k] = v
+        last_run = today_state.get("lastRun")
+        if last_run:
+            last_run_date = last_run[:10]
+    # 扫描历史目录（最近 30 天），合并之前下载的记录
+    try:
+        for i in range(1, 31):
+            d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
+            path = os.path.join(output_base, d, state_filename)
+            if os.path.exists(path):
+                try:
+                    with open(path, "r", encoding="utf-8") as f:
+                        prev = json.load(f)
+                except:
+                    continue
+                for k, v in prev.items():
+                    if k != "lastRun" and k not in accumulated:
+                        accumulated[k] = v
+                    if k == "lastRun" and not last_run_date:
+                        last_run_date = str(v)[:10]
+    except:
+        pass
+    return accumulated, last_run_date
+
+
 def generate_markdown(notices, output_dir, date_range):
     """生成公告清单 Markdown"""
     lines = [
@@ -195,10 +230,16 @@ def run():
 
     today = date.today()
 
+    # 加载历史已下载记录（跨天去重）
+    today_str = today.strftime("%Y-%m-%d")
+    accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
+
     if args.from_date:
         date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
+    elif not args.force and last_run_date:
+        date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
     else:
-        date_from = today - timedelta(days=7)  # 默认抓取最近 7 天
+        date_from = today - timedelta(days=7)
 
     if args.to_date:
         date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
@@ -212,7 +253,7 @@ def run():
     notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
 
     # 输出目录
-    output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
+    output_dir = os.path.join(OUTPUT_BASE, today_str)
     os.makedirs(output_dir, exist_ok=True)
 
     # 子目录：html（源码） + pdf（转存）
@@ -221,9 +262,9 @@ def run():
     os.makedirs(html_dir, exist_ok=True)
     os.makedirs(pdf_dir, exist_ok=True)
 
-    # 状态文件
+    # 状态文件（使用累积状态做去重）
     state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
-    state = load_state(state_file)
+    state = accumulated_state
 
     all_notices = []
     new_count = 0
@@ -268,6 +309,8 @@ def run():
                 type_skip += 1
                 continue
 
+            # 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
+            title_region = extract_region_from_text(notice["title"], notice["org"])
             all_notices.append(notice)
             state[notice_id] = {
                 "title": notice["title"],
@@ -276,6 +319,7 @@ def run():
                 "type": notice["type"],
                 "detail_url": notice["detail_url"],
                 "downloaded_at": datetime.now().isoformat(),
+                "region": title_region,
             }
             new_count += 1
             type_new += 1
@@ -283,7 +327,8 @@ def run():
         print(f"      新增: {type_new} | 跳过: {type_skip}")
         print()
 
-    # 保存状态
+    # 保存状态（含 lastRun 时间戳，用于下次增量爬取）
+    state["lastRun"] = datetime.now().isoformat()
     save_state(state_file, state)
 
     # 为缺少金额的记录从详情页提取
@@ -296,7 +341,9 @@ def run():
     # 详情页转存 PDF + HTML（新公告 + 已有状态中缺文件的）
     all_pending = list(all_notices)
     if not args.force:
-        for notice_id, info in state.items():
+        for notice_id, info in list(state.items()):
+            if not isinstance(info, dict):
+                continue
             pdf_key = info.get("pdf", "")
             html_key = info.get("html", "")
             pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
@@ -323,6 +370,129 @@ def run():
     print(f"📁 保存目录: {output_dir}")
 
 
+def extract_deadline_from_text(text):
+    """从公告正文中提取截止日期（采购文件获取截止时间）"""
+    if not text:
+        return ""
+    # 模式1: "起至2026年6月22日19:00（北京时间）止" → 取截止日期
+    m = re.search(r"起至(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2}):(\d{2})", text)
+    if m:
+        return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
+    # 模式2: "获取时间：...至2026-06-22 19:00"
+    m = re.search(r"至\s*(\d{4})-(\d{1,2})-(\d{1,2})\s+\d{1,2}:\d{2}", text)
+    if m:
+        return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
+    # 模式3: "截止时间：2026年6月22日" 或 "截止日期：2026-06-22"
+    for pat in [
+        r"截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
+        r"截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
+        r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
+        r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
+    ]:
+        m = re.search(pat, text)
+        if m:
+            return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
+    return ""
+
+
+# 全国省份/直辖市/自治区
+PROVINCES = [
+    '北京', '天津', '上海', '重庆',
+    '河北', '山西', '辽宁', '吉林', '黑龙江',
+    '江苏', '浙江', '安徽', '福建', '江西', '山东',
+    '河南', '湖北', '湖南', '广东', '海南',
+    '四川', '贵州', '云南', '陕西', '甘肃', '青海',
+    '台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
+]
+TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
+SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
+
+
+def extract_province_best(text):
+    """从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）"""
+    if not text:
+        return ''
+    text_clean = text.replace(' ', '')
+    # 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截）
+    for sp in SPECIAL_PROVS:
+        if sp in text_clean:
+            return sp
+    # 2. 直辖市
+    for dc in ['北京', '天津', '上海', '重庆']:
+        if dc in text_clean:
+            return dc
+    # 3. "XX省"（如"陕西省"、"广东省"）
+    m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
+    if m:
+        return m.group(1)[:2]
+    # 4. 两字省份关键词
+    for p in TWO_CHAR_PROVS:
+        if p in text_clean:
+            return p
+    return ''
+
+
+def extract_city_from_text(text):
+    """从文本中提取城市名（XX市）"""
+    if not text:
+        return ''
+    m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
+    if m:
+        city = m.group(1)
+        if city not in ['北京市', '天津市', '上海市', '重庆市']:
+            return city
+    return ''
+
+
+def extract_region_from_text(text, org=""):
+    """从公告正文或公司名中提取地区（省份）"""
+    if not text and not org:
+        return ""
+    # 1. 从正文找项目/交货/施工地点
+    if text:
+        for pat in [
+            r"(?:项目|交货|送货|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
+            r"(?:地[址点]|所在地区)[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
+            r"位\s*于\s*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
+            r"(?:项目|工程)\s*(?:位于|所在地|所在)[：:：\s]*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
+        ]:
+            m = re.search(pat, text)
+            if m:
+                region = m.group(1)
+                for sp in SPECIAL_PROVS:
+                    if sp in region:
+                        return sp
+                return region[:2]  # 取前两个字
+        # 2. 从全文扫描省份关键词
+        prov = extract_province_best(text)
+        if prov:
+            return prov
+    # 3. 从公司名推断
+    if org:
+        org_prov = extract_province_best(org)
+        if org_prov:
+            return org_prov
+    return ""
+
+
+def extract_region_detail_from_text(text):
+    """从公告正文中提取详细收货/交货地址"""
+    if not text:
+        return ""
+    for pat in [
+        r"(?:交货|收货|送货|项目|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
+        r"(?:地[址点])[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
+        r"建设地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
+        r"项目地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
+    ]:
+        m = re.search(pat, text)
+        if m:
+            addr = m.group(1).strip()
+            if len(addr) >= 6:
+                return addr
+    return ""
+
+
 def extract_amount_from_text(text):
     """从公告正文中提取金额（万元）"""
     if not text:
@@ -367,6 +537,8 @@ def extract_amounts_for_existing(state, output_dir):
     for notice_id, info in state.items():
         if notice_id == "lastRun":
             continue
+        if not isinstance(info, dict):
+            continue
         if info.get("amount"):
             continue
         pending.append(notice_id)
@@ -396,12 +568,27 @@ def extract_amounts_for_existing(state, output_dir):
                     await page.close()
 
                     amount = extract_amount_from_text(text)
+                    deadline = extract_deadline_from_text(text)
+                    region = extract_region_from_text(text, info.get("org", ""))
+                    region_detail = extract_region_detail_from_text(text)
+                    updated = False
                     if amount:
                         state[notice_id]["amount"] = amount
+                        updated = True
+                    if deadline:
+                        state[notice_id]["deadline"] = deadline
+                        updated = True
+                    if region:
+                        state[notice_id]["region"] = region
+                        updated = True
+                    if region_detail:
+                        state[notice_id]["region_detail"] = region_detail
+                        updated = True
+                    if updated:
                         success += 1
-                        print(f"✅ {amount}")
+                        print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
                     else:
-                        print("⏭️ 未找到金额")
+                        print("⏭️ 未找到金额/截止日期/地区")
                 except Exception as e:
                     print(f"❌ {e}")
 
@@ -472,6 +659,15 @@ def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file
                     amount = extract_amount_from_text(page_text)
                     if amount and state is not None and notice_id in state:
                         state[notice_id]["amount"] = amount
+                    deadline = extract_deadline_from_text(page_text)
+                    if deadline and state is not None and notice_id in state:
+                        state[notice_id]["deadline"] = deadline
+                    region = extract_region_from_text(page_text, notice.get("org", ""))
+                    if region and state is not None and notice_id in state:
+                        state[notice_id]["region"] = region
+                    region_detail = extract_region_detail_from_text(page_text)
+                    if region_detail and state is not None and notice_id in state:
+                        state[notice_id]["region_detail"] = region_detail
 
                     # 保存 HTML 源码到 html/ 目录（供 AI 阅读）
                     html_source = await page.content()