From eb158491684a7e01fbfaaa20ad8ff77393a30d6c Mon Sep 17 00:00:00 2001 From: Sisyphus Date: Wed, 30 Sep 2026 09:34:58 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=AF=A2=E4=BB=B7=E6=A0=87=E9=A2=98AI?= =?UTF-8?q?=E5=8E=8B=E7=BC=A9=20+=20=E9=87=87=E9=9B=86=E5=99=A8=E5=93=81?= =?UTF-8?q?=E7=B1=BB=E5=8C=96=20+=20=E8=AF=A2=E4=BB=B7=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E4=B8=8E=E5=93=81=E7=B1=BB=E5=A4=9A=E9=80=89=E6=A1=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TitleCompressorService: 发布前超长标题(>50字)调 MiniMax 压缩(保留公司名+物料+采购类型), JSON结构化输出对抗推理模型思考噪音, 失败硬截断兜底 - publishToDirev 集成标题压缩, 保留原50字预校验作为防御 - crawler resolveKeywords: 采集器配置优先按关联品类展开关键词库, 中电建每关键词独立状态文件 - InquiryPage: 新增关键词搜索(fSearch → GET /api/leads?search=) - RuleCenterPage: 采集器配置由 keywords 改为品类多选框(categories JSON数组) --- apps/api/src/crawler/crawler.service.ts | 76 +++++++++----- apps/api/src/lead/lead.module.ts | 3 +- apps/api/src/lead/lead.service.ts | 16 +-- apps/api/src/lead/title-compressor.service.ts | 88 +++++++++++++++++ apps/web/src/components/InquiryPage.tsx | 28 +++++- apps/web/src/components/RuleCenterPage.tsx | 98 ++++++++++++++----- scripts/tender-fetcher/fetch-v1.2.js | 30 ++++-- 7 files changed, 271 insertions(+), 68 deletions(-) create mode 100644 apps/api/src/lead/title-compressor.service.ts diff --git a/apps/api/src/crawler/crawler.service.ts b/apps/api/src/crawler/crawler.service.ts index 4814a8a..2424179 100644 --- a/apps/api/src/crawler/crawler.service.ts +++ b/apps/api/src/crawler/crawler.service.ts @@ -118,9 +118,11 @@ export class CrawlerService implements OnModuleInit { results = await this.runGenericScript(config) } + this.logger.log(`脚本返回 ${results.length} 条原始结果`) let successCount = 0 const validResults = results.filter(r => r.noticeId) + this.logger.log(`有效结果 ${validResults.length} 条 (有 noticeId)`) const categoryMap = await this.categoryInference.batchInfer( validResults.map(r => ({ id: r.noticeId, title: r.title })), ) @@ -161,11 +163,13 @@ export class CrawlerService implements OnModuleInit { }, }) successCount++ - } catch { - // 其他异常跳过 + } catch (e: unknown) { + this.logger.warn(`入库失败 [${r.noticeId}] ${r.title.slice(0, 30)}: ${(e as Error).message}`) } } + this.logger.log(`入库成功 ${successCount} / ${validResults.length} 条`) + return this.prisma.crawlerConfig.update({ where: { id }, data: { @@ -185,19 +189,42 @@ export class CrawlerService implements OnModuleInit { } } - private async runPowerChina(config: { keywords?: string | null }): Promise { + private async resolveKeywords(config: { categories?: string | null; keywords?: string | null }): Promise { + if (config.categories) { + try { + const catNames: string[] = JSON.parse(config.categories) + if (catNames.length > 0) { + const cats = await this.prisma.productCategory.findMany({ + where: { name: { in: catNames } }, + }) + const kwSet = new Set() + for (const cat of cats) { + try { + const kws: string[] = JSON.parse(cat.keywords) + kws.forEach((k) => kwSet.add(k)) + } catch { /* skip */ } + } + if (kwSet.size > 0) return Array.from(kwSet) + } + } catch { /* fallback */ } + } + + if (config.keywords) { + try { + return JSON.parse(config.keywords) + } catch { /* skip */ } + } + + return [''] + } + + private async runPowerChina(config: { categories?: string | null; keywords?: string | null }): Promise { const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js') if (!fs.existsSync(scriptPath)) { throw new Error('中电建抓取脚本不存在') } - let keywords: string[] - try { - keywords = config.keywords ? JSON.parse(config.keywords) : [] - } catch { - keywords = [] - } - if (keywords.length === 0) keywords = [''] + const keywords = await this.resolveKeywords(config) const today = new Date().toISOString().split('T')[0] const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today) @@ -207,22 +234,29 @@ export class CrawlerService implements OnModuleInit { for (const kw of keywords) { const kwFlag = kw ? ` --keyword "${kw}"` : '' try { + this.logger.log(`执行关键词 [${kw || '(空)'}] 抓取脚本`) execSync(`node "${scriptPath}"${kwFlag}`, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024, timeout: 180000, }) - } catch { - // 单个 keyword 失败不影响其他 keyword + } catch (e: unknown) { + this.logger.warn(`关键词 [${kw || '(空)'}] 抓取失败: ${(e as { stderr?: string; message?: string }).stderr || (e as { message?: string }).message}`) continue } - const statePath = path.join(baseDir, 'downloaded.json') - if (!fs.existsSync(statePath)) continue + // 每个关键词有独立的状态文件: downloaded_.json + const stateFileName = kw ? `downloaded_${kw}.json` : 'downloaded.json' + const statePath = path.join(baseDir, stateFileName) + if (!fs.existsSync(statePath)) { + this.logger.warn(`关键词 [${kw || '(空)'}] 状态文件不存在: ${statePath}`) + continue + } try { const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) const downloaded = state.downloaded || {} + this.logger.log(`关键词 [${kw || '(空)'}] 状态文件读取成功: ${Object.keys(downloaded).length} 条记录`) const outputDir = path.dirname(statePath) const pdfDir = path.join(outputDir, 'pdf') @@ -265,15 +299,17 @@ export class CrawlerService implements OnModuleInit { regionDetail, }) } - } catch { + } catch (e: unknown) { + this.logger.error(`关键词 [${kw || '(空)'}] 状态文件解析失败: ${(e as Error).message}`) continue } } + this.logger.log(`抓取汇总: ${allResults.length} 条公告待入库`) return allResults } - private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise { + private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; categories?: string | null; keywords?: string | null }): Promise { const slug = config.type.replace(/^bid_/, '') const scriptFile = config.scriptPath || `fetch-${slug}.py` const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile) @@ -284,13 +320,7 @@ export class CrawlerService implements OnModuleInit { throw new Error(`抓取脚本不存在: ${scriptFile}`) } - let keywords: string[] - try { - keywords = config.keywords ? JSON.parse(config.keywords) : [] - } catch { - keywords = [] - } - if (keywords.length === 0) keywords = [''] + const keywords = await this.resolveKeywords(config) const today = new Date().toISOString().split('T')[0] const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today) diff --git a/apps/api/src/lead/lead.module.ts b/apps/api/src/lead/lead.module.ts index 1a65a2b..38bec26 100644 --- a/apps/api/src/lead/lead.module.ts +++ b/apps/api/src/lead/lead.module.ts @@ -5,13 +5,14 @@ import { CrawlerModule } from '../crawler/crawler.module' import { ProductCategoryModule } from '../product-category/product-category.module' import { LeadService } from './lead.service' import { MaterialExtractorService } from './material-extractor.service' +import { TitleCompressorService } from './title-compressor.service' import { LeadController } from './lead.controller' import { LeadNotesController } from './lead-notes.controller' @Module({ imports: [PrismaModule, IntegrationModule, CrawlerModule, ProductCategoryModule], controllers: [LeadController, LeadNotesController], - providers: [LeadService, MaterialExtractorService], + providers: [LeadService, MaterialExtractorService, TitleCompressorService], exports: [LeadService], }) export class LeadModule {} diff --git a/apps/api/src/lead/lead.service.ts b/apps/api/src/lead/lead.service.ts index c872530..dbd6cb6 100644 --- a/apps/api/src/lead/lead.service.ts +++ b/apps/api/src/lead/lead.service.ts @@ -4,6 +4,7 @@ import { IntegrationService } from '../integration/integration.service' import { ProductCategoryService } from '../product-category/product-category.service' import { CategoryInferenceService } from '../product-category/category-inference.service' import { MaterialExtractorService } from './material-extractor.service' +import { TitleCompressorService } from './title-compressor.service' @Injectable() export class LeadService { @@ -15,6 +16,7 @@ export class LeadService { private categoryService: ProductCategoryService, private categoryInference: CategoryInferenceService, private materialExtractor: MaterialExtractorService, + private titleCompressor: TitleCompressorService, ) {} async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string; timeRange?: string }) { @@ -35,11 +37,7 @@ export class LeadService { } if (params?.source) where.source = params.source if (params?.category) { - const categoryRecord = await this.prisma.productCategory.findUnique({ where: { name: params.category } }) - if (categoryRecord) { - const keywords: string[] = JSON.parse(categoryRecord.keywords) - where.OR = keywords.map(kw => ({ title: { contains: kw } })) - } + where.category = params.category } if (params?.assignedTo) { where.assignedTo = params.assignedTo === '未分配' ? null : params.assignedTo @@ -316,9 +314,15 @@ export class LeadService { } // 预校验: B2B 平台字段长度限制 (tb_inquiry.title varchar(50)) - // 在调 MCP 之前先报错，避免拿到通用的"服务器内部异常" + // 超长标题先 AI 压缩（保留 公司名+物料+采购类型），压缩失败自动硬截断兜底 const TITLE_MAX_LEN = 50 if (publishArgs.title && String(publishArgs.title).length > TITLE_MAX_LEN) { + publishArgs.title = await this.titleCompressor.compress( + String(publishArgs.title), + TITLE_MAX_LEN, + ) + } + if (String(publishArgs.title).length > TITLE_MAX_LEN) { const titleLen = String(publishArgs.title).length throw new ServiceUnavailableException( [ diff --git a/apps/api/src/lead/title-compressor.service.ts b/apps/api/src/lead/title-compressor.service.ts new file mode 100644 index 0000000..f9a3760 --- /dev/null +++ b/apps/api/src/lead/title-compressor.service.ts @@ -0,0 +1,88 @@ +import { Injectable, Logger } from '@nestjs/common' + +const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions' +const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7' +const MINIMAX_TIMEOUT_MS = 30000 + +// 压缩规则: 只保留 公司名(简称) + 物料 + 采购类型 +// 给 B2B 平台 50 字上限留 5 字余量, 避免平台侧校验边界 +const MAX_COMPRESS_LEN = 45 +// M2.7 推理模型会把思考过程输出在正文里且风格多变, 行级正则过滤不可靠, +// 改为要求 JSON 结构化输出, 解析不到就硬截断兜底 +const PROMPT = `把采购公告标题压缩成询价标题, 只保留: 采购公司名称(简称)、物料、采购类型。 +要求: +1. 压缩后不超过 45 个汉字 +2. 只输出 JSON: {"title": "压缩后的标题"} +3. 不要输出任何其他内容、解释、序号、markdown + +示例: +输入: 中国电建集团港航建设有限公司 定襄县农村饮水管网改造工程第1标段项目管材采购公开询比采购公告 +输出: {"title": "电建港航定襄县管网改造一标管材询比采购"}` + +@Injectable() +export class TitleCompressorService { + private readonly logger = new Logger(TitleCompressorService.name) + + async compress(title: string, maxLength: number): Promise { + if (!title || title.length <= maxLength) return title + + const apiKey = process.env.MINIMAX_API_KEY + if (!apiKey) { + this.logger.warn('MINIMAX_API_KEY 未配置, 标题硬截断') + return title.slice(0, maxLength) + } + + try { + const raw = await this.callMinimax(title, apiKey, Math.min(maxLength, MAX_COMPRESS_LEN)) + const parsedTitle = this.extractJsonTitle(raw) + + if (!parsedTitle) { + this.logger.warn(`AI 压缩无有效输出, 硬截断: ${raw.slice(0, 100)}`) + return title.slice(0, maxLength) + } + + const result = parsedTitle.length > maxLength ? parsedTitle.slice(0, maxLength) : parsedTitle + this.logger.log(`标题 AI 压缩 (${title.length}字→${result.length}字): "${result}"`) + return result + } catch (e: any) { + this.logger.warn(`AI 压缩失败, 硬截断: ${e.message}`) + return title.slice(0, maxLength) + } + } + + /** M2.7 常在 JSON 前后夹杂思考文本, 用正则提取第一个合法的 {"title":"..."} 片段 */ + private extractJsonTitle(raw: string): string { + const m = raw.match(/\{"title"\s*:\s*"([^"]+)"\}/) + if (m) return m[1] + const loose = raw.match(/"title"\s*:\s*"([^"]+)"/) + return loose ? loose[1] : '' + } + + private async callMinimax(title: string, apiKey: string, maxLen: number): Promise { + const r = await fetch(MINIMAX_ENDPOINT, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: MINIMAX_MODEL, + messages: [ + { role: 'system', content: PROMPT.replace('45', String(maxLen)) }, + { role: 'user', content: title }, + ], + temperature: 0, + max_tokens: 800, // M2.7 推理模型会输出明文思考, 400 会被思考过程耗尽导致答案截断 (AGENTS.md: 标题提取用 800) + }), + signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS), + }) + + if (!r.ok) { + const errText = await r.text().catch(() => '') + throw new Error(`MiniMax API ${r.status}: ${errText.slice(0, 200)}`) + } + + const data = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> } + return data.choices?.[0]?.message?.content || '' + } +} \ No newline at end of file diff --git a/apps/web/src/components/InquiryPage.tsx b/apps/web/src/components/InquiryPage.tsx index 3fe736c..1070645 100644 --- a/apps/web/src/components/InquiryPage.tsx +++ b/apps/web/src/components/InquiryPage.tsx @@ -111,6 +111,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri const { showToast } = useToast() const pageSize = 10 + const [fSearch, setFSearch] = useState('') const [fTime, setFTime] = useState('全部日期') const [fStar, setFStar] = useState('全部星级') const [fStatus, setFStatus] = useState('全部状态') @@ -144,9 +145,10 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri const [fSource, setFSource] = useState('全部来源') const [fPerson, setFPerson] = useState('全部人员') - const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string, timeRange?: string) => { + const loadLeads = useCallback(async (search?: string, star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string, timeRange?: string) => { setLoading(true) const params = new URLSearchParams() + if (search && search.trim()) params.set('search', search.trim()) if (star && star !== '全部星级') { if (star === '5星') params.set('star', '5') else if (star === '4星') params.set('star', '4') @@ -177,10 +179,10 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri }).catch(() => {}) }, []) - useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime]) + useEffect(() => { loadLeads(fSearch, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) }, [loadLeads, fSearch, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime]) const refreshLeads = async () => { - await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) + await loadLeads(fSearch, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) } const totalPages = Math.ceil(filtered.length / pageSize) @@ -501,6 +503,26 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri )} +
+ 搜索 +
+ setFSearch(e.target.value)} + placeholder="输入关键词搜索" + className="bg-surface-container-low border border-border-subtle rounded pl-2 pr-6 py-1 text-xs w-40 focus:ring-secondary outline-none placeholder:text-slate-400" + /> + {fSearch && ( + + )} +
+
批量操作: diff --git a/apps/web/src/components/RuleCenterPage.tsx b/apps/web/src/components/RuleCenterPage.tsx index 4bf9509..3aaf86e 100644 --- a/apps/web/src/components/RuleCenterPage.tsx +++ b/apps/web/src/components/RuleCenterPage.tsx @@ -138,9 +138,9 @@ export default function RuleCenterPage() { const [crawlers, setCrawlers] = useState([]) const [crawlersLoading, setCrawlersLoading] = useState(true) const [editCrawlerId, setEditCrawlerId] = useState(null) - const [editForm, setEditForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '', keywords: '', categories: '' }) + const [editForm, setEditForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '', keywords: '', categories: '[]' }) const [showAddModal, setShowAddModal] = useState(false) - const [addForm, setAddForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '' }) + const [addForm, setAddForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '[]' }) const [runModal, setRunModal] = useState<{ id: string; name: string } | null>(null) const [runResult, setRunResult] = useState<'running' | 'success' | 'error'>('running') @@ -290,18 +290,17 @@ export default function RuleCenterPage() { sourceName: c.sourceName || '', scriptPath: c.scriptPath || '', schedule: c.schedule || '', - keywords: c.keywords ? (() => { try { return JSON.parse(c.keywords).join(', ') } catch { return c.keywords } })() : '', - categories: c.categories ? (() => { try { return JSON.parse(c.categories).join(', ') } catch { return c.categories } })() : '', + keywords: '', + categories: c.categories ? (() => { try { const p = JSON.parse(c.categories); return Array.isArray(p) ? JSON.stringify(p) : '[]' } catch { return '[]' } })() : '[]', }) } const saveCrawler = async (id: string) => { const payload: Record = { name: editForm.name, targetUrl: editForm.targetUrl, type: editForm.type, sourceName: editForm.sourceName || null, scriptPath: editForm.scriptPath || null } if (editForm.schedule) payload.schedule = editForm.schedule - const kwArr = editForm.keywords.split(/[,，、\s]+/).filter(Boolean) - payload.keywords = kwArr.length > 0 ? JSON.stringify(kwArr) : '' - const catArr = editForm.categories.split(/[,，、\s]+/).filter(Boolean) - payload.categories = catArr.length > 0 ? JSON.stringify(catArr) : '' + const catArr: string[] = editForm.categories ? JSON.parse(editForm.categories) : [] + payload.categories = catArr.length > 0 ? JSON.stringify(catArr) : null + // 新配置不再使用 keywords 字段，但不清理旧数据（resolveKeywords 有 fallback） try { const res = await fetch(`${CRAWLER_API}/${id}`, { method: 'PUT', @@ -334,8 +333,7 @@ export default function RuleCenterPage() { const addCrawler = async () => { if (!addForm.name.trim() || !addForm.targetUrl.trim()) return - const kwArr = addForm.keywords.split(/[,，、\s]+/).filter(Boolean) - const catArr = addForm.categories.split(/[,，、\s]+/).filter(Boolean) + const catArr: string[] = addForm.categories ? JSON.parse(addForm.categories) : [] try { const res = await fetch(CRAWLER_API, { method: 'POST', @@ -347,7 +345,7 @@ export default function RuleCenterPage() { sourceName: addForm.sourceName || null, scriptPath: addForm.scriptPath || null, schedule: addForm.schedule || null, - keywords: kwArr.length > 0 ? JSON.stringify(kwArr) : null, + keywords: null, categories: catArr.length > 0 ? JSON.stringify(catArr) : null, }), }) @@ -355,7 +353,7 @@ export default function RuleCenterPage() { const created: CrawlerConfig = await res.json() setCrawlers(prev => [...prev, created]) setShowAddModal(false) - setAddForm({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '' }) + setAddForm({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '[]' }) } catch { alert('创建失败') } } @@ -739,7 +737,7 @@ export default function RuleCenterPage() { 采集器名称 目标网址 抓取定时 - 过滤关键词 + 关联品类 状态 运行统计 操作 @@ -762,7 +760,28 @@ export default function RuleCenterPage() { - setEditForm(f => ({ ...f, keywords: e.target.value }))} placeholder="用逗号分隔" className="w-full border border-border-subtle rounded px-2 py-1 text-sm outline-none focus:ring-2 focus:ring-secondary/20" /> +
+ {categories.length === 0 ? ( + 请先在「关键词库管理」添加品类 + ) : categories.map(cat => { + const current: string[] = editForm.categories ? JSON.parse(editForm.categories) : [] + const selected = current.includes(cat.name) + return ( + + ) + })} +
@@ -785,15 +804,22 @@ export default function RuleCenterPage() { {c.targetUrl} {getCronLabel(c.schedule)} - {c.keywords ? ( + {c.categories ? (
- {(JSON.parse(c.keywords) as string[]).slice(0, 3).map((kw, i) => ( - {kw} + {(JSON.parse(c.categories) as string[]).slice(0, 3).map((cat, i) => ( + {cat} ))} - {(JSON.parse(c.keywords) as string[]).length > 3 && ( - +{JSON.parse(c.keywords).length - 3} + {(JSON.parse(c.categories) as string[]).length > 3 && ( + +{JSON.parse(c.categories).length - 3} )}
+ ) : c.keywords ? ( +
+ {(JSON.parse(c.keywords) as string[]).slice(0, 3).map((kw, i) => ( + {kw} + ))} + (旧) +
) : 全部} @@ -1085,12 +1111,34 @@ export default function RuleCenterPage() {
- 过滤关键词（逗号分隔，留空表示抓取全部） - setAddForm(f => ({ ...f, keywords: e.target.value }))} placeholder="电缆, 防火门, 配电箱" className="w-full border border-border-subtle rounded-lg px-3 py-2 text-sm outline-none focus:ring-2 focus:ring-secondary/20" /> -
-
- 所属品类（逗号分隔，留空表示全部品类） - setAddForm(f => ({ ...f, categories: e.target.value }))} placeholder="工程建材, 电力设备, 钢材" className="w-full border border-border-subtle rounded-lg px-3 py-2 text-sm outline-none focus:ring-2 focus:ring-secondary/20" /> + 关联品类（选中的品类关键词用于过滤公告） +
+ {categories.length === 0 ? ( +

暂无品类，请先在「关键词库管理」中添加品类

+ ) : categories.map(cat => { + const current: string[] = addForm.categories ? JSON.parse(addForm.categories) : [] + const selected = current.includes(cat.name) + return ( + + ) + })} +
diff --git a/scripts/tender-fetcher/fetch-v1.2.js b/scripts/tender-fetcher/fetch-v1.2.js index 9189cf0..f606344 100755 --- a/scripts/tender-fetcher/fetch-v1.2.js +++ b/scripts/tender-fetcher/fetch-v1.2.js @@ -31,17 +31,20 @@ const args = process.argv.slice(2) const keywordIdx = args.indexOf('--keyword') const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : '' -// 公告类型：可按需增减。支持的类型值见 API 返回的 announcementType 字段 -const TYPES = ['招采公告']; -// 可选：留空则从上次抓取日期继续；有值则从指定日期开始 -const START_DATE = ''; +// 公告类型：与 API 返回的 announcementType 字段值严格一致 +const TYPES = ['招采公告', '变更公告']; +// 可选：留空则从上次抓取日期继续；有值则从指定日期开始（支持环境变量覆盖） +const START_DATE = process.env.START_DATE || ''; const END_DATE = new Date().toISOString().split('T')[0]; // 今天 // 跨天首次运行(state 为空)时回溯 N 天, 避免错过前一天后半天发布的公告 const BACKFILL_DAYS = 3; const OUTPUT_DIR = path.join(process.env.HOME, 'desktop/招标公告', END_DATE); const PDF_DIR = path.join(OUTPUT_DIR, 'pdf'); // PDF 附件 const HTML_DIR = path.join(OUTPUT_DIR, 'html'); // HTML 转换（待安装转换技能） -const STATE_FILE = path.join(OUTPUT_DIR, 'downloaded.json'); // 已下载记录 +// 每个关键词使用独立的状态文件，避免多关键词共用状态导致后续关键词跳过 +const STATE_FILE = KEYWORD + ? path.join(OUTPUT_DIR, `downloaded_${KEYWORD}.json`) + : path.join(OUTPUT_DIR, 'downloaded.json'); const LOG_FILE = path.join(OUTPUT_DIR, 'fetch.log'); // 运行日志 const CONCURRENCY = 3; // 并发下载数 const RETRY_TIMES = 3; @@ -104,9 +107,10 @@ function request(url, options = {}, retries = RETRY_TIMES) { }; const req = protocol.request(reqOptions, (res) => { - let data = ''; - res.on('data', chunk => data += chunk); + const chunks = []; + res.on('data', chunk => chunks.push(chunk)); res.on('end', () => { + const data = Buffer.concat(chunks).toString('utf8'); if (res.statusCode === 301 || res.statusCode === 302) { const location = res.headers.location; if (location.startsWith('http')) { @@ -348,15 +352,21 @@ async function main() { log(`========== 开始抓取: ${KEYWORD} ==========`); - // 确定起始日期（差量：上次运行日期 或 配置的 START_DATE；跨天首次运行回溯 N 天） - const sinceDate = START_DATE || (state.lastRun ? state.lastRun.split('T')[0] : null) || (() => { + // 确定起始日期：START_DATE 为空时退缩到上次运行的前 1 天（重叠 24 小时，覆盖 cron 间隔内漏抓的公告） + // state.downloaded 通过 systemId 去重，重叠不会造成重复入库 + let sinceDate = START_DATE || (state.lastRun ? state.lastRun.split('T')[0] : null) || (() => { const d = new Date(Date.now() - BACKFILL_DAYS * 86400 * 1000); return d.toISOString().split('T')[0]; })(); + // 增加 1 天回溯，覆盖 lastRun 之后但在 cron 间隔内发布的公告 + if (!START_DATE && state.lastRun) { + const yesterday = new Date(Date.now() - 86400 * 1000).toISOString().split('T')[0]; + sinceDate = yesterday; + } if (START_DATE) { log(`指定日期范围: ${START_DATE} ~ ${END_DATE}`); } else if (state.lastRun) { - log(`差量模式: 自 ${state.lastRun.substring(0, 10)} 以来新增`); + log(`增量模式(含 1 天回溯): ${sinceDate} ~ ${END_DATE}`); } else if (sinceDate !== END_DATE) { log(`跨天补抓: ${sinceDate} ~ ${END_DATE}`); } else {