feat: 询价标题AI压缩 + 采集器品类化 + 询价搜索与品类多选框

- TitleCompressorService: 发布前超长标题(>50字)调 MiniMax 压缩(保留公司名+物料+采购类型), JSON结构化输出对抗推理模型思考噪音, 失败硬截断兜底
- publishToDirev 集成标题压缩, 保留原50字预校验作为防御
- crawler resolveKeywords: 采集器配置优先按关联品类展开关键词库, 中电建每关键词独立状态文件
- InquiryPage: 新增关键词搜索(fSearch → GET /api/leads?search=)
- RuleCenterPage: 采集器配置由 keywords 改为品类多选框(categories JSON数组)
This commit is contained in:
Sisyphus 2026-09-30 09:34:58 +08:00
parent f12722eed9
commit eb15849168
7 changed files with 271 additions and 68 deletions

View File

@ -118,9 +118,11 @@ export class CrawlerService implements OnModuleInit {
results = await this.runGenericScript(config)
}
this.logger.log(`脚本返回 ${results.length} 条原始结果`)
let successCount = 0
const validResults = results.filter(r => r.noticeId)
this.logger.log(`有效结果 ${validResults.length} 条 (有 noticeId)`)
const categoryMap = await this.categoryInference.batchInfer(
validResults.map(r => ({ id: r.noticeId, title: r.title })),
)
@ -161,11 +163,13 @@ export class CrawlerService implements OnModuleInit {
},
})
successCount++
} catch {
// 其他异常跳过
} catch (e: unknown) {
this.logger.warn(`入库失败 [${r.noticeId}] ${r.title.slice(0, 30)}: ${(e as Error).message}`)
}
}
this.logger.log(`入库成功 ${successCount} / ${validResults.length} 条`)
return this.prisma.crawlerConfig.update({
where: { id },
data: {
@ -185,19 +189,42 @@ export class CrawlerService implements OnModuleInit {
}
}
private async runPowerChina(config: { keywords?: string | null }): Promise<FetchResult[]> {
private async resolveKeywords(config: { categories?: string | null; keywords?: string | null }): Promise<string[]> {
if (config.categories) {
try {
const catNames: string[] = JSON.parse(config.categories)
if (catNames.length > 0) {
const cats = await this.prisma.productCategory.findMany({
where: { name: { in: catNames } },
})
const kwSet = new Set<string>()
for (const cat of cats) {
try {
const kws: string[] = JSON.parse(cat.keywords)
kws.forEach((k) => kwSet.add(k))
} catch { /* skip */ }
}
if (kwSet.size > 0) return Array.from(kwSet)
}
} catch { /* fallback */ }
}
if (config.keywords) {
try {
return JSON.parse(config.keywords)
} catch { /* skip */ }
}
return ['']
}
private async runPowerChina(config: { categories?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js')
if (!fs.existsSync(scriptPath)) {
throw new Error('中电建抓取脚本不存在')
}
let keywords: string[]
try {
keywords = config.keywords ? JSON.parse(config.keywords) : []
} catch {
keywords = []
}
if (keywords.length === 0) keywords = ['']
const keywords = await this.resolveKeywords(config)
const today = new Date().toISOString().split('T')[0]
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
@ -207,22 +234,29 @@ export class CrawlerService implements OnModuleInit {
for (const kw of keywords) {
const kwFlag = kw ? ` --keyword "${kw}"` : ''
try {
this.logger.log(`执行关键词 [${kw || '(空)'}] 抓取脚本`)
execSync(`node "${scriptPath}"${kwFlag}`, {
encoding: 'utf-8',
maxBuffer: 10 * 1024 * 1024,
timeout: 180000,
})
} catch {
// 单个 keyword 失败不影响其他 keyword
} catch (e: unknown) {
this.logger.warn(`关键词 [${kw || '(空)'}] 抓取失败: ${(e as { stderr?: string; message?: string }).stderr || (e as { message?: string }).message}`)
continue
}
const statePath = path.join(baseDir, 'downloaded.json')
if (!fs.existsSync(statePath)) continue
// 每个关键词有独立的状态文件: downloaded_<keyword>.json
const stateFileName = kw ? `downloaded_${kw}.json` : 'downloaded.json'
const statePath = path.join(baseDir, stateFileName)
if (!fs.existsSync(statePath)) {
this.logger.warn(`关键词 [${kw || '(空)'}] 状态文件不存在: ${statePath}`)
continue
}
try {
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
const downloaded = state.downloaded || {}
this.logger.log(`关键词 [${kw || '(空)'}] 状态文件读取成功: ${Object.keys(downloaded).length} 条记录`)
const outputDir = path.dirname(statePath)
const pdfDir = path.join(outputDir, 'pdf')
@ -265,15 +299,17 @@ export class CrawlerService implements OnModuleInit {
regionDetail,
})
}
} catch {
} catch (e: unknown) {
this.logger.error(`关键词 [${kw || '(空)'}] 状态文件解析失败: ${(e as Error).message}`)
continue
}
}
this.logger.log(`抓取汇总: ${allResults.length} 条公告待入库`)
return allResults
}
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null; categories?: string | null; keywords?: string | null }): Promise<FetchResult[]> {
const slug = config.type.replace(/^bid_/, '')
const scriptFile = config.scriptPath || `fetch-${slug}.py`
const scriptPath = path.join(TENDER_FETCHER_DIR, scriptFile)
@ -284,13 +320,7 @@ export class CrawlerService implements OnModuleInit {
throw new Error(`抓取脚本不存在: ${scriptFile}`)
}
let keywords: string[]
try {
keywords = config.keywords ? JSON.parse(config.keywords) : []
} catch {
keywords = []
}
if (keywords.length === 0) keywords = ['']
const keywords = await this.resolveKeywords(config)
const today = new Date().toISOString().split('T')[0]
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)

View File

@ -5,13 +5,14 @@ import { CrawlerModule } from '../crawler/crawler.module'
import { ProductCategoryModule } from '../product-category/product-category.module'
import { LeadService } from './lead.service'
import { MaterialExtractorService } from './material-extractor.service'
import { TitleCompressorService } from './title-compressor.service'
import { LeadController } from './lead.controller'
import { LeadNotesController } from './lead-notes.controller'
@Module({
imports: [PrismaModule, IntegrationModule, CrawlerModule, ProductCategoryModule],
controllers: [LeadController, LeadNotesController],
providers: [LeadService, MaterialExtractorService],
providers: [LeadService, MaterialExtractorService, TitleCompressorService],
exports: [LeadService],
})
export class LeadModule {}

View File

@ -4,6 +4,7 @@ import { IntegrationService } from '../integration/integration.service'
import { ProductCategoryService } from '../product-category/product-category.service'
import { CategoryInferenceService } from '../product-category/category-inference.service'
import { MaterialExtractorService } from './material-extractor.service'
import { TitleCompressorService } from './title-compressor.service'
@Injectable()
export class LeadService {
@ -15,6 +16,7 @@ export class LeadService {
private categoryService: ProductCategoryService,
private categoryInference: CategoryInferenceService,
private materialExtractor: MaterialExtractorService,
private titleCompressor: TitleCompressorService,
) {}
async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string; timeRange?: string }) {
@ -35,11 +37,7 @@ export class LeadService {
}
if (params?.source) where.source = params.source
if (params?.category) {
const categoryRecord = await this.prisma.productCategory.findUnique({ where: { name: params.category } })
if (categoryRecord) {
const keywords: string[] = JSON.parse(categoryRecord.keywords)
where.OR = keywords.map(kw => ({ title: { contains: kw } }))
}
where.category = params.category
}
if (params?.assignedTo) {
where.assignedTo = params.assignedTo === '未分配' ? null : params.assignedTo
@ -316,9 +314,15 @@ export class LeadService {
}
// 预校验: B2B 平台字段长度限制 (tb_inquiry.title varchar(50))
// 在调 MCP 之前先报错,避免拿到通用的"服务器内部异常"
// 超长标题先 AI 压缩(保留 公司名+物料+采购类型),压缩失败自动硬截断兜底
const TITLE_MAX_LEN = 50
if (publishArgs.title && String(publishArgs.title).length > TITLE_MAX_LEN) {
publishArgs.title = await this.titleCompressor.compress(
String(publishArgs.title),
TITLE_MAX_LEN,
)
}
if (String(publishArgs.title).length > TITLE_MAX_LEN) {
const titleLen = String(publishArgs.title).length
throw new ServiceUnavailableException(
[

View File

@ -0,0 +1,88 @@
import { Injectable, Logger } from '@nestjs/common'
const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions'
const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7'
const MINIMAX_TIMEOUT_MS = 30000
// 压缩规则: 只保留 公司名(简称) + 物料 + 采购类型
// 给 B2B 平台 50 字上限留 5 字余量, 避免平台侧校验边界
const MAX_COMPRESS_LEN = 45
// M2.7 推理模型会把思考过程输出在正文里且风格多变, 行级正则过滤不可靠,
// 改为要求 JSON 结构化输出, 解析不到就硬截断兜底
const PROMPT = `把采购公告标题压缩成询价标题, 只保留: 采购公司名称(简称)、物料、采购类型。
要求:
1. 压缩后不超过 45 个汉字
2. 只输出 JSON: {"title": "压缩后的标题"}
3. 不要输出任何其他内容、解释、序号、markdown
示例:
输入: 中国电建集团港航建设有限公司 定襄县农村饮水管网改造工程第1标段项目管材采购公开询比采购公告
输出: {"title": "电建港航定襄县管网改造一标管材询比采购"}`
@Injectable()
export class TitleCompressorService {
private readonly logger = new Logger(TitleCompressorService.name)
async compress(title: string, maxLength: number): Promise<string> {
if (!title || title.length <= maxLength) return title
const apiKey = process.env.MINIMAX_API_KEY
if (!apiKey) {
this.logger.warn('MINIMAX_API_KEY 未配置, 标题硬截断')
return title.slice(0, maxLength)
}
try {
const raw = await this.callMinimax(title, apiKey, Math.min(maxLength, MAX_COMPRESS_LEN))
const parsedTitle = this.extractJsonTitle(raw)
if (!parsedTitle) {
this.logger.warn(`AI 压缩无有效输出, 硬截断: ${raw.slice(0, 100)}`)
return title.slice(0, maxLength)
}
const result = parsedTitle.length > maxLength ? parsedTitle.slice(0, maxLength) : parsedTitle
this.logger.log(`标题 AI 压缩 (${title.length}字→${result.length}字): "${result}"`)
return result
} catch (e: any) {
this.logger.warn(`AI 压缩失败, 硬截断: ${e.message}`)
return title.slice(0, maxLength)
}
}
/** M2.7 常在 JSON 前后夹杂思考文本, 用正则提取第一个合法的 {"title":"..."} 片段 */
private extractJsonTitle(raw: string): string {
const m = raw.match(/\{"title"\s*:\s*"([^"]+)"\}/)
if (m) return m[1]
const loose = raw.match(/"title"\s*:\s*"([^"]+)"/)
return loose ? loose[1] : ''
}
private async callMinimax(title: string, apiKey: string, maxLen: number): Promise<string> {
const r = await fetch(MINIMAX_ENDPOINT, {
method: 'POST',
headers: {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: MINIMAX_MODEL,
messages: [
{ role: 'system', content: PROMPT.replace('45', String(maxLen)) },
{ role: 'user', content: title },
],
temperature: 0,
max_tokens: 800, // M2.7 推理模型会输出明文思考, 400 会被思考过程耗尽导致答案截断 (AGENTS.md: 标题提取用 800)
}),
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
})
if (!r.ok) {
const errText = await r.text().catch(() => '')
throw new Error(`MiniMax API ${r.status}: ${errText.slice(0, 200)}`)
}
const data = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> }
return data.choices?.[0]?.message?.content || ''
}
}

View File

@ -111,6 +111,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
const { showToast } = useToast()
const pageSize = 10
const [fSearch, setFSearch] = useState('')
const [fTime, setFTime] = useState('全部日期')
const [fStar, setFStar] = useState('全部星级')
const [fStatus, setFStatus] = useState('全部状态')
@ -144,9 +145,10 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
const [fSource, setFSource] = useState('全部来源')
const [fPerson, setFPerson] = useState('全部人员')
const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string, timeRange?: string) => {
const loadLeads = useCallback(async (search?: string, star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string, timeRange?: string) => {
setLoading(true)
const params = new URLSearchParams()
if (search && search.trim()) params.set('search', search.trim())
if (star && star !== '全部星级') {
if (star === '5星') params.set('star', '5')
else if (star === '4星') params.set('star', '4')
@ -177,10 +179,10 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
}).catch(() => {})
}, [])
useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime])
useEffect(() => { loadLeads(fSearch, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) }, [loadLeads, fSearch, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime])
const refreshLeads = async () => {
await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime)
await loadLeads(fSearch, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime)
}
const totalPages = Math.ceil(filtered.length / pageSize)
@ -501,6 +503,26 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
</div>
)}
</div>
<div className="flex flex-col gap-1">
<span className="text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">搜索</span>
<div className="relative">
<input
type="text"
value={fSearch}
onChange={e => setFSearch(e.target.value)}
placeholder="输入关键词搜索"
className="bg-surface-container-low border border-border-subtle rounded pl-2 pr-6 py-1 text-xs w-40 focus:ring-secondary outline-none placeholder:text-slate-400"
/>
{fSearch && (
<button
type="button"
onClick={() => setFSearch('')}
className="absolute right-1 top-1/2 -translate-y-1/2 w-4 h-4 flex items-center justify-center text-slate-400 hover:text-slate-600 hover:bg-slate-200 rounded-full text-[10px]"
title="清空搜索"
>×</button>
)}
</div>
</div>
</div>
<div className="flex items-center gap-2 ml-auto">
<span className="text-xs text-on-surface-variant">批量操作:</span>

View File

@ -138,9 +138,9 @@ export default function RuleCenterPage() {
const [crawlers, setCrawlers] = useState<CrawlerConfig[]>([])
const [crawlersLoading, setCrawlersLoading] = useState(true)
const [editCrawlerId, setEditCrawlerId] = useState<string | null>(null)
const [editForm, setEditForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '', keywords: '', categories: '' })
const [editForm, setEditForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '', keywords: '', categories: '[]' })
const [showAddModal, setShowAddModal] = useState(false)
const [addForm, setAddForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '' })
const [addForm, setAddForm] = useState<{ name: string; targetUrl: string; type: string; sourceName: string; scriptPath: string; schedule: string; keywords: string; categories: string }>({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '[]' })
const [runModal, setRunModal] = useState<{ id: string; name: string } | null>(null)
const [runResult, setRunResult] = useState<'running' | 'success' | 'error'>('running')
@ -290,18 +290,17 @@ export default function RuleCenterPage() {
sourceName: c.sourceName || '',
scriptPath: c.scriptPath || '',
schedule: c.schedule || '',
keywords: c.keywords ? (() => { try { return JSON.parse(c.keywords).join(', ') } catch { return c.keywords } })() : '',
categories: c.categories ? (() => { try { return JSON.parse(c.categories).join(', ') } catch { return c.categories } })() : '',
keywords: '',
categories: c.categories ? (() => { try { const p = JSON.parse(c.categories); return Array.isArray(p) ? JSON.stringify(p) : '[]' } catch { return '[]' } })() : '[]',
})
}
const saveCrawler = async (id: string) => {
const payload: Record<string, string | null> = { name: editForm.name, targetUrl: editForm.targetUrl, type: editForm.type, sourceName: editForm.sourceName || null, scriptPath: editForm.scriptPath || null }
if (editForm.schedule) payload.schedule = editForm.schedule
const kwArr = editForm.keywords.split(/[,,、\s]+/).filter(Boolean)
payload.keywords = kwArr.length > 0 ? JSON.stringify(kwArr) : ''
const catArr = editForm.categories.split(/[,,、\s]+/).filter(Boolean)
payload.categories = catArr.length > 0 ? JSON.stringify(catArr) : ''
const catArr: string[] = editForm.categories ? JSON.parse(editForm.categories) : []
payload.categories = catArr.length > 0 ? JSON.stringify(catArr) : null
// 新配置不再使用 keywords 字段,但不清理旧数据(resolveKeywords 有 fallback)
try {
const res = await fetch(`${CRAWLER_API}/${id}`, {
method: 'PUT',
@ -334,8 +333,7 @@ export default function RuleCenterPage() {
const addCrawler = async () => {
if (!addForm.name.trim() || !addForm.targetUrl.trim()) return
const kwArr = addForm.keywords.split(/[,,、\s]+/).filter(Boolean)
const catArr = addForm.categories.split(/[,,、\s]+/).filter(Boolean)
const catArr: string[] = addForm.categories ? JSON.parse(addForm.categories) : []
try {
const res = await fetch(CRAWLER_API, {
method: 'POST',
@ -347,7 +345,7 @@ export default function RuleCenterPage() {
sourceName: addForm.sourceName || null,
scriptPath: addForm.scriptPath || null,
schedule: addForm.schedule || null,
keywords: kwArr.length > 0 ? JSON.stringify(kwArr) : null,
keywords: null,
categories: catArr.length > 0 ? JSON.stringify(catArr) : null,
}),
})
@ -355,7 +353,7 @@ export default function RuleCenterPage() {
const created: CrawlerConfig = await res.json()
setCrawlers(prev => [...prev, created])
setShowAddModal(false)
setAddForm({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '' })
setAddForm({ name: '', targetUrl: '', type: 'bid', sourceName: '', scriptPath: '', schedule: '0 */6 * * *', keywords: '', categories: '[]' })
} catch { alert('创建失败') }
}
@ -739,7 +737,7 @@ export default function RuleCenterPage() {
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">采集器名称</th>
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">目标网址</th>
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">抓取定时</th>
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">过滤关键词</th>
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">关联品类</th>
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">状态</th>
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">运行统计</th>
<th className="p-3 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase">操作</th>
@ -762,7 +760,28 @@ export default function RuleCenterPage() {
</select>
</td>
<td className="p-3">
<input value={editForm.keywords} onChange={e => setEditForm(f => ({ ...f, keywords: e.target.value }))} placeholder="用逗号分隔" className="w-full border border-border-subtle rounded px-2 py-1 text-sm outline-none focus:ring-2 focus:ring-secondary/20" />
<div className="max-h-[150px] overflow-y-auto space-y-1 text-xs">
{categories.length === 0 ? (
<span className="text-slate-400">请先在「关键词库管理」添加品类</span>
) : categories.map(cat => {
const current: string[] = editForm.categories ? JSON.parse(editForm.categories) : []
const selected = current.includes(cat.name)
return (
<label key={cat.id} className="flex items-center gap-1.5 cursor-pointer hover:bg-slate-50 rounded p-0.5">
<input
type="checkbox"
checked={selected}
onChange={() => {
const next = selected ? current.filter(n => n !== cat.name) : [...current, cat.name]
setEditForm(f => ({ ...f, categories: JSON.stringify(next) }))
}}
className="accent-secondary"
/>
{cat.name}
</label>
)
})}
</div>
</td>
<td className="p-3">
<span className={`px-2 py-0.5 text-[10px] font-medium rounded-full border ${getStatusStyle(c.status)}`}>
@ -785,15 +804,22 @@ export default function RuleCenterPage() {
<td className="p-3 text-xs text-on-surface-variant max-w-[200px] truncate">{c.targetUrl}</td>
<td className="p-3 text-xs text-on-surface-variant">{getCronLabel(c.schedule)}</td>
<td className="p-3">
{c.keywords ? (
{c.categories ? (
<div className="flex flex-wrap gap-1">
{(JSON.parse(c.keywords) as string[]).slice(0, 3).map((kw, i) => (
<span key={i} className="px-1.5 py-0.5 bg-blue-50 text-blue-600 text-[10px] rounded">{kw}</span>
{(JSON.parse(c.categories) as string[]).slice(0, 3).map((cat, i) => (
<span key={i} className="px-1.5 py-0.5 bg-purple-50 text-purple-600 text-[10px] rounded">{cat}</span>
))}
{(JSON.parse(c.keywords) as string[]).length > 3 && (
<span className="text-[10px] text-slate-400">+{JSON.parse(c.keywords).length - 3}</span>
{(JSON.parse(c.categories) as string[]).length > 3 && (
<span className="text-[10px] text-slate-400">+{JSON.parse(c.categories).length - 3}</span>
)}
</div>
) : c.keywords ? (
<div className="flex flex-wrap gap-1 items-center">
{(JSON.parse(c.keywords) as string[]).slice(0, 3).map((kw, i) => (
<span key={i} className="px-1.5 py-0.5 bg-slate-100 text-slate-500 text-[10px] rounded">{kw}</span>
))}
<span className="text-[10px] text-slate-400">(旧)</span>
</div>
) : <span className="text-xs text-slate-400">全部</span>}
</td>
<td className="p-3">
@ -1085,12 +1111,34 @@ export default function RuleCenterPage() {
</select>
</div>
<div className="col-span-2">
<span className="text-[10px] font-bold tracking-wider text-on-surface-variant uppercase block mb-1">过滤关键词(逗号分隔,留空表示抓取全部)</span>
<input value={addForm.keywords} onChange={e => setAddForm(f => ({ ...f, keywords: e.target.value }))} placeholder="电缆, 防火门, 配电箱" className="w-full border border-border-subtle rounded-lg px-3 py-2 text-sm outline-none focus:ring-2 focus:ring-secondary/20" />
</div>
<div className="col-span-2">
<span className="text-[10px] font-bold tracking-wider text-on-surface-variant uppercase block mb-1">所属品类(逗号分隔,留空表示全部品类)</span>
<input value={addForm.categories} onChange={e => setAddForm(f => ({ ...f, categories: e.target.value }))} placeholder="工程建材, 电力设备, 钢材" className="w-full border border-border-subtle rounded-lg px-3 py-2 text-sm outline-none focus:ring-2 focus:ring-secondary/20" />
<span className="text-[10px] font-bold tracking-wider text-on-surface-variant uppercase block mb-2">关联品类(选中的品类关键词用于过滤公告)</span>
<div className="max-h-[200px] overflow-y-auto border border-border-subtle rounded-lg p-3 space-y-2">
{categories.length === 0 ? (
<p className="text-xs text-on-surface-variant">暂无品类,请先在「关键词库管理」中添加品类</p>
) : categories.map(cat => {
const current: string[] = addForm.categories ? JSON.parse(addForm.categories) : []
const selected = current.includes(cat.name)
return (
<label key={cat.id} className="flex items-center gap-2 cursor-pointer hover:bg-slate-50 p-1 rounded">
<input
type="checkbox"
checked={selected}
onChange={() => {
const next = selected ? current.filter(n => n !== cat.name) : [...current, cat.name]
setAddForm(f => ({ ...f, categories: JSON.stringify(next) }))
}}
className="accent-secondary"
/>
<span className="text-sm">{cat.name}</span>
{(() => { try { const k = JSON.parse(cat.keywords); return Array.isArray(k) ? k : [] } catch { return [] } })().length > 0 && (
<span className="text-[10px] text-on-surface-variant">
({(() => { try { const k = JSON.parse(cat.keywords); return Array.isArray(k) ? k.slice(0, 3).join(', ') : '' } catch { return '' } })()})
</span>
)}
</label>
)
})}
</div>
</div>
</div>
<div className="flex gap-3 pt-2">

View File

@ -31,17 +31,20 @@ const args = process.argv.slice(2)
const keywordIdx = args.indexOf('--keyword')
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
// 公告类型:可按需增减。支持的类型值见 API 返回的 announcementType 字段
const TYPES = ['招采公告'];
// 可选:留空则从上次抓取日期继续;有值则从指定日期开始
const START_DATE = '';
// 公告类型:与 API 返回的 announcementType 字段值严格一致
const TYPES = ['招采公告', '变更公告'];
// 可选:留空则从上次抓取日期继续;有值则从指定日期开始(支持环境变量覆盖)
const START_DATE = process.env.START_DATE || '';
const END_DATE = new Date().toISOString().split('T')[0]; // 今天
// 跨天首次运行(state 为空)时回溯 N 天, 避免错过前一天后半天发布的公告
const BACKFILL_DAYS = 3;
const OUTPUT_DIR = path.join(process.env.HOME, 'desktop/招标公告', END_DATE);
const PDF_DIR = path.join(OUTPUT_DIR, 'pdf'); // PDF 附件
const HTML_DIR = path.join(OUTPUT_DIR, 'html'); // HTML 转换(待安装转换技能)
const STATE_FILE = path.join(OUTPUT_DIR, 'downloaded.json'); // 已下载记录
// 每个关键词使用独立的状态文件,避免多关键词共用状态导致后续关键词跳过
const STATE_FILE = KEYWORD
? path.join(OUTPUT_DIR, `downloaded_${KEYWORD}.json`)
: path.join(OUTPUT_DIR, 'downloaded.json');
const LOG_FILE = path.join(OUTPUT_DIR, 'fetch.log'); // 运行日志
const CONCURRENCY = 3; // 并发下载数
const RETRY_TIMES = 3;
@ -104,9 +107,10 @@ function request(url, options = {}, retries = RETRY_TIMES) {
};
const req = protocol.request(reqOptions, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
const chunks = [];
res.on('data', chunk => chunks.push(chunk));
res.on('end', () => {
const data = Buffer.concat(chunks).toString('utf8');
if (res.statusCode === 301 || res.statusCode === 302) {
const location = res.headers.location;
if (location.startsWith('http')) {
@ -348,15 +352,21 @@ async function main() {
log(`========== 开始抓取: ${KEYWORD} ==========`);
// 确定起始日期(差量:上次运行日期 或 配置的 START_DATE;跨天首次运行回溯 N 天)
const sinceDate = START_DATE || (state.lastRun ? state.lastRun.split('T')[0] : null) || (() => {
// 确定起始日期:START_DATE 为空时退缩到上次运行的前 1 天(重叠 24 小时,覆盖 cron 间隔内漏抓的公告)
// state.downloaded 通过 systemId 去重,重叠不会造成重复入库
let sinceDate = START_DATE || (state.lastRun ? state.lastRun.split('T')[0] : null) || (() => {
const d = new Date(Date.now() - BACKFILL_DAYS * 86400 * 1000);
return d.toISOString().split('T')[0];
})();
// 增加 1 天回溯,覆盖 lastRun 之后但在 cron 间隔内发布的公告
if (!START_DATE && state.lastRun) {
const yesterday = new Date(Date.now() - 86400 * 1000).toISOString().split('T')[0];
sinceDate = yesterday;
}
if (START_DATE) {
log(`指定日期范围: ${START_DATE} ~ ${END_DATE}`);
} else if (state.lastRun) {
log(`差量模式: 自 ${state.lastRun.substring(0, 10)} 以来新增`);
log(`增量模式(含 1 天回溯): ${sinceDate} ~ ${END_DATE}`);
} else if (sinceDate !== END_DATE) {
log(`跨天补抓: ${sinceDate} ~ ${END_DATE}`);
} else {