From 58ac14080272761120e2daf977f89ac9e2483ffb Mon Sep 17 00:00:00 2001 From: Sisyphus Date: Tue, 30 Jun 2026 18:37:23 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=93=81=E7=B1=BB=E6=96=B9=E6=A1=88C?= =?UTF-8?q?=20-=20AI=E5=88=86=E7=B1=BB=20+=20=E5=85=B3=E9=94=AE=E8=AF=8D?= =?UTF-8?q?=E8=A1=A8=E5=8F=8D=E5=93=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 ProductCategory 表（schema + migration + seed 15个品类） - 新增 product-category CRUD API（GET/POST/PUT/DELETE /api/categories） - 新增 CategoryInferenceService：关键词匹配优先，失败时调 DeepSeek AI 分类，结果反写品类表 - 更新 CrawlerService：采集入库时使用 batchInfer 替代硬编码 inferCategory - 更新 LeadService：品类筛选改为 DB 关键词匹配，移除硬编码 CATEGORY_KEYWORDS - 修复 LeadModule/CrawlerModule 依赖关系，避免循环依赖 --- .../migration.sql | 12 ++ apps/api/prisma/schema.prisma | 9 + apps/api/prisma/seed.ts | 27 +++ apps/api/src/app.module.ts | 2 + apps/api/src/crawler/crawler.module.ts | 3 +- apps/api/src/crawler/crawler.service.ts | 16 +- .../src/lead/category-inference.service.ts | 166 ++++++++++++++++++ apps/api/src/lead/lead.module.ts | 3 +- apps/api/src/lead/lead.service.ts | 39 +--- .../product-category.controller.ts | 27 +++ .../product-category.module.ts | 13 ++ .../product-category.service.ts | 72 ++++++++ 12 files changed, 350 insertions(+), 39 deletions(-) create mode 100644 apps/api/prisma/migrations/20260630000000_add_product_category/migration.sql create mode 100644 apps/api/src/lead/category-inference.service.ts create mode 100644 apps/api/src/product-category/product-category.controller.ts create mode 100644 apps/api/src/product-category/product-category.module.ts create mode 100644 apps/api/src/product-category/product-category.service.ts diff --git a/apps/api/prisma/migrations/20260630000000_add_product_category/migration.sql b/apps/api/prisma/migrations/20260630000000_add_product_category/migration.sql new file mode 100644 index 0000000..513a111 --- /dev/null +++ b/apps/api/prisma/migrations/20260630000000_add_product_category/migration.sql @@ -0,0 +1,12 @@ +-- CreateTable +CREATE TABLE "ProductCategory" ( + "id" TEXT NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL, + "keywords" TEXT NOT NULL, + "isAIGenerated" BOOLEAN NOT NULL DEFAULT false, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL +); + +-- CreateIndex +CREATE UNIQUE INDEX "ProductCategory_name_key" ON "ProductCategory"("name"); diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 11de51d..b93a4aa 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -230,3 +230,12 @@ model Integration { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt } + +model ProductCategory { + id String @id @default(uuid()) + name String @unique // 品类名, 如 "电线电缆" + keywords String // JSON 数组: ["电缆","电线","导线"] + isAIGenerated Boolean @default(false) // 是否由 AI 自动分类生成 + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} diff --git a/apps/api/prisma/seed.ts b/apps/api/prisma/seed.ts index 6f89086..acfbe20 100644 --- a/apps/api/prisma/seed.ts +++ b/apps/api/prisma/seed.ts @@ -262,6 +262,33 @@ async function main() { }) console.log('Seed data created') + + const categories = [ + { name: '防火门', keywords: ['防火门', '防火卷帘', '防火窗', '钢质防火', '木质防火'] }, + { name: '配电箱', keywords: ['配电箱', '配电柜', '配电盘', '照明配电箱', '动力配电箱', '电表箱'] }, + { name: '电线电缆', keywords: ['电线', '电缆', '导线', '电力线', '控制电缆', '光缆', '架空线'] }, + { name: '阀门', keywords: ['阀门', '闸阀', '球阀', '蝶阀', '截止阀', '止回阀', '调节阀'] }, + { name: '钢材', keywords: ['钢材', '钢管', '钢板', '钢筋', '型钢', '槽钢', '角钢', '工字钢', 'H型钢', '螺纹钢'] }, + { name: '涂料', keywords: ['涂料', '油漆', '防腐漆', '防火涂料', '乳胶漆', '防水涂料'] }, + { name: '混凝土', keywords: ['混凝土', '商混', '商品砼', '水泥', '砂浆', '砼'] }, + { name: '变压器', keywords: ['变压器', '箱变', '干式变', '油浸变', '电力变压器'] }, + { name: '水泵', keywords: ['水泵', '离心泵', '潜污泵', '消防泵', '排污泵', '循环泵'] }, + { name: '风机', keywords: ['风机', '通风机', '排风机', '轴流风机', '离心风机'] }, + { name: '管材', keywords: ['管材', '管道', 'PVC管', 'PE管', '镀锌管', '无缝管', '螺旋管', '给水管'] }, + { name: '开关', keywords: ['开关', '断路器', '接触器', '继电器', '熔断器', '隔离开关'] }, + { name: '灯具', keywords: ['灯具', '照明灯', '路灯', '防爆灯', 'LED灯', '太阳能灯'] }, + { name: '仪表', keywords: ['仪表', '仪器', '传感器', '变送器', '流量计', '压力表', '温度计'] }, + { name: '电机', keywords: ['电机', '电动机', '减速机', '减速器', '变频电机'] }, + ] + + for (const cat of categories) { + await prisma.productCategory.upsert({ + where: { name: cat.name }, + update: { keywords: JSON.stringify(cat.keywords) }, + create: { name: cat.name, keywords: JSON.stringify(cat.keywords) }, + }) + } + console.log('ProductCategory seed data created') } main() diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 344e9f0..772f1cf 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -13,6 +13,7 @@ import { AuthModule } from './auth/auth.module' import { UsersModule } from './users/users.module' import { AdminModule } from './admin/admin.module' import { IntegrationModule } from './integration/integration.module' +import { ProductCategoryModule } from './product-category/product-category.module' import { HealthController } from './common/health.controller' @Module({ @@ -31,6 +32,7 @@ import { HealthController } from './common/health.controller' CrawlerModule, AuditLogModule, IntegrationModule, + ProductCategoryModule, ], controllers: [HealthController], }) diff --git a/apps/api/src/crawler/crawler.module.ts b/apps/api/src/crawler/crawler.module.ts index 4d2ea58..943fde9 100644 --- a/apps/api/src/crawler/crawler.module.ts +++ b/apps/api/src/crawler/crawler.module.ts @@ -1,12 +1,13 @@ import { Module } from '@nestjs/common' import { ScheduleModule } from '@nestjs/schedule' import { PrismaModule } from '../prisma/prisma.module' +import { ProductCategoryModule } from '../product-category/product-category.module' import { CrawlerService } from './crawler.service' import { CrawlerController } from './crawler.controller' import { NotesExtractorService } from './notes-extractor.service' @Module({ - imports: [PrismaModule, ScheduleModule.forRoot()], + imports: [PrismaModule, ScheduleModule.forRoot(), ProductCategoryModule], controllers: [CrawlerController], providers: [CrawlerService, NotesExtractorService], exports: [NotesExtractorService], diff --git a/apps/api/src/crawler/crawler.service.ts b/apps/api/src/crawler/crawler.service.ts index 8b35bf0..69cccb5 100644 --- a/apps/api/src/crawler/crawler.service.ts +++ b/apps/api/src/crawler/crawler.service.ts @@ -5,7 +5,7 @@ import { execSync } from 'child_process' import * as path from 'path' import * as fs from 'fs' import { PrismaService } from '../prisma/prisma.service' -import { inferCategory } from '../lead/lead.service' +import { CategoryInferenceService } from '../product-category/category-inference.service' const SCRIPTS_DIR = path.resolve(__dirname, '../../../../../scripts') const TENDER_FETCHER_DIR = path.join(SCRIPTS_DIR, 'tender-fetcher') @@ -30,6 +30,7 @@ export class CrawlerService implements OnModuleInit { constructor( private prisma: PrismaService, private schedulerRegistry: SchedulerRegistry, + private categoryInference: CategoryInferenceService, ) {} async onModuleInit() { @@ -118,8 +119,13 @@ export class CrawlerService implements OnModuleInit { } let successCount = 0 - for (const r of results) { - if (!r.noticeId) continue + + const validResults = results.filter(r => r.noticeId) + const categoryMap = await this.categoryInference.batchInfer( + validResults.map(r => ({ id: r.noticeId, title: r.title })), + ) + + for (const r of validResults) { try { await this.prisma.lead.upsert({ where: { @@ -131,7 +137,7 @@ export class CrawlerService implements OnModuleInit { deadline: r.deadline, detailUrl: r.detailUrl, publishTime: r.publishTime || null, - category: inferCategory(r.title), + category: categoryMap.get(r.noticeId) || '', ...(r.amount ? { amount: r.amount } : {}), ...(r.region ? { region: r.region } : {}), ...(r.regionDetail ? { regionDetail: r.regionDetail } : {}), @@ -148,7 +154,7 @@ export class CrawlerService implements OnModuleInit { amount: r.amount || '待确认', status: '待复核', level: 'C', - category: inferCategory(r.title), + category: categoryMap.get(r.noticeId) || '', region: r.region || '', regionDetail: r.regionDetail || '', summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`, diff --git a/apps/api/src/lead/category-inference.service.ts b/apps/api/src/lead/category-inference.service.ts new file mode 100644 index 0000000..bd54ea2 --- /dev/null +++ b/apps/api/src/lead/category-inference.service.ts @@ -0,0 +1,166 @@ +import { Injectable, Logger } from '@nestjs/common' +import { PrismaService } from '../prisma/prisma.service' +import { ProductCategoryService } from '../product-category/product-category.service' + +const DEEPSEEK_ENDPOINT = 'https://api.deepseek.com/v1/chat/completions' +const DEEPSEEK_MODEL = 'deepseek-chat' +const DEEPSEEK_TIMEOUT_MS = 30000 + +@Injectable() +export class CategoryInferenceService { + private readonly logger = new Logger(CategoryInferenceService.name) + + constructor( + private prisma: PrismaService, + private categoryService: ProductCategoryService, + ) {} + + async infer(title: string, summary?: string | null): Promise { + if (!title) return '' + + const keywordResult = await this.categoryService.matchByKeywords(title) + if (keywordResult) return keywordResult.name + + const apiKey = process.env.DEEPSEEK_API_KEY + if (!apiKey) { + this.logger.warn('DEEPSEEK_API_KEY 未配置, 无法 AI 分类') + return '' + } + + try { + const aiCategory = await this.callDeepSeek(title, summary, apiKey) + if (aiCategory) { + await this.categoryService.addKeywordToCategory(aiCategory, this.extractMainKeyword(title)) + this.logger.log(`AI 分类: "${title.slice(0, 40)}..." → ${aiCategory}`) + return aiCategory + } + } catch (e) { + this.logger.warn(`AI 分类失败: ${e.message}`) + } + return '' + } + + async batchInfer(leads: Array<{ id: string; title: string; summary?: string | null }>): Promise> { + const result = new Map() + const needsInference = [] + + for (const lead of leads) { + if (!lead.title) { result.set(lead.id, ''); continue } + const keywordResult = await this.categoryService.matchByKeywords(lead.title) + if (keywordResult) { + result.set(lead.id, keywordResult.name) + } else { + needsInference.push(lead) + } + } + + if (needsInference.length === 0) return result + + const apiKey = process.env.DEEPSEEK_API_KEY + if (!apiKey) { + for (const lead of needsInference) result.set(lead.id, '') + return result + } + + const BATCH_PROMPT = `你是建材工业品分类助手。根据采购公告标题，从以下已有品类中选择最匹配的一个。如果都不匹配，返回一个新的简短品类名（2-4个中文字）。 + +已有品类：{{CATEGORIES}} + +返回格式：只返回品类名，不要其他文字。` + + const categories = await this.categoryService.findAll() + const catNames = categories.map(c => c.name).join('、') + const systemPrompt = BATCH_PROMPT.replace('{{CATEGORIES}}', catNames) + + const BATCH_SIZE = 20 + for (let i = 0; i < needsInference.length; i += BATCH_SIZE) { + const batch = needsInference.slice(i, i + BATCH_SIZE) + const items = batch.map((l, idx) => `${idx + 1}. ${l.title}`).join('\n') + + try { + const r = await fetch(DEEPSEEK_ENDPOINT, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: DEEPSEEK_MODEL, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: items }, + ], + temperature: 0.1, + max_tokens: 500, + }), + signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS), + }) + + if (!r.ok) { + const errText = await r.text().catch(() => '') + this.logger.warn(`DeepSeek batch API ${r.status}: ${errText.slice(0, 200)}`) + for (const lead of batch) result.set(lead.id, '') + continue + } + + const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> } + const content = data.choices?.[0]?.message?.content || '' + const lines = content.split('\n').map(l => l.trim()).filter(l => l.length > 0) + + for (let j = 0; j < batch.length; j++) { + const catName = lines[j]?.replace(/^\d+[\.\、]\s*/, '').trim() || '' + if (catName && catName.length <= 10) { + result.set(batch[j].id, catName) + await this.categoryService.addKeywordToCategory(catName, this.extractMainKeyword(batch[j].title)) + } else { + result.set(batch[j].id, '') + } + } + } catch (e) { + this.logger.warn(`AI batch 分类失败: ${e.message}`) + for (const lead of batch) result.set(lead.id, '') + } + } + + return result + } + + private async callDeepSeek(title: string, summary: string | null | undefined, apiKey: string): Promise { + const existingCats = (await this.categoryService.findAll()).map(c => c.name).join('、') + const userContent = `标题: ${title}\n${summary ? `摘要: ${summary.slice(0, 500)}` : ''}` + + const r = await fetch(DEEPSEEK_ENDPOINT, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: DEEPSEEK_MODEL, + messages: [ + { + role: 'system', + content: `你是建材工业品分类助手。根据采购公告标题，从以下已有品类中选择最匹配的一个。如果都不匹配，返回一个新的简短品类名（2-4个中文字）。\n已有品类：${existingCats}\n\n返回格式：只返回品类名，不要其他文字。`, + }, + { role: 'user', content: userContent }, + ], + temperature: 0.1, + max_tokens: 50, + }), + signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS), + }) + + if (!r.ok) { + const errText = await r.text().catch(() => '') + throw new Error(`DeepSeek API ${r.status}: ${errText.slice(0, 200)}`) + } + + const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> } + return data.choices?.[0]?.message?.content?.trim() || '' + } + + private extractMainKeyword(title: string): string { + const match = title.match(/([\u4e00-\u9fa5]{2,6})采购/) + return match ? match[1] : title.slice(0, 20) + } +} diff --git a/apps/api/src/lead/lead.module.ts b/apps/api/src/lead/lead.module.ts index ff48ab8..ba4101f 100644 --- a/apps/api/src/lead/lead.module.ts +++ b/apps/api/src/lead/lead.module.ts @@ -2,12 +2,13 @@ import { Module } from '@nestjs/common' import { PrismaModule } from '../prisma/prisma.module' import { IntegrationModule } from '../integration/integration.module' import { CrawlerModule } from '../crawler/crawler.module' +import { ProductCategoryModule } from '../product-category/product-category.module' import { LeadService } from './lead.service' import { LeadController } from './lead.controller' import { LeadNotesController } from './lead-notes.controller' @Module({ - imports: [PrismaModule, IntegrationModule, CrawlerModule], + imports: [PrismaModule, IntegrationModule, CrawlerModule, ProductCategoryModule], controllers: [LeadController, LeadNotesController], providers: [LeadService], exports: [LeadService], diff --git a/apps/api/src/lead/lead.service.ts b/apps/api/src/lead/lead.service.ts index 2b91f40..57d050f 100644 --- a/apps/api/src/lead/lead.service.ts +++ b/apps/api/src/lead/lead.service.ts @@ -1,36 +1,8 @@ import { Injectable, NotFoundException, Logger, ServiceUnavailableException } from '@nestjs/common' import { PrismaService } from '../prisma/prisma.service' import { IntegrationService } from '../integration/integration.service' - -// 品类 → 标题关键词映射（筛选 + 入库均使用） -export const CATEGORY_KEYWORDS: Record = { - '防火门': ['防火门', '防火卷帘', '防火窗', '钢质防火', '木质防火'], - '配电箱': ['配电箱', '配电柜', '配电盘', '照明配电箱', '动力配电箱', '电表箱'], - '电线电缆': ['电线', '电缆', '导线', '电力线', '控制电缆', '光缆', '架空线'], - '阀门': ['阀门', '闸阀', '球阀', '蝶阀', '截止阀', '止回阀', '调节阀'], - '钢材': ['钢材', '钢管', '钢板', '钢筋', '型钢', '槽钢', '角钢', '工字钢', 'H型钢', '螺纹钢'], - '涂料': ['涂料', '油漆', '防腐漆', '防火涂料', '乳胶漆', '防水涂料'], - '混凝土': ['混凝土', '商混', '商品砼', '水泥', '砂浆', '砼'], - '变压器': ['变压器', '箱变', '干式变', '油浸变', '电力变压器'], - '水泵': ['水泵', '离心泵', '潜污泵', '消防泵', '排污泵', '循环泵'], - '风机': ['风机', '通风机', '排风机', '轴流风机', '离心风机'], - '管材': ['管材', '管道', 'PVC管', 'PE管', '镀锌管', '无缝管', '螺旋管', '给水管'], - '开关': ['开关', '断路器', '接触器', '继电器', '熔断器', '隔离开关'], - '灯具': ['灯具', '照明灯', '路灯', '防爆灯', 'LED灯', '太阳能灯'], - '仪表': ['仪表', '仪器', '传感器', '变送器', '流量计', '压力表', '温度计'], - '电机': ['电机', '电动机', '减速机', '减速器', '变频电机'], -} - -/** 根据标题推断品类，未匹配返回空字符串 */ -export function inferCategory(title: string): string { - if (!title) return '' - for (const [cat, keywords] of Object.entries(CATEGORY_KEYWORDS)) { - for (const kw of keywords) { - if (title.includes(kw)) return cat - } - } - return '' -} +import { ProductCategoryService } from '../product-category/product-category.service' +import { CategoryInferenceService } from './category-inference.service' @Injectable() export class LeadService { @@ -39,6 +11,8 @@ export class LeadService { constructor( private prisma: PrismaService, private integrationService: IntegrationService, + private categoryService: ProductCategoryService, + private categoryInference: CategoryInferenceService, ) {} async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string; timeRange?: string }) { @@ -59,8 +33,9 @@ export class LeadService { } if (params?.source) where.source = params.source if (params?.category) { - const keywords = CATEGORY_KEYWORDS[params.category] - if (keywords) { + const categoryRecord = await this.prisma.productCategory.findUnique({ where: { name: params.category } }) + if (categoryRecord) { + const keywords: string[] = JSON.parse(categoryRecord.keywords) where.OR = keywords.map(kw => ({ title: { contains: kw } })) } } diff --git a/apps/api/src/product-category/product-category.controller.ts b/apps/api/src/product-category/product-category.controller.ts new file mode 100644 index 0000000..61c4b1d --- /dev/null +++ b/apps/api/src/product-category/product-category.controller.ts @@ -0,0 +1,27 @@ +import { Controller, Get, Post, Put, Delete, Body, Param } from '@nestjs/common' +import { ProductCategoryService } from './product-category.service' + +@Controller('api/categories') +export class ProductCategoryController { + constructor(private readonly categoryService: ProductCategoryService) {} + + @Get() + findAll() { + return this.categoryService.findAll() + } + + @Post() + create(@Body() body: { name: string; keywords: string[] }) { + return this.categoryService.create(body.name, body.keywords) + } + + @Put(':id') + update(@Param('id') id: string, @Body() body: { name: string; keywords: string[] }) { + return this.categoryService.update(id, body.name, body.keywords) + } + + @Delete(':id') + remove(@Param('id') id: string) { + return this.categoryService.remove(id) + } +} diff --git a/apps/api/src/product-category/product-category.module.ts b/apps/api/src/product-category/product-category.module.ts new file mode 100644 index 0000000..7a17d49 --- /dev/null +++ b/apps/api/src/product-category/product-category.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common' +import { PrismaModule } from '../prisma/prisma.module' +import { ProductCategoryService } from './product-category.service' +import { ProductCategoryController } from './product-category.controller' +import { CategoryInferenceService } from '../lead/category-inference.service' + +@Module({ + imports: [PrismaModule], + providers: [ProductCategoryService, CategoryInferenceService], + controllers: [ProductCategoryController], + exports: [ProductCategoryService, CategoryInferenceService], +}) +export class ProductCategoryModule {} diff --git a/apps/api/src/product-category/product-category.service.ts b/apps/api/src/product-category/product-category.service.ts new file mode 100644 index 0000000..9a26702 --- /dev/null +++ b/apps/api/src/product-category/product-category.service.ts @@ -0,0 +1,72 @@ +import { Injectable, Logger } from '@nestjs/common' +import { PrismaService } from '../prisma/prisma.service' + +export interface CategoryMatch { + name: string + matchedBy: 'keyword' | 'ai' | null +} + +@Injectable() +export class ProductCategoryService { + private readonly logger = new Logger(ProductCategoryService.name) + + constructor(private prisma: PrismaService) {} + + async findAll() { + return this.prisma.productCategory.findMany({ orderBy: { name: 'asc' } }) + } + + async create(name: string, keywords: string[]) { + return this.prisma.productCategory.create({ + data: { name, keywords: JSON.stringify(keywords) }, + }) + } + + async update(id: string, name: string, keywords: string[]) { + return this.prisma.productCategory.update({ + where: { id }, + data: { name, keywords: JSON.stringify(keywords) }, + }) + } + + async remove(id: string) { + return this.prisma.productCategory.delete({ where: { id } }) + } + + async matchByKeywords(title: string): Promise { + const categories = await this.prisma.productCategory.findMany() + for (const cat of categories) { + const keywords: string[] = JSON.parse(cat.keywords) + for (const kw of keywords) { + if (title.includes(kw)) { + return { name: cat.name, matchedBy: 'keyword' } + } + } + } + return null + } + + async addKeywordToCategory(categoryName: string, keyword: string) { + const cat = await this.prisma.productCategory.findUnique({ where: { name: categoryName } }) + if (!cat) { + await this.prisma.productCategory.create({ + data: { + name: categoryName, + keywords: JSON.stringify([keyword]), + isAIGenerated: true, + }, + }) + this.logger.log(`AI 新建品类: ${categoryName} (关键词: ${keyword})`) + return + } + const keywords: string[] = JSON.parse(cat.keywords) + if (!keywords.includes(keyword)) { + keywords.push(keyword) + await this.prisma.productCategory.update({ + where: { id: cat.id }, + data: { keywords: JSON.stringify(keywords) }, + }) + this.logger.log(`AI 补充品类 ${categoryName} 关键词: ${keyword}`) + } + } +}