diff --git a/apps/api/src/crawler/notes-extractor.service.ts b/apps/api/src/crawler/notes-extractor.service.ts index 5543985..acb562a 100644 --- a/apps/api/src/crawler/notes-extractor.service.ts +++ b/apps/api/src/crawler/notes-extractor.service.ts @@ -1,9 +1,9 @@ import { Injectable, Logger } from '@nestjs/common' import { PrismaService } from '../prisma/prisma.service' -const DEEPSEEK_ENDPOINT = 'https://api.deepseek.com/v1/chat/completions' -const DEEPSEEK_MODEL = 'deepseek-chat' -const DEEPSEEK_TIMEOUT_MS = 60000 +const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions' +const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7' +const MINIMAX_TIMEOUT_MS = 60000 const MAX_INPUT_CHARS = 4000 const SYSTEM_PROMPT = `你是一个招标公告信息提取助手。请从以下公告内容中提取关键商业要求，整理成简明要点。 @@ -40,8 +40,8 @@ export interface ExtractNotesResult { * * 流程: * 1. PDF → text (PyMuPDF) / HTML → text (cheerio) / URL → text (fetch) - * 2. text → DeepSeek API (deepseek-chat) - * 3. DeepSeek 返回 6 条 "- " 开头的要点 + * 2. text → MiniMax API (MiniMax-M2.7) + * 3. MiniMax 返回 6 条 "- " 开头的要点 * 4. 缓存到 Lead.notes (避免重复调 API) */ @Injectable() @@ -87,12 +87,12 @@ export class NotesExtractorService { this.logger.log(`[notes] leadId=${leadId} source=${source} textLen=${rawText.length}`) // 2. 调 DeepSeek 提取 - const apiKey = process.env.DEEPSEEK_API_KEY + const apiKey = process.env.MINIMAX_API_KEY if (!apiKey) { - throw new Error('DEEPSEEK_API_KEY 未配置 (.env 加 DEEPSEEK_API_KEY=sk-xxx)') + throw new Error('MINIMAX_API_KEY 未配置 (.env 加 MINIMAX_API_KEY=sk-xxx)') } - const notes = await this.callDeepSeek(rawText, apiKey) + const notes = await this.callMinimax(rawText, apiKey) this.logger.log(`[notes] 提取完成 ${notes.split('\n').length} 条`) // 3. 存到 Lead.notes @@ -159,19 +159,19 @@ export class NotesExtractorService { .join('\n') } - private async callDeepSeek(rawText: string, apiKey: string): Promise { + private async callMinimax(rawText: string, apiKey: string): Promise { const text = rawText.length > MAX_INPUT_CHARS ? rawText.slice(0, MAX_INPUT_CHARS) + '\n[内容过长已截断]' : rawText - const r = await fetch(DEEPSEEK_ENDPOINT, { + const r = await fetch(MINIMAX_ENDPOINT, { method: 'POST', headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ - model: DEEPSEEK_MODEL, + model: MINIMAX_MODEL, messages: [ { role: 'system', content: SYSTEM_PROMPT }, { role: 'user', content: text }, @@ -179,17 +179,17 @@ export class NotesExtractorService { temperature: 0.1, max_tokens: 500, }), - signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS), + signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS), }) if (!r.ok) { const errText = await r.text().catch(() => '') - throw new Error(`DeepSeek API ${r.status}: ${errText.slice(0, 200)}`) + throw new Error(`MiniMax API ${r.status}: ${errText.slice(0, 200)}`) } const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> } const content = data.choices?.[0]?.message?.content || '' - if (!content) throw new Error(`DeepSeek 返回空: ${JSON.stringify(data).slice(0, 200)}`) + if (!content) throw new Error(`MiniMax 返回空: ${JSON.stringify(data).slice(0, 200)}`) // 清理输出: 每条以 "- " 开头, 最多 6 条 const lines = content diff --git a/apps/api/src/product-category/category-inference.service.ts b/apps/api/src/product-category/category-inference.service.ts index bd54ea2..ba2e0b5 100644 --- a/apps/api/src/product-category/category-inference.service.ts +++ b/apps/api/src/product-category/category-inference.service.ts @@ -2,9 +2,9 @@ import { Injectable, Logger } from '@nestjs/common' import { PrismaService } from '../prisma/prisma.service' import { ProductCategoryService } from '../product-category/product-category.service' -const DEEPSEEK_ENDPOINT = 'https://api.deepseek.com/v1/chat/completions' -const DEEPSEEK_MODEL = 'deepseek-chat' -const DEEPSEEK_TIMEOUT_MS = 30000 +const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions' +const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7' +const MINIMAX_TIMEOUT_MS = 30000 @Injectable() export class CategoryInferenceService { @@ -21,14 +21,14 @@ export class CategoryInferenceService { const keywordResult = await this.categoryService.matchByKeywords(title) if (keywordResult) return keywordResult.name - const apiKey = process.env.DEEPSEEK_API_KEY + const apiKey = process.env.MINIMAX_API_KEY if (!apiKey) { - this.logger.warn('DEEPSEEK_API_KEY 未配置, 无法 AI 分类') + this.logger.warn('MINIMAX_API_KEY 未配置, 无法 AI 分类') return '' } try { - const aiCategory = await this.callDeepSeek(title, summary, apiKey) + const aiCategory = await this.callMinimax(title, summary, apiKey) if (aiCategory) { await this.categoryService.addKeywordToCategory(aiCategory, this.extractMainKeyword(title)) this.logger.log(`AI 分类: "${title.slice(0, 40)}..." → ${aiCategory}`) @@ -56,7 +56,7 @@ export class CategoryInferenceService { if (needsInference.length === 0) return result - const apiKey = process.env.DEEPSEEK_API_KEY + const apiKey = process.env.MINIMAX_API_KEY if (!apiKey) { for (const lead of needsInference) result.set(lead.id, '') return result @@ -78,14 +78,14 @@ export class CategoryInferenceService { const items = batch.map((l, idx) => `${idx + 1}. ${l.title}`).join('\n') try { - const r = await fetch(DEEPSEEK_ENDPOINT, { + const r = await fetch(MINIMAX_ENDPOINT, { method: 'POST', headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ - model: DEEPSEEK_MODEL, + model: MINIMAX_MODEL, messages: [ { role: 'system', content: systemPrompt }, { role: 'user', content: items }, @@ -93,12 +93,12 @@ export class CategoryInferenceService { temperature: 0.1, max_tokens: 500, }), - signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS), + signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS), }) if (!r.ok) { const errText = await r.text().catch(() => '') - this.logger.warn(`DeepSeek batch API ${r.status}: ${errText.slice(0, 200)}`) + this.logger.warn(`MiniMax batch API ${r.status}: ${errText.slice(0, 200)}`) for (const lead of batch) result.set(lead.id, '') continue } @@ -125,18 +125,18 @@ export class CategoryInferenceService { return result } - private async callDeepSeek(title: string, summary: string | null | undefined, apiKey: string): Promise { + private async callMinimax(title: string, summary: string | null | undefined, apiKey: string): Promise { const existingCats = (await this.categoryService.findAll()).map(c => c.name).join('、') const userContent = `标题: ${title}\n${summary ? `摘要: ${summary.slice(0, 500)}` : ''}` - const r = await fetch(DEEPSEEK_ENDPOINT, { + const r = await fetch(MINIMAX_ENDPOINT, { method: 'POST', headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ - model: DEEPSEEK_MODEL, + model: MINIMAX_MODEL, messages: [ { role: 'system', @@ -147,12 +147,12 @@ export class CategoryInferenceService { temperature: 0.1, max_tokens: 50, }), - signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS), + signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS), }) if (!r.ok) { const errText = await r.text().catch(() => '') - throw new Error(`DeepSeek API ${r.status}: ${errText.slice(0, 200)}`) + throw new Error(`MiniMax API ${r.status}: ${errText.slice(0, 200)}`) } const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> } diff --git a/scripts/backfill_category.js b/scripts/backfill_category.js new file mode 100644 index 0000000..45740ae --- /dev/null +++ b/scripts/backfill_category.js @@ -0,0 +1,113 @@ +const { PrismaClient } = require('@prisma/client') +const prisma = new PrismaClient() +const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions' +const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7' + +const KNOWN_CATEGORIES = [ + '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土', + '变压器', '水泵', '风机', '管材', '开关', '灯具', '仪表', '电机', +] + +const EXPLANATION_PREFIX = /^(让我|用户|我需要|好的|现在|首先|分析|根据|对于|这条|这个|考虑|我们|这里|注意|提示|重要|关键|总结|综上|该|其|它|他们|这些|那些|由于|因为|所以|因此|需要|可能|应该|可以|建议|希望|要求|必须)/ + +function cleanOne(raw) { + if (!raw) return '' + let s = String(raw).trim() + s = s.replace(/[\s\S]*?<\/think>/g, '').trim() + s = s.replace(/\*\*/g, '').trim() + s = s.replace(/^["「『]+|["」』]+$/g, '').trim() + const lines = s.split('\n').map((l) => l.trim()).filter((l) => l) + for (const line of lines) { + if (EXPLANATION_PREFIX.test(line)) continue + if (line.length > 20) continue + if (line.includes('，') || line.includes('。') || line.includes('：') || line.includes(':')) continue + const cleaned = line.replace(/^[\d]+[\.、:：]\s*/, '').trim() + if (!cleaned || cleaned.length > 10) continue + if (KNOWN_CATEGORIES.includes(cleaned)) return cleaned + const m = cleaned.match(/^([\u4e00-\u9fa5]{2,6})$/) + if (m && !EXPLANATION_PREFIX.test(m[1])) return m[1] + } + return '' +} + +async function inferSingle(title, categories) { + const apiKey = process.env.MINIMAX_API_KEY + if (!apiKey) return '' + const systemPrompt = `你是建材工业品分类助手。任务：给一条采购公告分配一个品类名。 + +严格规则： +1. 只输出 2-6 个中文字的品类名，不要任何其他内容 +2. 不要输出标签 +3. 不要输出"让我""根据"等解释文字 +4. 不要输出标点符号 + +参考品类：${categories.join('、')}` + try { + const r = await fetch(MINIMAX_ENDPOINT, { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: MINIMAX_MODEL, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: title }, + ], + temperature: 0, + max_tokens: 500, + }), + }) + if (!r.ok) return '' + const data = await r.json() + const content = data.choices?.[0]?.message?.content || '' + return cleanOne(content) + } catch { + return '' + } +} + +async function main() { + const categories = await prisma.productCategory.findMany() + const catNames = categories.map((c) => c.name) + + const emptyLeads = await prisma.lead.findMany({ + where: { OR: [{ category: null }, { category: '' }] }, + select: { id: true, title: true }, + }) + console.log(`空品类线索: ${emptyLeads.length} 条`) + + let okCount = 0 + let failCount = 0 + for (let i = 0; i < emptyLeads.length; i++) { + const lead = emptyLeads[i] + const catName = await inferSingle(lead.title, catNames) + if (catName) { + await prisma.lead.update({ where: { id: lead.id }, data: { category: catName } }) + okCount++ + if (!catNames.includes(catName)) { + await prisma.productCategory.upsert({ + where: { name: catName }, + update: {}, + create: { name: catName, keywords: JSON.stringify([lead.title.slice(0, 10)]) }, + }) + catNames.push(catName) + } + } else { + failCount++ + } + if ((i + 1) % 20 === 0 || i === emptyLeads.length - 1) { + console.log(`进度 ${i + 1}/${emptyLeads.length} 成功 ${okCount} 失败 ${failCount}`) + } + } + + const remaining = await prisma.lead.count({ + where: { OR: [{ category: null }, { category: '' }] }, + }) + console.log(`完成: 成功 ${okCount}, 失败 ${failCount}, 剩余空品类: ${remaining}`) +} + +main() + .catch((e) => { + console.error(e) + process.exit(1) + }) + .finally(() => prisma.$disconnect()) diff --git a/scripts/clean_category.js b/scripts/clean_category.js new file mode 100644 index 0000000..a7647b4 --- /dev/null +++ b/scripts/clean_category.js @@ -0,0 +1,79 @@ +const { PrismaClient } = require('@prisma/client') +const prisma = new PrismaClient() + +const KNOWN_CATEGORIES = new Set([ + '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土', + '变压器', '水泵', '风机', '管材', '开关', '灯具', '仪表', '电机', +]) + +const EXPLANATION_PREFIX = /^(让我|用户|我需要|好的|现在|首先|分析|根据|对于|这条|这个|考虑|我们|这里|注意|提示|重要|关键|总结|综上|该|其|它|他们|这些|那些|由于|因为|所以|因此|需要|可能|应该|可以|建议|希望|要求|必须)/ + +function cleanCategory(raw) { + if (!raw) return '' + let s = String(raw).trim() + s = s.replace(/[\s\S]*?<\/think>/g, '').trim() + s = s.replace(/\*\*/g, '').trim() + s = s.replace(/^["「『]+|["」』]+$/g, '').trim() + + const lines = s.split('\n').map((l) => l.trim()).filter((l) => l) + for (const line of lines) { + if (EXPLANATION_PREFIX.test(line)) continue + if (line.length > 20) continue + if (line.includes('，') || line.includes('。') || line.includes('：') || line.includes(':')) continue + if (line.includes(' - ') || line.includes(' – ') || line.includes('--')) { + const parts = line.split(/\s*[-–]\s*/).map((p) => p.trim()).filter((p) => p) + for (const p of parts) { + if (KNOWN_CATEGORIES.has(p)) return p + const m = p.match(/^([\u4e00-\u9fa5]{2,6})$/) + if (m && !EXPLANATION_PREFIX.test(p)) return m[1] + } + continue + } + const cleaned = line.replace(/^[\d]+[\.、:：]\s*/, '').trim() + if (!cleaned || cleaned.length > 10) continue + if (KNOWN_CATEGORIES.has(cleaned)) return cleaned + const m = cleaned.match(/^([\u4e00-\u9fa5]{2,6})$/) + if (m && !EXPLANATION_PREFIX.test(m[1])) return m[1] + } + return '' +} + +async function main() { + const leads = await prisma.lead.findMany({ + where: { NOT: { category: null } }, + select: { id: true, category: true }, + }) + console.log(`共 ${leads.length} 条有品类的线索`) + + let fixed = 0 + let cleared = 0 + for (const lead of leads) { + const cleaned = cleanCategory(lead.category) + if (cleaned !== lead.category) { + await prisma.lead.update({ where: { id: lead.id }, data: { category: cleaned } }) + if (cleaned === '') cleared++ + else fixed++ + } + } + console.log(`修正: ${fixed}, 清空: ${cleared}`) + + const remaining = await prisma.lead.count({ where: { OR: [{ category: null }, { category: '' }] } }) + console.log(`剩余空品类: ${remaining}`) + + const dist = await prisma.lead.groupBy({ + by: ['category'], + where: { NOT: { category: '' } }, + _count: true, + orderBy: { _count: { category: 'desc' } }, + take: 30, + }) + console.log('品类分布:') + dist.forEach((d) => console.log(` ${d.category}: ${d._count}`)) +} + +main() + .catch((e) => { + console.error(e) + process.exit(1) + }) + .finally(() => prisma.$disconnect())