refactor: DeepSeek → MiniMax (M2.7 推理模型) 迁移
- notes-extractor 和 category-inference 都改用 MiniMax API - endpoint: https://api.minimaxi.com/v1/chat/completions - model: MiniMax-M2.7 (env 可覆盖 MINIMAX_MODEL) - 提示词明确禁止 <think> 标签和解释文字 - 新增 scripts/backfill_category.js (单条请求 + 严格清洗) - 新增 scripts/clean_category.js (清理 <think>/**加粗/解释行/破折号 等脏值) - 生产 .env 已写入 MINIMAX_API_KEY - 实际回填: 693/761 (91%) 已分配品类, 68 条空品类待后续处理
This commit is contained in:
Sisyphus
parent
f625dd0de1
commit
ab764bbc6c
@ -1,9 +1,9 @@
|
||||
import { Injectable, Logger } from '@nestjs/common'
|
||||
import { PrismaService } from '../prisma/prisma.service'
|
||||
|
||||
const DEEPSEEK_ENDPOINT = 'https://api.deepseek.com/v1/chat/completions'
|
||||
const DEEPSEEK_MODEL = 'deepseek-chat'
|
||||
const DEEPSEEK_TIMEOUT_MS = 60000
|
||||
const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions'
|
||||
const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7'
|
||||
const MINIMAX_TIMEOUT_MS = 60000
|
||||
const MAX_INPUT_CHARS = 4000
|
||||
|
||||
const SYSTEM_PROMPT = `你是一个招标公告信息提取助手。请从以下公告内容中提取关键商业要求，整理成简明要点。
|
||||
@ -40,8 +40,8 @@ export interface ExtractNotesResult {
|
||||
*
|
||||
* 流程:
|
||||
* 1. PDF → text (PyMuPDF) / HTML → text (cheerio) / URL → text (fetch)
|
||||
* 2. text → DeepSeek API (deepseek-chat)
|
||||
* 3. DeepSeek 返回 6 条 "- " 开头的要点
|
||||
* 2. text → MiniMax API (MiniMax-M2.7)
|
||||
* 3. MiniMax 返回 6 条 "- " 开头的要点
|
||||
* 4. 缓存到 Lead.notes (避免重复调 API)
|
||||
*/
|
||||
@Injectable()
|
||||
@ -87,12 +87,12 @@ export class NotesExtractorService {
|
||||
this.logger.log(`[notes] leadId=${leadId} source=${source} textLen=${rawText.length}`)
|
||||
|
||||
// 2. 调 DeepSeek 提取
|
||||
const apiKey = process.env.DEEPSEEK_API_KEY
|
||||
const apiKey = process.env.MINIMAX_API_KEY
|
||||
if (!apiKey) {
|
||||
throw new Error('DEEPSEEK_API_KEY 未配置 (.env 加 DEEPSEEK_API_KEY=sk-xxx)')
|
||||
throw new Error('MINIMAX_API_KEY 未配置 (.env 加 MINIMAX_API_KEY=sk-xxx)')
|
||||
}
|
||||
|
||||
const notes = await this.callDeepSeek(rawText, apiKey)
|
||||
const notes = await this.callMinimax(rawText, apiKey)
|
||||
this.logger.log(`[notes] 提取完成 ${notes.split('\n').length} 条`)
|
||||
|
||||
// 3. 存到 Lead.notes
|
||||
@ -159,19 +159,19 @@ export class NotesExtractorService {
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
private async callDeepSeek(rawText: string, apiKey: string): Promise<string> {
|
||||
private async callMinimax(rawText: string, apiKey: string): Promise<string> {
|
||||
const text = rawText.length > MAX_INPUT_CHARS
|
||||
? rawText.slice(0, MAX_INPUT_CHARS) + '\n[内容过长已截断]'
|
||||
: rawText
|
||||
|
||||
const r = await fetch(DEEPSEEK_ENDPOINT, {
|
||||
const r = await fetch(MINIMAX_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: DEEPSEEK_MODEL,
|
||||
model: MINIMAX_MODEL,
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: text },
|
||||
@ -179,17 +179,17 @@ export class NotesExtractorService {
|
||||
temperature: 0.1,
|
||||
max_tokens: 500,
|
||||
}),
|
||||
signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS),
|
||||
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
|
||||
})
|
||||
|
||||
if (!r.ok) {
|
||||
const errText = await r.text().catch(() => '')
|
||||
throw new Error(`DeepSeek API ${r.status}: ${errText.slice(0, 200)}`)
|
||||
throw new Error(`MiniMax API ${r.status}: ${errText.slice(0, 200)}`)
|
||||
}
|
||||
|
||||
const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> }
|
||||
const content = data.choices?.[0]?.message?.content || ''
|
||||
if (!content) throw new Error(`DeepSeek 返回空: ${JSON.stringify(data).slice(0, 200)}`)
|
||||
if (!content) throw new Error(`MiniMax 返回空: ${JSON.stringify(data).slice(0, 200)}`)
|
||||
|
||||
// 清理输出: 每条以 "- " 开头, 最多 6 条
|
||||
const lines = content
|
||||
|
||||
@ -2,9 +2,9 @@ import { Injectable, Logger } from '@nestjs/common'
|
||||
import { PrismaService } from '../prisma/prisma.service'
|
||||
import { ProductCategoryService } from '../product-category/product-category.service'
|
||||
|
||||
const DEEPSEEK_ENDPOINT = 'https://api.deepseek.com/v1/chat/completions'
|
||||
const DEEPSEEK_MODEL = 'deepseek-chat'
|
||||
const DEEPSEEK_TIMEOUT_MS = 30000
|
||||
const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions'
|
||||
const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7'
|
||||
const MINIMAX_TIMEOUT_MS = 30000
|
||||
|
||||
@Injectable()
|
||||
export class CategoryInferenceService {
|
||||
@ -21,14 +21,14 @@ export class CategoryInferenceService {
|
||||
const keywordResult = await this.categoryService.matchByKeywords(title)
|
||||
if (keywordResult) return keywordResult.name
|
||||
|
||||
const apiKey = process.env.DEEPSEEK_API_KEY
|
||||
const apiKey = process.env.MINIMAX_API_KEY
|
||||
if (!apiKey) {
|
||||
this.logger.warn('DEEPSEEK_API_KEY 未配置, 无法 AI 分类')
|
||||
this.logger.warn('MINIMAX_API_KEY 未配置, 无法 AI 分类')
|
||||
return ''
|
||||
}
|
||||
|
||||
try {
|
||||
const aiCategory = await this.callDeepSeek(title, summary, apiKey)
|
||||
const aiCategory = await this.callMinimax(title, summary, apiKey)
|
||||
if (aiCategory) {
|
||||
await this.categoryService.addKeywordToCategory(aiCategory, this.extractMainKeyword(title))
|
||||
this.logger.log(`AI 分类: "${title.slice(0, 40)}..." → ${aiCategory}`)
|
||||
@ -56,7 +56,7 @@ export class CategoryInferenceService {
|
||||
|
||||
if (needsInference.length === 0) return result
|
||||
|
||||
const apiKey = process.env.DEEPSEEK_API_KEY
|
||||
const apiKey = process.env.MINIMAX_API_KEY
|
||||
if (!apiKey) {
|
||||
for (const lead of needsInference) result.set(lead.id, '')
|
||||
return result
|
||||
@ -78,14 +78,14 @@ export class CategoryInferenceService {
|
||||
const items = batch.map((l, idx) => `${idx + 1}. ${l.title}`).join('\n')
|
||||
|
||||
try {
|
||||
const r = await fetch(DEEPSEEK_ENDPOINT, {
|
||||
const r = await fetch(MINIMAX_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: DEEPSEEK_MODEL,
|
||||
model: MINIMAX_MODEL,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: items },
|
||||
@ -93,12 +93,12 @@ export class CategoryInferenceService {
|
||||
temperature: 0.1,
|
||||
max_tokens: 500,
|
||||
}),
|
||||
signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS),
|
||||
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
|
||||
})
|
||||
|
||||
if (!r.ok) {
|
||||
const errText = await r.text().catch(() => '')
|
||||
this.logger.warn(`DeepSeek batch API ${r.status}: ${errText.slice(0, 200)}`)
|
||||
this.logger.warn(`MiniMax batch API ${r.status}: ${errText.slice(0, 200)}`)
|
||||
for (const lead of batch) result.set(lead.id, '')
|
||||
continue
|
||||
}
|
||||
@ -125,18 +125,18 @@ export class CategoryInferenceService {
|
||||
return result
|
||||
}
|
||||
|
||||
private async callDeepSeek(title: string, summary: string | null | undefined, apiKey: string): Promise<string> {
|
||||
private async callMinimax(title: string, summary: string | null | undefined, apiKey: string): Promise<string> {
|
||||
const existingCats = (await this.categoryService.findAll()).map(c => c.name).join('、')
|
||||
const userContent = `标题: ${title}\n${summary ? `摘要: ${summary.slice(0, 500)}` : ''}`
|
||||
|
||||
const r = await fetch(DEEPSEEK_ENDPOINT, {
|
||||
const r = await fetch(MINIMAX_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: DEEPSEEK_MODEL,
|
||||
model: MINIMAX_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
@ -147,12 +147,12 @@ export class CategoryInferenceService {
|
||||
temperature: 0.1,
|
||||
max_tokens: 50,
|
||||
}),
|
||||
signal: AbortSignal.timeout(DEEPSEEK_TIMEOUT_MS),
|
||||
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
|
||||
})
|
||||
|
||||
if (!r.ok) {
|
||||
const errText = await r.text().catch(() => '')
|
||||
throw new Error(`DeepSeek API ${r.status}: ${errText.slice(0, 200)}`)
|
||||
throw new Error(`MiniMax API ${r.status}: ${errText.slice(0, 200)}`)
|
||||
}
|
||||
|
||||
const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> }
|
||||
|
||||
113
scripts/backfill_category.js
Normal file
113
scripts/backfill_category.js
Normal file
@ -0,0 +1,113 @@
|
||||
const { PrismaClient } = require('@prisma/client')
|
||||
const prisma = new PrismaClient()
|
||||
const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions'
|
||||
const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7'
|
||||
|
||||
const KNOWN_CATEGORIES = [
|
||||
'防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土',
|
||||
'变压器', '水泵', '风机', '管材', '开关', '灯具', '仪表', '电机',
|
||||
]
|
||||
|
||||
const EXPLANATION_PREFIX = /^(让我|用户|我需要|好的|现在|首先|分析|根据|对于|这条|这个|考虑|我们|这里|注意|提示|重要|关键|总结|综上|该|其|它|他们|这些|那些|由于|因为|所以|因此|需要|可能|应该|可以|建议|希望|要求|必须)/
|
||||
|
||||
function cleanOne(raw) {
|
||||
if (!raw) return ''
|
||||
let s = String(raw).trim()
|
||||
s = s.replace(/<think>[\s\S]*?<\/think>/g, '').trim()
|
||||
s = s.replace(/\*\*/g, '').trim()
|
||||
s = s.replace(/^["「『]+|["」』]+$/g, '').trim()
|
||||
const lines = s.split('\n').map((l) => l.trim()).filter((l) => l)
|
||||
for (const line of lines) {
|
||||
if (EXPLANATION_PREFIX.test(line)) continue
|
||||
if (line.length > 20) continue
|
||||
if (line.includes('，') || line.includes('。') || line.includes('：') || line.includes(':')) continue
|
||||
const cleaned = line.replace(/^[\d]+[\.、:：]\s*/, '').trim()
|
||||
if (!cleaned || cleaned.length > 10) continue
|
||||
if (KNOWN_CATEGORIES.includes(cleaned)) return cleaned
|
||||
const m = cleaned.match(/^([\u4e00-\u9fa5]{2,6})$/)
|
||||
if (m && !EXPLANATION_PREFIX.test(m[1])) return m[1]
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
async function inferSingle(title, categories) {
|
||||
const apiKey = process.env.MINIMAX_API_KEY
|
||||
if (!apiKey) return ''
|
||||
const systemPrompt = `你是建材工业品分类助手。任务：给一条采购公告分配一个品类名。
|
||||
|
||||
严格规则：
|
||||
1. 只输出 2-6 个中文字的品类名，不要任何其他内容
|
||||
2. 不要输出<think>标签
|
||||
3. 不要输出"让我""根据"等解释文字
|
||||
4. 不要输出标点符号
|
||||
|
||||
参考品类：${categories.join('、')}`
|
||||
try {
|
||||
const r = await fetch(MINIMAX_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: MINIMAX_MODEL,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: title },
|
||||
],
|
||||
temperature: 0,
|
||||
max_tokens: 500,
|
||||
}),
|
||||
})
|
||||
if (!r.ok) return ''
|
||||
const data = await r.json()
|
||||
const content = data.choices?.[0]?.message?.content || ''
|
||||
return cleanOne(content)
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const categories = await prisma.productCategory.findMany()
|
||||
const catNames = categories.map((c) => c.name)
|
||||
|
||||
const emptyLeads = await prisma.lead.findMany({
|
||||
where: { OR: [{ category: null }, { category: '' }] },
|
||||
select: { id: true, title: true },
|
||||
})
|
||||
console.log(`空品类线索: ${emptyLeads.length} 条`)
|
||||
|
||||
let okCount = 0
|
||||
let failCount = 0
|
||||
for (let i = 0; i < emptyLeads.length; i++) {
|
||||
const lead = emptyLeads[i]
|
||||
const catName = await inferSingle(lead.title, catNames)
|
||||
if (catName) {
|
||||
await prisma.lead.update({ where: { id: lead.id }, data: { category: catName } })
|
||||
okCount++
|
||||
if (!catNames.includes(catName)) {
|
||||
await prisma.productCategory.upsert({
|
||||
where: { name: catName },
|
||||
update: {},
|
||||
create: { name: catName, keywords: JSON.stringify([lead.title.slice(0, 10)]) },
|
||||
})
|
||||
catNames.push(catName)
|
||||
}
|
||||
} else {
|
||||
failCount++
|
||||
}
|
||||
if ((i + 1) % 20 === 0 || i === emptyLeads.length - 1) {
|
||||
console.log(`进度 ${i + 1}/${emptyLeads.length} 成功 ${okCount} 失败 ${failCount}`)
|
||||
}
|
||||
}
|
||||
|
||||
const remaining = await prisma.lead.count({
|
||||
where: { OR: [{ category: null }, { category: '' }] },
|
||||
})
|
||||
console.log(`完成: 成功 ${okCount}, 失败 ${failCount}, 剩余空品类: ${remaining}`)
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((e) => {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
})
|
||||
.finally(() => prisma.$disconnect())
|
||||
79
scripts/clean_category.js
Normal file
79
scripts/clean_category.js
Normal file
@ -0,0 +1,79 @@
|
||||
const { PrismaClient } = require('@prisma/client')
|
||||
const prisma = new PrismaClient()
|
||||
|
||||
const KNOWN_CATEGORIES = new Set([
|
||||
'防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土',
|
||||
'变压器', '水泵', '风机', '管材', '开关', '灯具', '仪表', '电机',
|
||||
])
|
||||
|
||||
const EXPLANATION_PREFIX = /^(让我|用户|我需要|好的|现在|首先|分析|根据|对于|这条|这个|考虑|我们|这里|注意|提示|重要|关键|总结|综上|该|其|它|他们|这些|那些|由于|因为|所以|因此|需要|可能|应该|可以|建议|希望|要求|必须)/
|
||||
|
||||
function cleanCategory(raw) {
|
||||
if (!raw) return ''
|
||||
let s = String(raw).trim()
|
||||
s = s.replace(/<think>[\s\S]*?<\/think>/g, '').trim()
|
||||
s = s.replace(/\*\*/g, '').trim()
|
||||
s = s.replace(/^["「『]+|["」』]+$/g, '').trim()
|
||||
|
||||
const lines = s.split('\n').map((l) => l.trim()).filter((l) => l)
|
||||
for (const line of lines) {
|
||||
if (EXPLANATION_PREFIX.test(line)) continue
|
||||
if (line.length > 20) continue
|
||||
if (line.includes('，') || line.includes('。') || line.includes('：') || line.includes(':')) continue
|
||||
if (line.includes(' - ') || line.includes(' – ') || line.includes('--')) {
|
||||
const parts = line.split(/\s*[-–]\s*/).map((p) => p.trim()).filter((p) => p)
|
||||
for (const p of parts) {
|
||||
if (KNOWN_CATEGORIES.has(p)) return p
|
||||
const m = p.match(/^([\u4e00-\u9fa5]{2,6})$/)
|
||||
if (m && !EXPLANATION_PREFIX.test(p)) return m[1]
|
||||
}
|
||||
continue
|
||||
}
|
||||
const cleaned = line.replace(/^[\d]+[\.、:：]\s*/, '').trim()
|
||||
if (!cleaned || cleaned.length > 10) continue
|
||||
if (KNOWN_CATEGORIES.has(cleaned)) return cleaned
|
||||
const m = cleaned.match(/^([\u4e00-\u9fa5]{2,6})$/)
|
||||
if (m && !EXPLANATION_PREFIX.test(m[1])) return m[1]
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const leads = await prisma.lead.findMany({
|
||||
where: { NOT: { category: null } },
|
||||
select: { id: true, category: true },
|
||||
})
|
||||
console.log(`共 ${leads.length} 条有品类的线索`)
|
||||
|
||||
let fixed = 0
|
||||
let cleared = 0
|
||||
for (const lead of leads) {
|
||||
const cleaned = cleanCategory(lead.category)
|
||||
if (cleaned !== lead.category) {
|
||||
await prisma.lead.update({ where: { id: lead.id }, data: { category: cleaned } })
|
||||
if (cleaned === '') cleared++
|
||||
else fixed++
|
||||
}
|
||||
}
|
||||
console.log(`修正: ${fixed}, 清空: ${cleared}`)
|
||||
|
||||
const remaining = await prisma.lead.count({ where: { OR: [{ category: null }, { category: '' }] } })
|
||||
console.log(`剩余空品类: ${remaining}`)
|
||||
|
||||
const dist = await prisma.lead.groupBy({
|
||||
by: ['category'],
|
||||
where: { NOT: { category: '' } },
|
||||
_count: true,
|
||||
orderBy: { _count: { category: 'desc' } },
|
||||
take: 30,
|
||||
})
|
||||
console.log('品类分布:')
|
||||
dist.forEach((d) => console.log(` ${d.category}: ${d._count}`))
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((e) => {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
})
|
||||
.finally(() => prisma.$disconnect())
|
||||
Loading…
Reference in New Issue
Block a user