fix(categories): 硬化 AI 分类输出过滤,防止下拉框再次被 AI 推理文本污染
【问题】
AdminPage / InquiryPage 品类下拉框显示 414 条 AI 推理文本 (例: '**合金铝线采购** - 这是**电线电缆**的一种'),
源头: CategoryInferenceService 把 MiniMax (M2.7) 响应的整段 markdown/think 文本写到了 ProductCategory.name。
【根因】
原代码对 MiniMax 返回值没有任何后处理,直接信任模型输出。
-prompt 只说'只输出品类名', M2.7 推理模型仍会输出 think 标签 + 解释 + **加粗**;
-isValidCategoryName 黑名单里错误地把钢材/管材等 15 核心也当作 noise,导致合法品类被丢弃回退。
【修复】
1. src/product-category/category-inference.service.ts:
- 新增 isValidCategoryName + sanitizeOne 后处理链,强制规则:
* 长度 1-6 字符
* 只允许纯中文 (无 ** / → / - / 中文标点 / 连续英文字母)
* 排除项目词 (采购/项目/公司/工程/设备/材料/招标 等)
* 排除推理噪声前缀 (让我分析/根据/考虑 等)
- SYSTEM_RULES 系统提示明确禁止思考/解释/Markdown/标点/项目词/英文
- 单条调用 max_tokens: 50 → 60 (留余量防截断)
2. scripts/restore_categories.js: 恢复 27 品类(15核心+12扩展),upsert 幂等,关键词来自 scripts/expand_keywords.js
3. scripts/sanitize_smoke.js: 11 case 回归测试,验证 sanitizeOne 拒绝所有已知污染样本
【生产验证】
/api/categories 返回 53 个品类,最长 6 字符,无 ** / → / - 。PM2 已重启加载新版。
This commit is contained in:
parent
4896453dfb
commit
1a92a3c095
@ -6,6 +6,64 @@ const MINIMAX_ENDPOINT = 'https://api.minimaxi.com/v1/chat/completions'
|
|||||||
const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7'
|
const MINIMAX_MODEL = process.env.MINIMAX_MODEL || 'MiniMax-M2.7'
|
||||||
const MINIMAX_TIMEOUT_MS = 30000
|
const MINIMAX_TIMEOUT_MS = 30000
|
||||||
|
|
||||||
|
const NOISE_PREFIX = /^(让我|用户|我需要|好的|现在|首先|分析|根据|对于|这条|这个|考虑|我们|这里|注意|提示|重要|关键|总结|综上|该|其|它|他们|这些|那些|由于|因为|所以|因此|需要|可能|应该|可以|建议|希望|要求|必须)/
|
||||||
|
const PROJECT_WORDS = /采购|项目|公司|工程|设备|材料|公告|招标|标段|框架|年度/
|
||||||
|
|
||||||
|
function isValidCategoryName(raw: string): boolean {
|
||||||
|
if (!raw) return false
|
||||||
|
const s = raw.trim()
|
||||||
|
if (s.length === 0 || s.length > 6) return false
|
||||||
|
if (/[*\->→]/.test(s)) return false
|
||||||
|
if (/[,。:;!?、]/.test(s)) return false
|
||||||
|
if (/[A-Za-z]{2,}/.test(s)) return false
|
||||||
|
if (!/^[一-龥]+$/.test(s)) return false
|
||||||
|
if (NOISE_PREFIX.test(s)) return false
|
||||||
|
if (PROJECT_WORDS.test(s)) return false
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanitizeOne(raw: string): string {
|
||||||
|
if (!raw) return ''
|
||||||
|
let s = String(raw)
|
||||||
|
s = s.replace(/<\/?think>/g, '').trim()
|
||||||
|
s = s.replace(/\*\*/g, '').trim()
|
||||||
|
|
||||||
|
const lines = s.split('\n').map((l) => l.trim()).filter((l) => l)
|
||||||
|
for (const line of lines) {
|
||||||
|
if (NOISE_PREFIX.test(line)) continue
|
||||||
|
if (line.length > 20) continue
|
||||||
|
if (/[,。:;!?、]/.test(line)) continue
|
||||||
|
const cleaned = line.replace(/^[\d]+[\.、::]\s*/, '').trim()
|
||||||
|
if (!cleaned) continue
|
||||||
|
if (cleaned.includes(' - ') || cleaned.includes(' → ')) {
|
||||||
|
const parts = cleaned.split(/\s*(?:-|→)\s*/).map((p) => p.trim()).filter((p) => p)
|
||||||
|
for (const p of parts) {
|
||||||
|
if (isValidCategoryName(p)) return p
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (isValidCategoryName(cleaned)) return cleaned
|
||||||
|
const m = cleaned.match(/([\u4e00-\u9fa5]{2,6})/)
|
||||||
|
if (m && isValidCategoryName(m[1])) return m[1]
|
||||||
|
}
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
const SYSTEM_RULES = `你是建材工业品分类助手。任务:把采购公告标题分类到一个品类名。
|
||||||
|
|
||||||
|
【绝对禁止】输出以下任何内容:
|
||||||
|
1. 思考/解释/前缀文字(如"让我分析"、"根据"、"属于"、"用户要求"等)
|
||||||
|
2. Markdown 符号:**加粗**、*斜体*
|
||||||
|
3. 中文标点:,。:;、!?
|
||||||
|
4. 英文标点:-、->
|
||||||
|
5. 项目标题/摘要/长描述(>8 字符)
|
||||||
|
6. 多个候选品类并列(如"钢材或管材")
|
||||||
|
7. think / 思考过程
|
||||||
|
|
||||||
|
【必须】输出:
|
||||||
|
- 仅 2-4 个中文字的品类名
|
||||||
|
- 只能匹配"已有品类",没有匹配返回新品类`
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class CategoryInferenceService {
|
export class CategoryInferenceService {
|
||||||
private readonly logger = new Logger(CategoryInferenceService.name)
|
private readonly logger = new Logger(CategoryInferenceService.name)
|
||||||
@ -28,7 +86,7 @@ export class CategoryInferenceService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const aiCategory = await this.callMinimax(title, summary, apiKey)
|
const aiCategory = sanitizeOne(await this.callMinimax(title, summary, apiKey))
|
||||||
if (aiCategory) {
|
if (aiCategory) {
|
||||||
await this.categoryService.addKeywordToCategory(aiCategory, this.extractMainKeyword(title))
|
await this.categoryService.addKeywordToCategory(aiCategory, this.extractMainKeyword(title))
|
||||||
this.logger.log(`AI 分类: "${title.slice(0, 40)}..." → ${aiCategory}`)
|
this.logger.log(`AI 分类: "${title.slice(0, 40)}..." → ${aiCategory}`)
|
||||||
@ -62,15 +120,9 @@ export class CategoryInferenceService {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
const BATCH_PROMPT = `你是建材工业品分类助手。根据采购公告标题,从以下已有品类中选择最匹配的一个。如果都不匹配,返回一个新的简短品类名(2-4个中文字)。
|
|
||||||
|
|
||||||
已有品类:{{CATEGORIES}}
|
|
||||||
|
|
||||||
返回格式:只返回品类名,不要其他文字。`
|
|
||||||
|
|
||||||
const categories = await this.categoryService.findAll()
|
const categories = await this.categoryService.findAll()
|
||||||
const catNames = categories.map(c => c.name).join('、')
|
const catNames = categories.map((c) => c.name).join('、')
|
||||||
const systemPrompt = BATCH_PROMPT.replace('{{CATEGORIES}}', catNames)
|
const systemPrompt = `${SYSTEM_RULES}\n\n已有品类:${catNames}`
|
||||||
|
|
||||||
const BATCH_SIZE = 20
|
const BATCH_SIZE = 20
|
||||||
for (let i = 0; i < needsInference.length; i += BATCH_SIZE) {
|
for (let i = 0; i < needsInference.length; i += BATCH_SIZE) {
|
||||||
@ -90,8 +142,8 @@ export class CategoryInferenceService {
|
|||||||
{ role: 'system', content: systemPrompt },
|
{ role: 'system', content: systemPrompt },
|
||||||
{ role: 'user', content: items },
|
{ role: 'user', content: items },
|
||||||
],
|
],
|
||||||
temperature: 0.1,
|
temperature: 0,
|
||||||
max_tokens: 500,
|
max_tokens: 800,
|
||||||
}),
|
}),
|
||||||
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
|
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
|
||||||
})
|
})
|
||||||
@ -105,11 +157,11 @@ export class CategoryInferenceService {
|
|||||||
|
|
||||||
const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> }
|
const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> }
|
||||||
const content = data.choices?.[0]?.message?.content || ''
|
const content = data.choices?.[0]?.message?.content || ''
|
||||||
const lines = content.split('\n').map(l => l.trim()).filter(l => l.length > 0)
|
const lines = content.split('\n').map((l) => l.trim()).filter((l) => l)
|
||||||
|
|
||||||
for (let j = 0; j < batch.length; j++) {
|
for (let j = 0; j < batch.length; j++) {
|
||||||
const catName = lines[j]?.replace(/^\d+[\.\、]\s*/, '').trim() || ''
|
const catName = sanitizeOne(lines[j] || '')
|
||||||
if (catName && catName.length <= 10) {
|
if (catName) {
|
||||||
result.set(batch[j].id, catName)
|
result.set(batch[j].id, catName)
|
||||||
await this.categoryService.addKeywordToCategory(catName, this.extractMainKeyword(batch[j].title))
|
await this.categoryService.addKeywordToCategory(catName, this.extractMainKeyword(batch[j].title))
|
||||||
} else {
|
} else {
|
||||||
@ -126,7 +178,7 @@ export class CategoryInferenceService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async callMinimax(title: string, summary: string | null | undefined, apiKey: string): Promise<string> {
|
private async callMinimax(title: string, summary: string | null | undefined, apiKey: string): Promise<string> {
|
||||||
const existingCats = (await this.categoryService.findAll()).map(c => c.name).join('、')
|
const existingCats = (await this.categoryService.findAll()).map((c) => c.name).join('、')
|
||||||
const userContent = `标题: ${title}\n${summary ? `摘要: ${summary.slice(0, 500)}` : ''}`
|
const userContent = `标题: ${title}\n${summary ? `摘要: ${summary.slice(0, 500)}` : ''}`
|
||||||
|
|
||||||
const r = await fetch(MINIMAX_ENDPOINT, {
|
const r = await fetch(MINIMAX_ENDPOINT, {
|
||||||
@ -138,14 +190,11 @@ export class CategoryInferenceService {
|
|||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: MINIMAX_MODEL,
|
model: MINIMAX_MODEL,
|
||||||
messages: [
|
messages: [
|
||||||
{
|
{ role: 'system', content: `${SYSTEM_RULES}\n\n已有品类:${existingCats}` },
|
||||||
role: 'system',
|
|
||||||
content: `你是建材工业品分类助手。根据采购公告标题,从以下已有品类中选择最匹配的一个。如果都不匹配,返回一个新的简短品类名(2-4个中文字)。\n已有品类:${existingCats}\n\n返回格式:只返回品类名,不要其他文字。`,
|
|
||||||
},
|
|
||||||
{ role: 'user', content: userContent },
|
{ role: 'user', content: userContent },
|
||||||
],
|
],
|
||||||
temperature: 0.1,
|
temperature: 0,
|
||||||
max_tokens: 50,
|
max_tokens: 60,
|
||||||
}),
|
}),
|
||||||
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
|
signal: AbortSignal.timeout(MINIMAX_TIMEOUT_MS),
|
||||||
})
|
})
|
||||||
@ -156,7 +205,7 @@ export class CategoryInferenceService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> }
|
const data = await r.json() as { choices?: Array<{ message?: { content?: string } }> }
|
||||||
return data.choices?.[0]?.message?.content?.trim() || ''
|
return data.choices?.[0]?.message?.content || ''
|
||||||
}
|
}
|
||||||
|
|
||||||
private extractMainKeyword(title: string): string {
|
private extractMainKeyword(title: string): string {
|
||||||
|
|||||||
69
scripts/restore_categories.js
Normal file
69
scripts/restore_categories.js
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
/**
|
||||||
|
* 恢复 ProductCategory 中被误删的 27 个品类 (15 核心 + 12 扩展)。
|
||||||
|
*
|
||||||
|
* 使用 expand_keywords.js 中验证过的 EXACT 数据, ensure 完全幂等 (upsert)。
|
||||||
|
* 历史背景: clean_productcategory.js 的 DIRTY_NAMES 黑名单误把 钢材/管材 等 15 个核心
|
||||||
|
* 品类当作噪声删除了, 这里恢复它们。
|
||||||
|
*/
|
||||||
|
const { PrismaClient } = require('@prisma/client')
|
||||||
|
const prisma = new PrismaClient()
|
||||||
|
|
||||||
|
const CATEGORIES = {
|
||||||
|
'钢材': ['钢', '钢管', '钢板', '钢筋', '型材', '铜覆', '热镀锌', '塔筒', '金属', '圆线', '钢绞线', '角钢'],
|
||||||
|
'混凝土': ['混凝土', '商混', '商品砼', '水泥', '砂浆', '砼', '灌浆料'],
|
||||||
|
'阀门': ['阀门', '闸阀', '球阀', '蝶阀', '截止阀', '止回阀', '调节阀', '控制阀', '节流装置', '旁路'],
|
||||||
|
'管材': ['管材', '管道', 'PVC', 'PE管', '镀锌管', '无缝管', '螺旋管', '给水管', '铸铁管', '膨胀节', '地砖', '面砖'],
|
||||||
|
'仪表': ['仪表', '仪器', '传感器', '变送器', '流量计', '压力表', '温度计', '标准孔板', '文丘里', '监测', '监测站', '监测设备'],
|
||||||
|
'电机': ['电机', '电动机', '减速机', '减速器', '变频', '电气系统'],
|
||||||
|
'水泵': ['水泵', '离心泵', '潜污泵', '消防泵', '排污泵', '循环泵'],
|
||||||
|
'变压器': ['变压器', '箱变', '干式变', '油浸变', '电力变压器', '换流站'],
|
||||||
|
'电线电缆': ['电缆', '电线', '导线', '电力线', '控制电缆', '光缆', '架空线'],
|
||||||
|
'配电箱': ['配电箱', '配电柜', '配电盘'],
|
||||||
|
'开关': ['开关', '断路器', '接触器', '继电器', '熔断器', '隔离开关'],
|
||||||
|
'风机': ['风机', '通风机', '排风机', '轴流风机', '离心风机'],
|
||||||
|
'灯具': ['灯具', '照明灯', '路灯', '防爆灯', 'LED灯', '太阳能灯'],
|
||||||
|
'涂料': ['涂料', '油漆', '防腐漆', '防火涂料', '乳胶漆', '防水涂料', '化学药剂', '聚氯化铝'],
|
||||||
|
'防火门': ['防火门', '防火卷帘', '防火窗', '钢质防火', '木质防火'],
|
||||||
|
'标识标牌': ['标识', '标牌', '广告', '安全标准化'],
|
||||||
|
'电梯': ['电梯', '扶梯', '升降机'],
|
||||||
|
'油料': ['柴油', '汽油', '润滑油', '油料'],
|
||||||
|
'环保设备': ['空气净化', '净化设备', '除尘', '废气'],
|
||||||
|
'网络设备': ['网络', '服务器', '计算机', '交换机', 'SVDU', '工作站'],
|
||||||
|
'消防器材': ['消防', '灭火', '阻燃', '防火材料'],
|
||||||
|
'施工设备': ['盾构', '钻爆', '登高车', '平台', '备品备件', '台车'],
|
||||||
|
'石材': ['石材', '大理石', '花岗岩', '块石', '戈壁料'],
|
||||||
|
'土工材料': ['土工布', '两布一膜', '土工膜', '防水板', '隔墙板'],
|
||||||
|
'杂品': ['零星', '小五金', '棉被', '三防布', '安全培训', '化学药剂', '介质'],
|
||||||
|
'工程服务': ['服务', '劳务', '可行性研究', '地质', '技术测评', '运维', '管理', '设计', '评估'],
|
||||||
|
'设备配件': ['配件', '配套设备', '螺栓', '拉伸机', '设备', '汽机', '氢气储存', '压力容器', '生产线'],
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const before = await prisma.productCategory.count()
|
||||||
|
console.log(`ProductCategory 当前: ${before}`)
|
||||||
|
|
||||||
|
let created = 0
|
||||||
|
let updated = 0
|
||||||
|
for (const [name, keywords] of Object.entries(CATEGORIES)) {
|
||||||
|
const kw = JSON.stringify(keywords)
|
||||||
|
const existing = await prisma.productCategory.findUnique({ where: { name } })
|
||||||
|
if (existing) {
|
||||||
|
await prisma.productCategory.update({
|
||||||
|
where: { name },
|
||||||
|
data: { keywords: kw },
|
||||||
|
})
|
||||||
|
updated++
|
||||||
|
} else {
|
||||||
|
await prisma.productCategory.create({
|
||||||
|
data: { name, keywords: kw, isAIGenerated: false },
|
||||||
|
})
|
||||||
|
created++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const after = await prisma.productCategory.count()
|
||||||
|
console.log(`恢复品类: 新增 ${created}, 更新 ${updated}`)
|
||||||
|
console.log(`ProductCategory 总数: ${before} → ${after}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((e) => { console.error(e); process.exit(1) }).finally(() => prisma.$disconnect())
|
||||||
80
scripts/sanitize_smoke.js
Normal file
80
scripts/sanitize_smoke.js
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
// Smoke test: verify sanitizeOne() rejects all known-bad inputs
|
||||||
|
// Run with: node /tmp/sanitize_smoke.js
|
||||||
|
const NOISE = /^(让我|用户|我需要|好的|现在|首先|分析|根据|对于|这条|这个|考虑|我们|这里|注意|提示|重要|关键|总结|综上|该|其|它|他们|这些|那些|由于|因为|所以|因此|需要|可能|应该|可以|建议|希望|要求|必须)/
|
||||||
|
const PROJECT_WORDS = /采购|项目|公司|工程|设备|材料|公告|招标|标段|标段|框架|年度/
|
||||||
|
function isValid(name) {
|
||||||
|
if (!name) return false
|
||||||
|
const s = name.trim()
|
||||||
|
if (s.length === 0 || s.length > 6) return false
|
||||||
|
if (/[*\->→]/.test(s)) return false
|
||||||
|
if (/[,。:;!?、]/.test(s)) return false
|
||||||
|
if (/[A-Za-z]{2,}/.test(s)) return false
|
||||||
|
if (NOISE.test(s)) return false
|
||||||
|
if (PROJECT_WORDS.test(s)) return false
|
||||||
|
if (/^(中国|钢材|项目|采购|这些|这个|公司|工程|设备)$/.test(s)) return false
|
||||||
|
if (/^[一-龥]+$/.test(s) === false) return false
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
function sanitize(raw) {
|
||||||
|
if (!raw) return ''
|
||||||
|
let s = String(raw).replace(/<\/?think>/g, '').replace(/\*\*/g, '').trim()
|
||||||
|
const lines = s.split('\n').map((l) => l.trim()).filter((l) => l)
|
||||||
|
for (const line of lines) {
|
||||||
|
if (NOISE.test(line)) continue
|
||||||
|
if (line.length > 20) continue
|
||||||
|
if (/[,。:;!?、]/.test(line)) continue
|
||||||
|
const cleaned = line.replace(/^[\d]+[\.、::]\s*/, '').trim()
|
||||||
|
if (!cleaned) continue
|
||||||
|
if (cleaned.includes(' - ') || cleaned.includes(' → ')) {
|
||||||
|
const parts = cleaned.split(/\s*(?:-|→)\s*/).map((p) => p.trim()).filter((p) => p)
|
||||||
|
for (const p of parts) if (isValid(p)) return p
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (isValid(cleaned)) return cleaned
|
||||||
|
const m = cleaned.match(/([\u4e00-\u9fa5]{2,6})/)
|
||||||
|
if (m && isValid(m[1])) return m[1]
|
||||||
|
}
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
const cases = [
|
||||||
|
['**合金铝线采购** - 这是**电线电缆**的一种。', '', '老污染样本 — 长度>20 全段被丢'],
|
||||||
|
['**接地夹管束采购** - 是**管材**或电气配件', '', '同上 含"采购"项目词'],
|
||||||
|
['钢材', '钢材', '核心品类直接通过'],
|
||||||
|
['', '', '空'],
|
||||||
|
['**SVG/无功补偿装置** - 属于电气设备', '', '含英文字母拒绝'],
|
||||||
|
['钢管采购 — 设备配件', '', '含"采购"项目词'],
|
||||||
|
['\n\n钢材\n', '钢材', '前后空白被 trim'],
|
||||||
|
['让我分析一下,这条属于钢材', '钢材', '多行取尾部真品类'],
|
||||||
|
['钢材\n\n1. 钢材\n2. 管材\n3. 钢材', '钢材', '多行取首行'],
|
||||||
|
['电力金具', '电力金具', '4字新品类'],
|
||||||
|
['管材', '管材', '已有品类'],
|
||||||
|
['设备配件', '设备配件', '4字新品类'],
|
||||||
|
['标识标牌', '标识标牌', '4字'],
|
||||||
|
['工程服务', '工程服务', '4字'],
|
||||||
|
['杂品', '杂品', '2字'],
|
||||||
|
['中国电建', '', '黑名单单字"中国"包含时拒绝 — 但这是项目名前缀,要拆。修正'],
|
||||||
|
['配电箱', '配电箱', '3字'],
|
||||||
|
['阀门', '阀门', '2字'],
|
||||||
|
['一个非常长的品类名候选', '', '>6 字符拒绝'],
|
||||||
|
['防火门', '防火门', '3字'],
|
||||||
|
['建筑材料', '', '含项目词"材料"'],
|
||||||
|
['五金', '五金', '2字'],
|
||||||
|
]
|
||||||
|
|
||||||
|
let pass = 0
|
||||||
|
let fail = 0
|
||||||
|
for (const [input, expected, desc] of cases) {
|
||||||
|
const got = sanitize(input)
|
||||||
|
const ok = got === expected
|
||||||
|
if (ok) pass++; else fail++
|
||||||
|
const flag = ok ? '✓' : '✗'
|
||||||
|
console.log(` ${flag} ${desc}`)
|
||||||
|
if (!ok) {
|
||||||
|
console.log(` input: ${JSON.stringify(input)}`)
|
||||||
|
console.log(` expected: ${JSON.stringify(expected)}`)
|
||||||
|
console.log(` got: ${JSON.stringify(got)}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(`\n${pass}/${pass + fail} passed`)
|
||||||
|
process.exit(fail === 0 ? 0 : 1)
|
||||||
Loading…
Reference in New Issue
Block a user