Compare commits

...

6 Commits

Author SHA1 Message Date
Sisyphus
8d8945f92b docs: 更新 AGENTS.md、gitignore(排除上传文件)、开发文档
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-26 16:25:24 +08:00
Sisyphus
44609be8c7 feat: 线索池品类选项扩展、隐藏数量列、采集器按钮样式优化
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-26 16:25:06 +08:00
Sisyphus
0a423af4d2 feat: 商品导入修复 brand 字段命名 + 添加 unit 和错误日志
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-26 16:24:54 +08:00
Sisyphus
ae72966766 feat: 线索池品类筛选改为按标题关键词匹配
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-26 16:24:43 +08:00
Sisyphus
bd213eb5ec feat: 采集器地址提取优化 - 省份关键词匹配、标题兜底、跨天去重
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-26 16:24:32 +08:00
Sisyphus
49f922444a feat(F002-P1): 调试期累积修复
- 商品上架 MCP: 简化 batchPublish, price=0 默认 0.01, sku 用 spec/name
- 集成 service: 完善 import_products 调用链
- 前端 ProductPage: 调整按钮与列表
- 文档: 本地开发工作流 + devbox F002 升级需求
- 脚本: rotate-mcp-key.sh (一键密钥轮换)
2026-06-25 10:01:45 +08:00
17 changed files with 736 additions and 188 deletions

3
.gitignore vendored
View File

@ -41,6 +41,9 @@ pnpm-debug.log*
# OS # OS
Thumbs.db Thumbs.db
# Uploads (不上传图片文件)
apps/api/public/uploads/
# Backup files (防误提交) # Backup files (防误提交)
*.bak *.bak
*.tmp *.tmp

View File

@ -2,6 +2,48 @@
语言:中文。所有对话、注释、文档、提交信息均使用中文。 语言:中文。所有对话、注释、文档、提交信息均使用中文。
## 🤝 协作流程标准(避免理解不一致造成返工)
**所有具体问题的修改,必须按这个流程执行:**
```
1. 用户描述需求(业务场景、看到的问题、截图、日志)
↓
2. 协作者反问 7 个技术维度(场景/触发/重要程度/数据/边界/样式/部署)
↓
3. 对齐范围:用户确认 + 协作者列技术方案
↓
4. 实施 + 验证
↓
5. 用户看效果,反馈或继续下一项
```
### 协作者必须问的 7 个技术维度
| 维度 | 问什么 |
|------|--------|
| **场景** | 哪个页面/哪个模块?用户路径是什么? |
| **触发条件** | 什么时候出现?手动还是自动?频率? |
| **重要程度** | 阻塞核心流程 / 体验优化 / 锦上添花? |
| **数据** | 影响多少数据?要不要数据迁移? |
| **边界** | 空数据/异常情况怎么处理? |
| **样式** | 改动大吗?要不要出截图对比? |
| **生产部署** | 改完要不要立即部署?还是攒一起? |
### 原则
- **不擅自开干**:未对齐范围前不做实现
- **不预设方案**:用户描述 ≠ 协作者的解决方案,先问清楚
- **不省略验证**:改完必须跑端到端测试,不只是"代码看起来对"
- **不静默改大动作**:涉及数据库结构、生产配置、依赖升级等要明确告知
### 例外(不需要问的简单场景)
- 拼写错误、明显笔误
- 单文件 typo 修复
- 用户明确说"按你判断做"
- 紧急 bug 修复(事后补问)
## 项目结构 ## 项目结构
``` ```

View File

@ -18,6 +18,8 @@ interface FetchResult {
noticeId: string noticeId: string
detailUrl: string detailUrl: string
amount?: string amount?: string
region?: string
regionDetail?: string
} }
@Injectable() @Injectable()
@ -109,7 +111,7 @@ export class CrawlerService implements OnModuleInit {
let results: FetchResult[] = [] let results: FetchResult[] = []
if (config.type === 'bid_powerchina') { if (config.type === 'bid_powerchina') {
results = await this.runPowerChina() results = await this.runPowerChina(config)
} else { } else {
results = await this.runGenericScript(config) results = await this.runGenericScript(config)
} }
@ -129,6 +131,8 @@ export class CrawlerService implements OnModuleInit {
detailUrl: r.detailUrl, detailUrl: r.detailUrl,
publishTime: r.publishTime || null, publishTime: r.publishTime || null,
...(r.amount ? { amount: r.amount } : {}), ...(r.amount ? { amount: r.amount } : {}),
...(r.region ? { region: r.region } : {}),
...(r.regionDetail ? { regionDetail: r.regionDetail } : {}),
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`, summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
}, },
create: { create: {
@ -143,7 +147,8 @@ export class CrawlerService implements OnModuleInit {
status: '待复核', status: '待复核',
level: 'C', level: 'C',
category: '', category: '',
region: '', region: r.region || '',
regionDetail: r.regionDetail || '',
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`, summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
}, },
}) })
@ -172,49 +177,92 @@ export class CrawlerService implements OnModuleInit {
} }
} }
private async runPowerChina(): Promise<FetchResult[]> { private async runPowerChina(config: { keywords?: string | null }): Promise<FetchResult[]> {
const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js') const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js')
if (!fs.existsSync(scriptPath)) { if (!fs.existsSync(scriptPath)) {
throw new Error('中电建抓取脚本不存在') throw new Error('中电建抓取脚本不存在')
} }
execSync(`node "${scriptPath}"`, { let keywords: string[]
try {
keywords = config.keywords ? JSON.parse(config.keywords) : []
} catch {
keywords = []
}
if (keywords.length === 0) keywords = ['']
const today = new Date().toISOString().split('T')[0]
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
const seen = new Set<string>()
const allResults: FetchResult[] = []
for (const kw of keywords) {
const kwFlag = kw ? ` --keyword "${kw}"` : ''
try {
execSync(`node "${scriptPath}"${kwFlag}`, {
encoding: 'utf-8', encoding: 'utf-8',
maxBuffer: 10 * 1024 * 1024, maxBuffer: 10 * 1024 * 1024,
timeout: 180000, timeout: 180000,
}) })
} catch {
// 单个 keyword 失败不影响其他 keyword
continue
}
const today = new Date().toISOString().split('T')[0] const statePath = path.join(baseDir, 'downloaded.json')
const statePath = path.join( if (!fs.existsSync(statePath)) continue
process.env.HOME || '',
'desktop/招标公告',
today,
'downloaded.json',
)
if (!fs.existsSync(statePath)) return []
try {
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
const downloaded = state.downloaded || {} const downloaded = state.downloaded || {}
const results: FetchResult[] = []
const outputDir = path.dirname(statePath)
const pdfDir = path.join(outputDir, 'pdf')
for (const [key, v] of Object.entries(downloaded)) { for (const [key, v] of Object.entries(downloaded)) {
const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string } const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string; publishTime?: string; path?: string }
if (info.skipped || info.failed) continue if (info.skipped || info.failed) continue
const [systemId] = key.split('_') const [systemId] = key.split('_')
if (seen.has(systemId)) continue
seen.add(systemId)
const realId = info.id || systemId const realId = info.id || systemId
results.push({
let region = ''
let regionDetail = ''
if (info.path && fs.existsSync(pdfDir)) {
const pdfPath = path.join(pdfDir, path.basename(info.path))
if (fs.existsSync(pdfPath)) {
try {
const extractor = path.join(SCRIPTS_DIR, 'extract_address_from_pdf.py')
const title = info.title || ''
const cmd = `python3 "${extractor}" "${pdfPath}" "${title}"`
const out = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim()
const parsed = JSON.parse(out)
region = parsed.region || ''
regionDetail = parsed.regionDetail || ''
} catch {}
}
}
allResults.push({
source: '中电建', source: '中电建',
title: info.title || '', title: info.title || '',
organization: '中电建', organization: '中电建',
publishTime: state.lastRun || '', publishTime: info.publishTime || state.lastRun || '',
deadline: info.deadline || '', deadline: info.deadline || '',
noticeId: systemId, noticeId: systemId,
detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`, detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`,
region,
regionDetail,
}) })
} }
} catch {
continue
}
}
return results return allResults
} }
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> { private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
@ -248,10 +296,13 @@ export class CrawlerService implements OnModuleInit {
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
const results: FetchResult[] = [] const results: FetchResult[] = []
const todayStr = new Date().toISOString().substring(0, 10)
for (const [noticeId, v] of Object.entries(state)) { for (const [noticeId, v] of Object.entries(state)) {
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string } const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
if (noticeId === 'lastRun') continue if (noticeId === 'lastRun') continue
// 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
const leadAmount = info.amount || '' const leadAmount = info.amount || ''
results.push({ results.push({
source, source,
@ -262,6 +313,8 @@ export class CrawlerService implements OnModuleInit {
noticeId: info.id || noticeId, noticeId: info.id || noticeId,
detailUrl: info.detail_url || '', detailUrl: info.detail_url || '',
amount: leadAmount, amount: leadAmount,
region: info.region || '',
regionDetail: info.region_detail || '',
}) })
} }

View File

@ -373,11 +373,11 @@ export class IntegrationService implements OnModuleDestroy {
* 解析 MCP tool 返回的 content 字段 * 解析 MCP tool 返回的 content 字段
* MCP 协议格式: { content: [{ type: 'text', text: '...' }], isError?: boolean } * MCP 协议格式: { content: [{ type: 'text', text: '...' }], isError?: boolean }
*/ */
private parseToolResult<T>(result: any): T | null { private parseToolResult<T>(result: any): T {
try { if (!result) throw new Error('MCP 工具返回结果为空')
if (!result) return null
if (result.isError) { if (result.isError) {
throw new Error(typeof result.content?.[0]?.text === 'string' ? result.content[0].text : 'Tool 调用返回错误') const errMsg = typeof result.content?.[0]?.text === 'string' ? result.content[0].text : 'Tool 调用返回错误'
throw new Error(errMsg)
} }
if (result.content && Array.isArray(result.content) && result.content.length > 0) { if (result.content && Array.isArray(result.content) && result.content.length > 0) {
const item = result.content[0] const item = result.content[0]
@ -390,10 +390,6 @@ export class IntegrationService implements OnModuleDestroy {
} }
} }
return result as T return result as T
} catch (e: any) {
this.logger.error(`解析 MCP tool 结果失败: ${e.message}`)
return null
}
} }
getClient(integrationId: string): Client | null { getClient(integrationId: string): Client | null {

View File

@ -14,8 +14,9 @@ export class LeadController {
@Query('source') source?: string, @Query('source') source?: string,
@Query('category') category?: string, @Query('category') category?: string,
@Query('assignedTo') assignedTo?: string, @Query('assignedTo') assignedTo?: string,
@Query('timeRange') timeRange?: string,
) { ) {
return this.leadService.findAll({ status, level, search, star, source, category, assignedTo }) return this.leadService.findAll({ status, level, search, star, source, category, assignedTo, timeRange })
} }
@Get('stats') @Get('stats')

View File

@ -2,6 +2,25 @@ import { Injectable, NotFoundException, Logger, ServiceUnavailableException } fr
import { PrismaService } from '../prisma/prisma.service' import { PrismaService } from '../prisma/prisma.service'
import { IntegrationService } from '../integration/integration.service' import { IntegrationService } from '../integration/integration.service'
// 品类 → 标题关键词映射(筛选时按标题模糊匹配,不走 category 字段)
const CATEGORY_KEYWORDS: Record<string, string[]> = {
'防火门': ['防火门', '防火卷帘', '防火窗', '钢质防火', '木质防火'],
'配电箱': ['配电箱', '配电柜', '配电盘', '照明配电箱', '动力配电箱', '电表箱'],
'电线电缆': ['电线', '电缆', '导线', '电力线', '控制电缆', '光缆', '架空线'],
'阀门': ['阀门', '闸阀', '球阀', '蝶阀', '截止阀', '止回阀', '调节阀'],
'钢材': ['钢材', '钢管', '钢板', '钢筋', '型钢', '槽钢', '角钢', '工字钢', 'H型钢', '螺纹钢'],
'涂料': ['涂料', '油漆', '防腐漆', '防火涂料', '乳胶漆', '防水涂料'],
'混凝土': ['混凝土', '商混', '商品砼', '水泥', '砂浆', '砼'],
'变压器': ['变压器', '箱变', '干式变', '油浸变', '电力变压器'],
'水泵': ['水泵', '离心泵', '潜污泵', '消防泵', '排污泵', '循环泵'],
'风机': ['风机', '通风机', '排风机', '轴流风机', '离心风机'],
'管材': ['管材', '管道', 'PVC管', 'PE管', '镀锌管', '无缝管', '螺旋管', '给水管'],
'开关': ['开关', '断路器', '接触器', '继电器', '熔断器', '隔离开关'],
'灯具': ['灯具', '照明灯', '路灯', '防爆灯', 'LED灯', '太阳能灯'],
'仪表': ['仪表', '仪器', '传感器', '变送器', '流量计', '压力表', '温度计'],
'电机': ['电机', '电动机', '减速机', '减速器', '变频电机'],
}
@Injectable() @Injectable()
export class LeadService { export class LeadService {
private readonly logger = new Logger(LeadService.name) private readonly logger = new Logger(LeadService.name)
@ -11,7 +30,7 @@ export class LeadService {
private integrationService: IntegrationService, private integrationService: IntegrationService,
) {} ) {}
async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string }) { async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string; timeRange?: string }) {
const where: any = {} const where: any = {}
if (params?.status) where.status = params.status if (params?.status) where.status = params.status
if (params?.level) where.level = params.level if (params?.level) where.level = params.level
@ -28,14 +47,44 @@ export class LeadService {
else if (!isNaN(starVal)) where.star = starVal else if (!isNaN(starVal)) where.star = starVal
} }
if (params?.source) where.source = params.source if (params?.source) where.source = params.source
if (params?.category) where.category = params.category if (params?.category) {
const keywords = CATEGORY_KEYWORDS[params.category]
if (keywords) {
where.OR = keywords.map(kw => ({ title: { contains: kw } }))
}
}
if (params?.assignedTo) { if (params?.assignedTo) {
where.assignedTo = params.assignedTo === '未分配' ? null : params.assignedTo where.assignedTo = params.assignedTo === '未分配' ? null : params.assignedTo
} }
if (params?.timeRange && params.timeRange !== '全部时间') {
const now = new Date()
const pad = (n: number) => n.toString().padStart(2, '0')
const today = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`
if (params.timeRange === '今天') {
where.publishTime = { startsWith: today }
} else if (params.timeRange === '昨天') {
const y = new Date(now)
y.setDate(y.getDate() - 1)
const yesterday = `${y.getFullYear()}-${pad(y.getMonth() + 1)}-${pad(y.getDate())}`
where.publishTime = { startsWith: yesterday }
} else if (params.timeRange === '本周') {
const day = now.getDay()
const diff = day === 0 ? 6 : day - 1 // Monday
const monday = new Date(now)
monday.setDate(now.getDate() - diff)
const sunday = new Date(monday)
sunday.setDate(monday.getDate() + 6)
const m = `${monday.getFullYear()}-${pad(monday.getMonth() + 1)}-${pad(monday.getDate())}`
const s = `${sunday.getFullYear()}-${pad(sunday.getMonth() + 1)}-${pad(sunday.getDate())}`
where.publishTime = { gte: m, lte: s + ' 23:59:59' }
} else if (params.timeRange === '本月') {
where.publishTime = { startsWith: `${now.getFullYear()}-${pad(now.getMonth() + 1)}` }
}
}
return this.prisma.lead.findMany({ return this.prisma.lead.findMany({
where, where,
orderBy: [ orderBy: [
{ level: 'asc' }, { deadline: 'desc' },
{ createdAt: 'desc' }, { createdAt: 'desc' },
], ],
}) })

View File

@ -40,14 +40,6 @@ export class ProductController {
return this.productService.batchPublish(body.ids) return this.productService.batchPublish(body.ids)
} }
/**
* F002 P0-A: 批量上架商品到 B2B 平台 (MCP import_products)
*/
@Post('publish-to-direv')
publishToDirev(@Body() body: { ids: string[] }) {
return this.productService.publishToDirev(body.ids)
}
@Delete(':id') @Delete(':id')
remove(@Param('id') id: string) { remove(@Param('id') id: string) {
return this.productService.remove(id) return this.productService.remove(id)

View File

@ -42,90 +42,87 @@ export class ProductService {
} }
async batchPublish(ids: string[]) { async batchPublish(ids: string[]) {
return this.prisma.product.updateMany({
where: { id: { in: ids } },
data: { status: 'published' },
})
}
async remove(id: string) {
return this.prisma.product.delete({ where: { id } })
}
/**
* F002 P0-A: 批量上架商品到 B2B 平台 (MCP import_products)
* 字段映射: Product → import_products schema
*/
async publishToDirev(ids: string[]) {
const b2bIntegration = await this.integrationService.findActiveMCP()
if (!b2bIntegration) {
throw new ServiceUnavailableException('未配置 B2B 集成')
}
const products = await this.prisma.product.findMany({ const products = await this.prisma.product.findMany({
where: { id: { in: ids } }, where: { id: { in: ids } },
}) })
if (products.length === 0) return { imported: 0, errors: [], ids: [] }
if (products.length === 0) { const b2bIntegration = await this.integrationService.findActiveMCP()
throw new ServiceUnavailableException('未找到指定商品') if (!b2bIntegration) {
throw new ServiceUnavailableException('未配置 B2B 平台 MCP 集成,请在系统配置中添加')
} }
// Category string → numeric ID mapping (B2B tb_good first_category_id)
const CATEGORY_MAP: Record<string, number> = { const CATEGORY_MAP: Record<string, number> = {
'电缆': 28, '电缆': 28, '电力电缆': 28, '配电箱': 29, '开关柜': 29,
'电力电缆': 28, '变压器': 27, '电线': 28, '母线槽': 30, '桥架': 31,
'配电箱': 29,
'开关柜': 29,
'变压器': 27,
'电线': 28,
'母线槽': 30,
'桥架': 31,
} }
const fallbackCategoryId = 1 const fallbackCategoryId = 1
const items = products.map(p => { const mappedItems = products.map(p => {
const catId = CATEGORY_MAP[p.category] || fallbackCategoryId const catId = CATEGORY_MAP[p.category] || fallbackCategoryId
const item: any = { const item: any = {
name: p.name, name: p.name, category_id: catId,
category_id: catId, price: p.price > 0 ? p.price : 0.01,
price: p.price, brand: p.brand || '',
brandName: p.brand || '', unit: p.unit || '',
stock: p.quantity, sku: p.spec || p.name || '',
} }
if (p.spec) item.sku = p.spec
if (p.imageUrl) item.images = [p.imageUrl] if (p.imageUrl) item.images = [p.imageUrl]
if (p.supplier) item.remark = `供应商:${p.supplier}` if (p.supplier) item.remark = `供应商:${p.supplier}`
return item return item
}) })
this.logger.log(`batchPublish: 发送 ${mappedItems.length} 个商品到 MCP import_products`)
const result = await this.integrationService.executeTool<any>( const result = await this.integrationService.executeTool<any>(
b2bIntegration.id, b2bIntegration.id, 'import_products',
'import_products', { products: mappedItems },
{ products: items },
{ retries: 2, retryDelayMs: 1000 }, { retries: 2, retryDelayMs: 1000 },
) )
this.logger.log(`batchPublish: MCP 返回 success=${result.success} data=${result.data ? JSON.stringify(result.data).slice(0, 300) : '(empty)'}`)
if (!result.success) { if (!result.success) {
const errorMsg = result.error || '未知错误' throw new ServiceUnavailableException(`B2B 同步失败: ${result.error || '未知错误'}`)
this.logger.error(`商品上架失败 (ids=${ids.join(',')}): ${errorMsg}`)
throw new ServiceUnavailableException(`上架失败: ${errorMsg}`)
} }
const data = result.data as any // MCP import_products 返回格式: { code:0, msg:"成功", data: [{ status:"success"|"failed", data:{id:"xxx"}, sku, name, error? }] }
const payload = data.data || data const items: Array<{ status: string; data?: any; sku?: string; name?: string; error?: string }> =
const imported = payload.imported || 0 (result.data as any)?.data || []
const errors = payload.errors || [] const imported = items.filter(i => i.status === 'success').length
this.logger.log(`商品上架成功: 导入 ${imported} 条, 失败 ${errors.length} 条`) const b2bIds: string[] = items
.filter(i => i.status === 'success' && i.data?.id)
.map(i => String(i.data.id))
const errors: Array<{ sku: string; name: string; error: string }> = items
.filter(i => i.status === 'failed')
.map(i => ({ sku: i.sku || '', name: i.name || '', error: i.error || '未知错误' }))
if (errors.length > 0) {
this.logger.warn(`B2B import_products 导入失败详情: ${errors.map(e => `[${e.sku}] ${e.name}: ${e.error}`).join(' | ')}`)
}
this.logger.log(`B2B import_products 完成: 导入 ${imported} 条, 失败 ${errors.length} 条, B2B IDs: ${b2bIds.join(',')}`)
if (b2bIds.length > 0) {
this.logger.log(`batchPublish: 上架 ${b2bIds.length} 个商品到 B2B publish_products`)
const publishResult = await this.integrationService.executeTool<any>(
b2bIntegration.id, 'publish_products',
{ product_ids: b2bIds },
{ retries: 1, retryDelayMs: 1000 },
)
this.logger.log(`batchPublish: publish_products 返回 success=${publishResult.success}`)
if (!publishResult.success) {
throw new ServiceUnavailableException(`B2B 上架失败: ${publishResult.error || 'publish_products 返回失败'}`)
}
this.logger.log(`B2B 上架成功: ${b2bIds.length} 个商品`)
}
await this.prisma.product.updateMany({ await this.prisma.product.updateMany({
where: { id: { in: products.map(p => p.id) } }, where: { id: { in: ids } },
data: { status: 'published' }, data: { status: 'published' },
}) })
return { return { imported, errors, ids: b2bIds }
imported, }
errors,
ids: payload.ids || [], async remove(id: string) {
} return this.prisma.product.delete({ where: { id } })
} }
} }

View File

@ -1 +1 @@
NEXT_PUBLIC_API_URL= NEXT_PUBLIC_API_URL=/api

View File

@ -35,7 +35,7 @@ function mapLead(raw: any): Lead {
id: raw.id, id: raw.id,
title: raw.title, title: raw.title,
star: raw.star || 0, star: raw.star || 0,
level: raw.level || 'C', level: raw.level ? raw.level + '级' : 'C级',
amount: raw.amount || '待确认', amount: raw.amount || '待确认',
status: raw.status || '待复核', status: raw.status || '待复核',
deadline: raw.deadline || '', deadline: raw.deadline || '',
@ -58,9 +58,9 @@ function mapLead(raw: any): Lead {
} }
const STATUS_OPTIONS = ['全部状态', '待复核', '待跟进', '跟进中', '已转询价', '已转协同', '已成交', '无效线索'] const STATUS_OPTIONS = ['全部状态', '待复核', '待跟进', '跟进中', '已转询价', '已转协同', '已成交', '无效线索']
const LEVEL_OPTIONS = ['全部级别', 'S级', 'A级', 'B级'] const LEVEL_OPTIONS = ['全部级别', 'S级', 'A级', 'B级', 'C级', 'D级']
const CAT_OPTIONS = ['全部品类', '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土'] const CAT_OPTIONS = ['全部品类', '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土', '变压器', '水泵', '风机', '管材', '开关', '灯具', '仪表', '电机']
const SOURCE_OPTIONS = ['全部来源', 'PowerChina 招采', '新矿集团', '发改委审批', '行业资讯'] const SOURCE_OPTIONS = ['全部来源', '中电建', '新矿集团', '中核集团']
const PERSON_OPTIONS = ['全部人员', '郑钊', '赵主任', '运营管理员', '未分配'] const PERSON_OPTIONS = ['全部人员', '郑钊', '赵主任', '运营管理员', '未分配']
function getStatusClass(status: string) { function getStatusClass(status: string) {
@ -110,7 +110,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
const showToast = (msg: string) => { setToast(msg); setTimeout(() => setToast(''), 2500) } const showToast = (msg: string) => { setToast(msg); setTimeout(() => setToast(''), 2500) }
const [fTime, setFTime] = useState('全部时间') const [fTime, setFTime] = useState('全部日期')
const [fStar, setFStar] = useState('全部星级') const [fStar, setFStar] = useState('全部星级')
const [fStatus, setFStatus] = useState('全部状态') const [fStatus, setFStatus] = useState('全部状态')
const [fLevel, setFLevel] = useState('全部级别') const [fLevel, setFLevel] = useState('全部级别')
@ -118,7 +118,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
const [fSource, setFSource] = useState('全部来源') const [fSource, setFSource] = useState('全部来源')
const [fPerson, setFPerson] = useState('全部人员') const [fPerson, setFPerson] = useState('全部人员')
const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string) => { const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string, timeRange?: string) => {
setLoading(true) setLoading(true)
const params = new URLSearchParams() const params = new URLSearchParams()
if (star && star !== '全部星级') { if (star && star !== '全部星级') {
@ -128,11 +128,12 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
else if (star === '2星及以下') params.set('star', '2星及以下') else if (star === '2星及以下') params.set('star', '2星及以下')
} }
if (status && status !== '全部状态') params.set('status', status) if (status && status !== '全部状态') params.set('status', status)
if (level && level !== '全部级别') params.set('level', level) if (level && level !== '全部级别') params.set('level', level.replace('级', ''))
if (cat && cat !== '全部品类') params.set('category', cat) if (cat && cat !== '全部品类') params.set('category', cat)
if (source && source !== '全部来源') params.set('source', source) if (source && source !== '全部来源') params.set('source', source)
if (person && person !== '全部人员' && person !== '未分配') params.set('assignedTo', person) if (person && person !== '全部人员' && person !== '未分配') params.set('assignedTo', person)
if (person === '未分配') params.set('assignedTo', '未分配') if (person === '未分配') params.set('assignedTo', '未分配')
if (timeRange && timeRange !== '全部日期') params.set('timeRange', timeRange)
try { try {
const qs = params.toString() const qs = params.toString()
const data: any[] = await fetch(`${API}${qs ? '?' + qs : ''}`).then(r => r.json()) const data: any[] = await fetch(`${API}${qs ? '?' + qs : ''}`).then(r => r.json())
@ -144,10 +145,10 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
setLoading(false) setLoading(false)
}, []) }, [])
useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson]) useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime])
const refreshLeads = async () => { const refreshLeads = async () => {
await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson) await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime)
} }
const totalPages = Math.ceil(filtered.length / pageSize) const totalPages = Math.ceil(filtered.length / pageSize)
@ -392,7 +393,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
<div className="bg-white p-4 border border-border-subtle rounded-lg shadow-sm flex flex-wrap items-center gap-4"> <div className="bg-white p-4 border border-border-subtle rounded-lg shadow-sm flex flex-wrap items-center gap-4">
<div className="flex flex-wrap items-center gap-4"> <div className="flex flex-wrap items-center gap-4">
{[ {[
{ label: '时间范围', value: fTime, set: setFTime, opts: ['全部时间', '今天', '昨天', '本周', '本月'] }, { label: '公告日期', value: fTime, set: setFTime, opts: ['全部日期', '今天', '昨天', '本周', '本月'] },
{ label: '星级', value: fStar, set: setFStar, opts: ['全部星级', '5星', '4星', '3星', '2星及以下'] }, { label: '星级', value: fStar, set: setFStar, opts: ['全部星级', '5星', '4星', '3星', '2星及以下'] },
{ label: '状态', value: fStatus, set: setFStatus, opts: STATUS_OPTIONS }, { label: '状态', value: fStatus, set: setFStatus, opts: STATUS_OPTIONS },
{ label: '甲方级别', value: fLevel, set: setFLevel, opts: LEVEL_OPTIONS }, { label: '甲方级别', value: fLevel, set: setFLevel, opts: LEVEL_OPTIONS },
@ -445,7 +446,6 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">星级</th> <th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">星级</th>
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">甲方级别</th> <th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">甲方级别</th>
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase text-right whitespace-nowrap">金额(万)</th> <th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase text-right whitespace-nowrap">金额(万)</th>
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">数量</th>
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">状态</th> <th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">状态</th>
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">投标截止</th> <th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">投标截止</th>
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">公告日期</th> <th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">公告日期</th>
@ -454,9 +454,9 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
</thead> </thead>
<tbody className="divide-y divide-border-subtle"> <tbody className="divide-y divide-border-subtle">
{loading ? ( {loading ? (
<tr><td colSpan={11} className="p-8 text-center text-sm text-on-surface-variant">加载中...</td></tr> <tr><td colSpan={10} className="p-8 text-center text-sm text-on-surface-variant">加载中...</td></tr>
) : pageLeads.length === 0 ? ( ) : pageLeads.length === 0 ? (
<tr><td colSpan={11} className="p-8 text-center text-sm text-on-surface-variant">暂无数据</td></tr> <tr><td colSpan={10} className="p-8 text-center text-sm text-on-surface-variant">暂无数据</td></tr>
) : pageLeads.map(lead => { ) : pageLeads.map(lead => {
const stars = '★'.repeat(lead.star) + '☆'.repeat(5 - lead.star) const stars = '★'.repeat(lead.star) + '☆'.repeat(5 - lead.star)
return ( return (
@ -476,7 +476,6 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
<td className="p-4 text-right whitespace-nowrap"> <td className="p-4 text-right whitespace-nowrap">
<span className={lead.amount === '待确认' ? 'text-[10px] text-slate-400' : 'text-[11px] font-medium'}>{lead.amount}</span> <span className={lead.amount === '待确认' ? 'text-[10px] text-slate-400' : 'text-[11px] font-medium'}>{lead.amount}</span>
</td> </td>
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.quantity ? `${lead.quantity}${lead.unit || ''}` : '-'}</td>
<td className="p-4 whitespace-nowrap"><span className={`px-2 py-0.5 text-[10px] font-medium rounded-full ${getStatusClass(lead.status)}`}>{lead.status}</span></td> <td className="p-4 whitespace-nowrap"><span className={`px-2 py-0.5 text-[10px] font-medium rounded-full ${getStatusClass(lead.status)}`}>{lead.status}</span></td>
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.deadline ? lead.deadline.slice(5, 10) : '-'}</td> <td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.deadline ? lead.deadline.slice(5, 10) : '-'}</td>
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.publishTime ? lead.publishTime.slice(0, 10) : '-'}</td> <td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.publishTime ? lead.publishTime.slice(0, 10) : '-'}</td>

View File

@ -172,21 +172,8 @@ export default function ProductPage() {
const batchSync = async () => { const batchSync = async () => {
if (selectedIds.size === 0) return if (selectedIds.size === 0) return
showToast(`正在同步 ${selectedIds.size} 个商品...`) showToast(`正在同步 ${selectedIds.size} 个商品...`)
await fetch(`${API}/batch-publish`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: Array.from(selectedIds) }),
})
setSelectedIds(new Set())
await loadProducts()
showToast('批量同步完成')
}
const batchPublishToDirev = async () => {
if (selectedIds.size === 0) return
showToast(`正在上架 ${selectedIds.size} 个商品到 B2B 平台...`)
try { try {
const res = await fetch(`${API}/publish-to-direv`, { const res = await fetch(`${API}/batch-publish`, {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: Array.from(selectedIds) }), body: JSON.stringify({ ids: Array.from(selectedIds) }),
@ -195,9 +182,9 @@ export default function ProductPage() {
if (!res.ok) throw new Error(data.message || data.error || '请求失败') if (!res.ok) throw new Error(data.message || data.error || '请求失败')
setSelectedIds(new Set()) setSelectedIds(new Set())
await loadProducts() await loadProducts()
showToast(`上架成功: ${data.imported || 0} 条${data.errors?.length ? `, 失败 ${data.errors.length} 条` : ''}`) showToast(`同步成功: ${data.imported || 0} 条导入 B2B${data.errors?.length ? `, 失败 ${data.errors.length} 条` : ''}`)
} catch (e: any) { } catch (e: any) {
showToast(`上架失败: ${e.message}`) showToast(`同步失败: ${e.message}`)
} }
} }
@ -472,11 +459,6 @@ export default function ProductPage() {
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" /></svg> <svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" /></svg>
批量同步 批量同步
</button> </button>
<button disabled={selectedIds.size === 0} onClick={batchPublishToDirev}
className="px-3 py-1.5 text-[10px] font-medium bg-emerald-600 text-white rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:opacity-90">
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M5 13l4 4L19 7" /></svg>
上架 B2B
</button>
<button disabled={selectedIds.size === 0} onClick={batchGenerate} <button disabled={selectedIds.size === 0} onClick={batchGenerate}
className="px-3 py-1.5 text-[10px] font-medium bg-white border border-purple-200 text-purple-700 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:bg-purple-50 transition-colors"> className="px-3 py-1.5 text-[10px] font-medium bg-white border border-purple-200 text-purple-700 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:bg-purple-50 transition-colors">
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" /></svg> <svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" /></svg>

View File

@ -799,9 +799,13 @@ export default function RuleCenterPage() {
<td className="p-3"> <td className="p-3">
<button <button
onClick={() => toggleCrawlerStatus(c.id, c.status)} onClick={() => toggleCrawlerStatus(c.id, c.status)}
className={`px-2 py-0.5 text-[10px] font-medium rounded-full border ${getStatusStyle(c.status)}`} className={`px-3 py-1 text-[11px] font-semibold rounded-md transition-all cursor-pointer ${
c.status === 'active'
? 'bg-amber-50 text-amber-700 hover:bg-amber-100 active:bg-amber-200'
: 'bg-emerald-50 text-emerald-700 hover:bg-emerald-100 active:bg-emerald-200'
}`}
> >
{{ active: '暂停', paused: '启动', error: '重启' }[c.status] || c.status} {{ active: '点击暂停', paused: '点击启动', error: '重启' }[c.status] || c.status}
</button> </button>
</td> </td>
<td className="p-3 text-xs text-on-surface-variant"> <td className="p-3 text-xs text-on-surface-variant">

View File

@ -80,7 +80,7 @@ sshpass -p 'Jubianshuzi@2026' ssh ubuntu@172.81.241.109 "
# 4. 验证 # 4. 验证
curl -s -o /dev/null -w 'API: %{http_code}\n' http://172.81.241.109:4000/api/health curl -s -o /dev/null -w 'API: %{http_code}\n' http://172.81.241.109:4000/api/health
curl -s -o /dev/null -w 'Web: %{http_code}\n' https://zoupeng.cloud curl -s -o /dev/null -w 'Web: %{http_code}\n' https://direv.cloud
``` ```
--- ---
@ -105,6 +105,8 @@ curl -s -o /dev/null -w 'Web: %{http_code}\n' https://zoupeng.cloud
| 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — | | 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — |
| 2026-06-23 | **密钥再次重置 + 司库系统重发**:devbox 傍晚又把密钥重置(hash 前缀 `m3hRN → diPRva`,update_date 18:18:22)。用 `reset-secret` 端点再次自助重置拿到新密钥 `fcb53684845648e2a7fc1082f5245180cfa4eea1f5404176`。**用户反馈现象**:选了 `司库系统升级完善项目` 点一键发布,本地显示"发布成功"但 B2B 后台查不到。**根因**:MCP 鉴权失败后代码降级为本地假订单号(X+时间戳),让用户误以为成功。 | scripts/rotate-mcp-key.sh | | 2026-06-23 | **密钥再次重置 + 司库系统重发**:devbox 傍晚又把密钥重置(hash 前缀 `m3hRN → diPRva`,update_date 18:18:22)。用 `reset-secret` 端点再次自助重置拿到新密钥 `fcb53684845648e2a7fc1082f5245180cfa4eea1f5404176`。**用户反馈现象**:选了 `司库系统升级完善项目` 点一键发布,本地显示"发布成功"但 B2B 后台查不到。**根因**:MCP 鉴权失败后代码降级为本地假订单号(X+时间戳),让用户误以为成功。 | scripts/rotate-mcp-key.sh |
| 2026-06-23 | **彻底移除降级本地模式 + MCP 失败弹警告**:(1) 后端 `publishToDirev` 不再写本地假 `inquiryNo=X...`;MCP 鉴权/调用失败时抛 `ServiceUnavailableException` 返回 503 + 详细错误信息;Lead 状态完全不动。(2) 前端 `doPublish` 收集每条失败原因(fetch `res.json().message`),失败时弹新 Modal(红框 + 错误列表 + 「去 MCP 配置」/「我知道了」按钮)。**端到端验证**:把密钥改成 `invalid_test_key_for_qa` → publish 返回 HTTP 503 + JSON 含 `appSecret 校验失败` + 引导文案;Lead 表 publish/inquiryNo/followLogs 全部不变。 | apps/api/src/lead/lead.service.ts, apps/web/src/components/InquiryPage.tsx | | 2026-06-23 | **彻底移除降级本地模式 + MCP 失败弹警告**:(1) 后端 `publishToDirev` 不再写本地假 `inquiryNo=X...`;MCP 鉴权/调用失败时抛 `ServiceUnavailableException` 返回 503 + 详细错误信息;Lead 状态完全不动。(2) 前端 `doPublish` 收集每条失败原因(fetch `res.json().message`),失败时弹新 Modal(红框 + 错误列表 + 「去 MCP 配置」/「我知道了」按钮)。**端到端验证**:把密钥改成 `invalid_test_key_for_qa` → publish 返回 HTTP 503 + JSON 含 `appSecret 校验失败` + 引导文案;Lead 表 publish/inquiryNo/followLogs 全部不变。 | apps/api/src/lead/lead.service.ts, apps/web/src/components/InquiryPage.tsx |
| 2026-06-25 | **生产部署(commit 49f9224)**:本地全量构建后 rsync 到 `/opt/pingtai/`;生产跑 `npx prisma db push + generate`;生产 `next build`(有 lockfile 警告但构建成功);`pm2 restart pingtai-api pingtai-web --update-env`。**手工补齐生产 db 缺列**(prisma db push 路径错):真生产 db 是 `/data/pingtai/dev.db`(不是 `/opt/pingtai/apps/api/prisma/dev.db`),缺 `Lead.regionDetail/quantity/unit/attachments/notes/notesExtractedAt/notesSource` + `Integration.companyUserId` 共 8 列。手工 `ALTER TABLE` 全部补齐,95 条 Lead 数据完整保留。**生产验证**:API + Web online 13m+;lead 95 条、product 75 条、crawler 3 条、inquiry 3 条都正常返回。**新建 Integration 配置**(生产 db 之前为空):在生产创建 MCP 集成 id=751ab09c-e6ab-4c97-bd2d-2c317df32893,appId=app_9dd07cb4e0e24fccbecd。**遗留问题**:生产服务器 172.81.241.109 → devbox 100.111.84.16:3002 网络不通(ping 100% 丢包,curl 5s 超时)。MCP publish_rfq 端到端在生产跑不通(生产 connect 集成返回 `fetch failed`),publish 接口在生产会返回 503 + "未配置 B2B 集成"或"网络不通",这是网络层面的限制不是代码问题。 | apps/api/src/lead/lead.service.ts, docs/本地开发工作流.md |
| 2026-06-25 | **主域名切换:弃用 zoupeng.cloud,改用 direv.cloud**。**根因**:`zoupeng.cloud` DNS 解析到 `222.137.3.123`(旧/无效 IP,ping 通但 443 不通),而 `direv.cloud` DNS 正确解析到 `172.81.241.109`(实际生产服务器 IP)。**改动**:文档第 83 行 `https://zoupeng.cloud` → `https://direv.cloud`,rsync 同步到生产。**长期建议**:去腾讯云 DNSPod 控制台把 `zoupeng.cloud` 的 A 记录改到 `172.81.241.109`;生产 nginx 加 `zoupeng.cloud` 的 server block(同 direv.cloud),否则 `zoupeng.cloud` 永远打不开。AGENTS.md 里"主 `https://zoupeng.cloud`、备 `https://direv.cloud`"是错的,应反过来。 | docs/本地开发工作流.md |
| 2026-06-22 | **devbox F002 升级完成 + 平台助手二次适配**:devbox 已把 publish_rfq schema 升级(W29.7, 补 11 商务字段),Java 后端 `InquiryServiceImpl.publish()` 也补全字段映射。平台助手这边把新参数加进 `publishToDirev` 的 `publishArgs`:deadline(直接传 Lead.deadline YYYY-MM-DD)、deliveryDate(自动 deadline+30 天)、shippingAddress(优先 Lead.regionDetail, fallback region)、contactName/Phone(从 Lead.contact/phone)、inquiryParty("郑州巨变")、freightFlag("含运费")、procurementType("定量采购")、invoiceType("增值税专票")、paymentMethod("货到付款") | apps/api/src/lead/lead.service.ts | | 2026-06-22 | **devbox F002 升级完成 + 平台助手二次适配**:devbox 已把 publish_rfq schema 升级(W29.7, 补 11 商务字段),Java 后端 `InquiryServiceImpl.publish()` 也补全字段映射。平台助手这边把新参数加进 `publishToDirev` 的 `publishArgs`:deadline(直接传 Lead.deadline YYYY-MM-DD)、deliveryDate(自动 deadline+30 天)、shippingAddress(优先 Lead.regionDetail, fallback region)、contactName/Phone(从 Lead.contact/phone)、inquiryParty("郑州巨变")、freightFlag("含运费")、procurementType("定量采购")、invoiceType("增值税专票")、paymentMethod("货到付款") | apps/api/src/lead/lead.service.ts |
| 2026-06-22 | **devbox 升级导致密钥重置**:`app_9dd07cb4e0e24fccbecd` 的 appSecret 被重置(MySQL 里 hash 前缀从 `$2a$10$iR3Rl` 变 `$2a$10$m3hRN`)。用 `POST /sys/openapi/app/reset-secret/{id}` 端点自助重置拿到新密钥 `d02dc00c6fbe4a34a5bf013134146c3f2759148c61a341d2`,本地集成密钥已轮换 | scripts/rotate-mcp-key.sh | | 2026-06-22 | **devbox 升级导致密钥重置**:`app_9dd07cb4e0e24fccbecd` 的 appSecret 被重置(MySQL 里 hash 前缀从 `$2a$10$iR3Rl` 变 `$2a$10$m3hRN`)。用 `POST /sys/openapi/app/reset-secret/{id}` 端点自助重置拿到新密钥 `d02dc00c6fbe4a34a5bf013134146c3f2759148c61a341d2`,本地集成密钥已轮换 | scripts/rotate-mcp-key.sh |
| 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — | | 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — |

View File

@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""
从招标公告 PDF 中提取地址信息。
用法: python3 extract_address_from_pdf.py <pdf文件路径> [公告标题]
输出: JSON {"region": "省/市", "regionDetail": "详细地址"}
"""
import fitz
import re
import json
import sys
# 全国省份/直辖市/自治区名称(含简称、别称)
PROVINCES = [
'北京', '天津', '上海', '重庆',
'河北', '山西', '辽宁', '吉林', '黑龙江',
'江苏', '浙江', '安徽', '福建', '江西', '山东',
'河南', '湖北', '湖南', '广东', '海南',
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
]
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
def extract_province_best(text):
"""从任意文本中提取最可能的省份(先关键词匹配,避免正则在多字省份上截断)"""
if not text:
return ''
text_clean = text.replace(' ', '')
# 1. 特殊多字省份(优先,避免"新疆维吾尔自治区"被截)
for prov in SPECIAL_PROVS:
if prov in text_clean:
return prov
# 2. 直辖市
for prov in ['北京', '天津', '上海', '重庆']:
if prov in text_clean:
return prov
# 3. "XX省"(如"陕西省"、"广东省")
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
if m:
return m.group(1)[:2]
# 4. 两字省份关键词
for prov in TWO_CHAR_PROVS:
if prov in text_clean:
return prov
return ''
def extract_city_from_text(text):
"""从文本中提取城市名(XX市)"""
if not text:
return ''
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
if m:
city = m.group(1)
if city not in ['北京市', '天津市', '上海市', '重庆市']:
return city
return ''
def extract_address(pdf_path: str, title: str = '') -> dict:
pdf_text = ''
try:
doc = fitz.open(pdf_path)
for page in doc:
pdf_text += page.get_text()
doc.close()
except Exception:
pdf_text = ''
result = {'region': '', 'regionDetail': ''}
# ── 1. 从PDF正文中提取详细地址 ──
addr_patterns = [
r'建设地点[::]\s*(.+?)(?:\n|$)',
r'项目地点[::]\s*(.+?)(?:\n|$)',
r'交货地点[::]\s*(.+?)(?:\n|$)',
r'施工地点[::]\s*(.+?)(?:\n|$)',
r'项目地址[::]\s*(.+?)(?:\n|$)',
r'供货地点[::]\s*(.+?)(?:\n|$)',
r'到货地点[::]\s*(.+?)(?:\n|$)',
r'送货地点[::]\s*(.+?)(?:\n|$)',
r'工程地点[::]\s*(.+?)(?:\n|$)',
r'实施地点[::]\s*(.+?)(?:\n|$)',
r'服务地点[::]\s*(.+?)(?:\n|$)',
r'收货地址[::]\s*(.+?)(?:\n|$)',
r'所在地[::]\s*(.+?)(?:\n|$)',
]
addr = ''
for p in addr_patterns:
m = re.search(p, pdf_text)
if m:
addr = m.group(1).strip()
if len(addr) >= 4:
break
addr = ''
if addr:
result['regionDetail'] = addr
prov = extract_province_best(addr)
if prov:
result['region'] = prov
else:
city = extract_city_from_text(addr)
if city:
result['region'] = city
# ── 2. 正文没提取到region,从整个PDF文本中搜索 ──
if not result['region'] and pdf_text:
prov = extract_province_best(pdf_text)
if prov:
result['region'] = prov
# ── 3. 还没提取到,从标题中搜索 ──
if not result['region'] and title:
prov = extract_province_best(title)
if prov:
result['region'] = prov
return result
if __name__ == '__main__':
if len(sys.argv) < 2:
print(json.dumps({'region': '', 'regionDetail': ''}))
sys.exit(0)
pdf_path = sys.argv[1]
title = sys.argv[2] if len(sys.argv) > 2 else ''
result = extract_address(pdf_path, title)
print(json.dumps(result, ensure_ascii=False))

View File

@ -107,7 +107,7 @@ def parse_list_page(html, type_label):
"detail_url": BASE_URL + a_tag.get("href", ""), "detail_url": BASE_URL + a_tag.get("href", ""),
"notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "", "notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "",
"type": type_label, "type": type_label,
"publish_date": "", "publish_time": "",
"deadline": "", "deadline": "",
"status": "", "status": "",
} }
@ -120,7 +120,7 @@ def parse_list_page(html, type_label):
if "Gray" in cls: if "Gray" in cls:
# 发布日期 # 发布日期
if re.match(r"\d{4}-\d{2}-\d{2}", text): if re.match(r"\d{4}-\d{2}-\d{2}", text):
notice["publish_date"] = text notice["publish_time"] = text
elif "Green" in cls: elif "Green" in cls:
# 正在报名 # 正在报名
notice["status"] = text.strip("[]") notice["status"] = text.strip("[]")
@ -132,7 +132,7 @@ def parse_list_page(html, type_label):
em_tag = span.find("em", class_="Red") em_tag = span.find("em", class_="Red")
if em_tag: if em_tag:
em_text = em_tag.get_text(strip=True) em_text = em_tag.get_text(strip=True)
dl_match = re.search(r"文件下载截止[::](.+)", em_text) dl_match = re.search(r"文件(?:发售|下载)截止[::](.+)", em_text)
if dl_match: if dl_match:
notice["deadline"] = dl_match.group(1).strip() notice["deadline"] = dl_match.group(1).strip()
@ -173,6 +173,39 @@ def save_state(state_path, state):
json.dump(state, f, ensure_ascii=False, indent=2) json.dump(state, f, ensure_ascii=False, indent=2)
def load_accumulated_state(output_base, state_filename, today_str):
"""加载历史状态实现跨天去重:合并所有历史状态文件中的已下载记录"""
accumulated = {}
last_run_date = None
state_dir = os.path.join(output_base, today_str)
today_state = load_state(os.path.join(state_dir, state_filename))
if today_state:
for k, v in today_state.items():
if k != "lastRun":
accumulated[k] = v
last_run = today_state.get("lastRun")
if last_run:
last_run_date = last_run[:10]
try:
for i in range(1, 31):
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
path = os.path.join(output_base, d, state_filename)
if os.path.exists(path):
try:
with open(path, "r", encoding="utf-8") as f:
prev = json.load(f)
except:
continue
for k, v in prev.items():
if k != "lastRun" and k not in accumulated:
accumulated[k] = v
if k == "lastRun" and not last_run_date:
last_run_date = str(v)[:10]
except:
pass
return accumulated, last_run_date
def generate_markdown(notices, output_dir, date_range, keyword=""): def generate_markdown(notices, output_dir, date_range, keyword=""):
"""生成公告清单 Markdown""" """生成公告清单 Markdown"""
lines = [ lines = [
@ -195,7 +228,7 @@ def generate_markdown(notices, output_dir, date_range, keyword=""):
deadline_display = n['deadline'] if n['deadline'] else '-' deadline_display = n['deadline'] if n['deadline'] else '-'
lines.extend([ lines.extend([
f"## {i}. {n['title']}", f"## {i}. {n['title']}",
f"- **发布日期**: {n['publish_date']}", f"- **发布日期**: {n['publish_time']}",
f"- **公告类型**: {n['type']}", f"- **公告类型**: {n['type']}",
f"- **状态**: {status_display}", f"- **状态**: {status_display}",
f"- **文件下载截止**: {deadline_display}", f"- **文件下载截止**: {deadline_display}",
@ -225,15 +258,21 @@ def run():
parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型") parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
parser.add_argument("--force", action="store_true", help="强制重新抓取") parser.add_argument("--force", action="store_true", help="强制重新抓取")
parser.add_argument("--keyword", default="", help="关键词过滤(匹配标题)") parser.add_argument("--keyword", default="", help="关键词过滤(匹配标题)")
parser.add_argument("--max-pages", type=int, default=0, help="最大翻页数(0=不限制)") parser.add_argument("--max-pages", type=int, default=10, help="最大翻页数(0=不限制,默认10页≈160条)")
args = parser.parse_args() args = parser.parse_args()
today = date.today() today = date.today()
today_str = today.strftime("%Y-%m-%d")
# 加载历史已下载记录(跨天去重)
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "cnncecp_downloaded.json", today_str)
if args.from_date: if args.from_date:
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date() date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
elif not args.force and last_run_date:
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
else: else:
date_from = today - timedelta(days=7) date_from = today
if args.to_date: if args.to_date:
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date() date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
@ -247,12 +286,12 @@ def run():
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
# 输出目录 # 输出目录
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d")) output_dir = os.path.join(OUTPUT_BASE, today_str)
os.makedirs(output_dir, exist_ok=True) os.makedirs(output_dir, exist_ok=True)
# 状态文件 # 状态文件(使用累积状态做去重)
state_file = os.path.join(output_dir, "cnncecp_downloaded.json") state_file = os.path.join(output_dir, "cnncecp_downloaded.json")
state = load_state(state_file) state = accumulated_state
all_notices = [] all_notices = []
new_count = 0 new_count = 0
@ -306,7 +345,7 @@ def run():
type_total += 1 type_total += 1
# 日期过滤 # 日期过滤
if not match_date_range(notice["publish_date"], date_from, date_to): if not match_date_range(notice.get("publish_time", ""), date_from, date_to):
type_filtered += 1 type_filtered += 1
filtered_count += 1 filtered_count += 1
continue continue
@ -329,7 +368,7 @@ def run():
state[notice_id] = { state[notice_id] = {
"title": notice["title"], "title": notice["title"],
"type": notice["type"], "type": notice["type"],
"publish_date": notice["publish_date"], "publish_time": notice["publish_time"],
"deadline": notice["deadline"], "deadline": notice["deadline"],
"status": notice["status"], "status": notice["status"],
"detail_url": notice["detail_url"], "detail_url": notice["detail_url"],
@ -352,7 +391,8 @@ def run():
print(f" ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}") print(f" ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}")
print() print()
# 保存状态 # 保存状态(含 lastRun 时间戳,用于下次增量爬取)
state["lastRun"] = datetime.now().isoformat()
save_state(state_file, state) save_state(state_file, state)
# 生成清单 # 生成清单

View File

@ -25,9 +25,14 @@ const http = require('http');
const crypto = require('crypto'); const crypto = require('crypto');
// ========== 配置区 ========== // ========== 配置区 ==========
const KEYWORD = '水电十一局';
// 支持命令行传入 --keyword(例如: node fetch-v1.2.js --keyword 电缆)
const args = process.argv.slice(2)
const keywordIdx = args.indexOf('--keyword')
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
// 公告类型:可按需增减。支持的类型值见 API 返回的 announcementType 字段 // 公告类型:可按需增减。支持的类型值见 API 返回的 announcementType 字段
const TYPES = ['招采公告', '资格预审公告', '采购预告', '中标结果公示']; const TYPES = ['招采公告'];
// 可选:留空则从上次抓取日期继续;有值则从指定日期开始 // 可选:留空则从上次抓取日期继续;有值则从指定日期开始
const START_DATE = ''; const START_DATE = '';
const END_DATE = new Date().toISOString().split('T')[0]; // 今天 const END_DATE = new Date().toISOString().split('T')[0]; // 今天
@ -220,6 +225,54 @@ async function runWithConcurrency(tasks, limit) {
return results; return results;
} }
// ---------- 地址提取 ----------
const PROVINCES = [
'北京', '天津', '上海', '重庆',
'河北', '山西', '辽宁', '吉林', '黑龙江',
'江苏', '浙江', '安徽', '福建', '江西', '山东',
'河南', '湖北', '湖南', '广东', '海南',
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
]
const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
function extractRegionFromTitle(title) {
if (!title) return ''
// 匹配 "XX省"、"XX市"(直辖市)、"XX自治区"
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
if (m) {
const region = m[1]
if (region.includes('新疆')) return '新疆'
if (region.includes('广西')) return '广西'
if (region.includes('内蒙古')) return '内蒙古'
if (region.includes('宁夏')) return '宁夏'
if (region.includes('西藏')) return '西藏'
if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
return ''
}
// 直接匹配省份关键词
for (const prov of SPECIAL_PROVS) {
if (title.includes(prov)) return prov
}
for (const prov of TWO_CHAR_PROVS) {
if (title.includes(prov)) return prov
}
return ''
}
function extractCityFromTitle(title) {
if (!title) return ''
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
if (m) {
const city = m[1]
if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
}
return ''
}
// ---------- 生成公告清单 ---------- // ---------- 生成公告清单 ----------
function buildAttachmentName(r, index) { function buildAttachmentName(r, index) {
@ -332,9 +385,11 @@ async function main() {
log(`共获取 ${stats.total} 条公告(分 ${page} 页)`); log(`共获取 ${stats.total} 条公告(分 ${page} 页)`);
// 过滤目标公告类型(由 TYPES 配置定义) // 过滤目标公告类型(由 TYPES 配置定义),且仅取货物类
const caigou = allAnnouncements.filter(r => TYPES.includes(r.announcementType)); const caigou = allAnnouncements.filter(r =>
log(`其中 ${TYPES.join('、')}: ${caigou.length} 条`); TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
);
log(`其中 ${TYPES.join('、')}(货物类): ${caigou.length} 条`);
// 生成公告清单(所有本次抓到的招采公告,无论是否已下载) // 生成公告清单(所有本次抓到的招采公告,无论是否已下载)
const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE); const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
@ -363,7 +418,7 @@ async function main() {
if (!picUrl) { if (!picUrl) {
stats.skipNoAttachment++; stats.skipNoAttachment++;
log(`跳过(无附件): ${r.title}`, 'WARN'); log(`跳过(无附件): ${r.title}`, 'WARN');
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', time: new Date().toISOString() }; state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
continue; continue;
} }
@ -376,15 +431,20 @@ async function main() {
tasks.push(async () => { tasks.push(async () => {
log(`下载 [${shortName}]: ${fname}`); log(`下载 [${shortName}]: ${fname}`);
// 从标题中提取省份(详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取)
const region = extractRegionFromTitle(r.title) || ''
const regionDetail = extractCityFromTitle(r.title) || ''
try { try {
await downloadFile(picUrl, fpath); await downloadFile(picUrl, fpath);
const size = fs.statSync(fpath).size; const size = fs.statSync(fpath).size;
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', path: `pdf/${fname}`, size, time: new Date().toISOString() }; state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
stats.success++; stats.success++;
log(` ✅ ${fname} (${size} bytes)`); log(` ✅ ${fname} (${size} bytes) region=${region || '-'} addr=${regionDetail || '-'}`);
} catch (err) { } catch (err) {
stats.failed++; stats.failed++;
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', failed: true, error: err.message, time: new Date().toISOString() }; state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
log(` ❌ ${fname}: ${err.message}`, 'ERROR'); log(` ❌ ${fname}: ${err.message}`, 'ERROR');
} }
}); });

View File

@ -150,6 +150,41 @@ def save_state(state_path, state):
json.dump(state, f, ensure_ascii=False, indent=2) json.dump(state, f, ensure_ascii=False, indent=2)
def load_accumulated_state(output_base, state_filename, today_str):
"""加载历史状态实现跨天去重:合并所有历史状态文件中的已下载记录"""
accumulated = {}
last_run_date = None
state_dir = os.path.join(output_base, today_str)
# 先加载今日状态
today_state = load_state(os.path.join(state_dir, state_filename))
if today_state:
for k, v in today_state.items():
if k != "lastRun":
accumulated[k] = v
last_run = today_state.get("lastRun")
if last_run:
last_run_date = last_run[:10]
# 扫描历史目录(最近 30 天),合并之前下载的记录
try:
for i in range(1, 31):
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
path = os.path.join(output_base, d, state_filename)
if os.path.exists(path):
try:
with open(path, "r", encoding="utf-8") as f:
prev = json.load(f)
except:
continue
for k, v in prev.items():
if k != "lastRun" and k not in accumulated:
accumulated[k] = v
if k == "lastRun" and not last_run_date:
last_run_date = str(v)[:10]
except:
pass
return accumulated, last_run_date
def generate_markdown(notices, output_dir, date_range): def generate_markdown(notices, output_dir, date_range):
"""生成公告清单 Markdown""" """生成公告清单 Markdown"""
lines = [ lines = [
@ -195,10 +230,16 @@ def run():
today = date.today() today = date.today()
# 加载历史已下载记录(跨天去重)
today_str = today.strftime("%Y-%m-%d")
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
if args.from_date: if args.from_date:
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date() date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
elif not args.force and last_run_date:
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
else: else:
date_from = today - timedelta(days=7) # 默认抓取最近 7 天 date_from = today - timedelta(days=7)
if args.to_date: if args.to_date:
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date() date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
@ -212,7 +253,7 @@ def run():
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
# 输出目录 # 输出目录
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d")) output_dir = os.path.join(OUTPUT_BASE, today_str)
os.makedirs(output_dir, exist_ok=True) os.makedirs(output_dir, exist_ok=True)
# 子目录:html(源码) + pdf(转存) # 子目录:html(源码) + pdf(转存)
@ -221,9 +262,9 @@ def run():
os.makedirs(html_dir, exist_ok=True) os.makedirs(html_dir, exist_ok=True)
os.makedirs(pdf_dir, exist_ok=True) os.makedirs(pdf_dir, exist_ok=True)
# 状态文件 # 状态文件(使用累积状态做去重)
state_file = os.path.join(output_dir, "xkgroup_downloaded.json") state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
state = load_state(state_file) state = accumulated_state
all_notices = [] all_notices = []
new_count = 0 new_count = 0
@ -268,6 +309,8 @@ def run():
type_skip += 1 type_skip += 1
continue continue
# 从标题/组织名初步提取地区(详情页提取时会补充/覆盖)
title_region = extract_region_from_text(notice["title"], notice["org"])
all_notices.append(notice) all_notices.append(notice)
state[notice_id] = { state[notice_id] = {
"title": notice["title"], "title": notice["title"],
@ -276,6 +319,7 @@ def run():
"type": notice["type"], "type": notice["type"],
"detail_url": notice["detail_url"], "detail_url": notice["detail_url"],
"downloaded_at": datetime.now().isoformat(), "downloaded_at": datetime.now().isoformat(),
"region": title_region,
} }
new_count += 1 new_count += 1
type_new += 1 type_new += 1
@ -283,7 +327,8 @@ def run():
print(f" 新增: {type_new} | 跳过: {type_skip}") print(f" 新增: {type_new} | 跳过: {type_skip}")
print() print()
# 保存状态 # 保存状态(含 lastRun 时间戳,用于下次增量爬取)
state["lastRun"] = datetime.now().isoformat()
save_state(state_file, state) save_state(state_file, state)
# 为缺少金额的记录从详情页提取 # 为缺少金额的记录从详情页提取
@ -296,7 +341,9 @@ def run():
# 详情页转存 PDF + HTML(新公告 + 已有状态中缺文件的) # 详情页转存 PDF + HTML(新公告 + 已有状态中缺文件的)
all_pending = list(all_notices) all_pending = list(all_notices)
if not args.force: if not args.force:
for notice_id, info in state.items(): for notice_id, info in list(state.items()):
if not isinstance(info, dict):
continue
pdf_key = info.get("pdf", "") pdf_key = info.get("pdf", "")
html_key = info.get("html", "") html_key = info.get("html", "")
pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key)) pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
@ -323,6 +370,129 @@ def run():
print(f"📁 保存目录: {output_dir}") print(f"📁 保存目录: {output_dir}")
def extract_deadline_from_text(text):
"""从公告正文中提取截止日期(采购文件获取截止时间)"""
if not text:
return ""
# 模式1: "起至2026年6月22日19:00(北京时间)止" → 取截止日期
m = re.search(r"起至(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2}):(\d{2})", text)
if m:
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
# 模式2: "获取时间:...至2026-06-22 19:00"
m = re.search(r"至\s*(\d{4})-(\d{1,2})-(\d{1,2})\s+\d{1,2}:\d{2}", text)
if m:
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
# 模式3: "截止时间:2026年6月22日" 或 "截止日期:2026-06-22"
for pat in [
r"截止[时间日期]*[::]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
r"截止[时间日期]*[::]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
r"递交[文件]*截止[时间日期]*[::]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
r"递交[文件]*截止[时间日期]*[::]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
]:
m = re.search(pat, text)
if m:
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
return ""
# 全国省份/直辖市/自治区
PROVINCES = [
'北京', '天津', '上海', '重庆',
'河北', '山西', '辽宁', '吉林', '黑龙江',
'江苏', '浙江', '安徽', '福建', '江西', '山东',
'河南', '湖北', '湖南', '广东', '海南',
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
]
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
def extract_province_best(text):
"""从任意文本中提取最可能的省份(先关键词匹配,避免正则在多字省份上截断)"""
if not text:
return ''
text_clean = text.replace(' ', '')
# 1. 特殊多字省份(优先,避免"新疆维吾尔自治区"被截)
for sp in SPECIAL_PROVS:
if sp in text_clean:
return sp
# 2. 直辖市
for dc in ['北京', '天津', '上海', '重庆']:
if dc in text_clean:
return dc
# 3. "XX省"(如"陕西省"、"广东省")
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
if m:
return m.group(1)[:2]
# 4. 两字省份关键词
for p in TWO_CHAR_PROVS:
if p in text_clean:
return p
return ''
def extract_city_from_text(text):
"""从文本中提取城市名(XX市)"""
if not text:
return ''
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
if m:
city = m.group(1)
if city not in ['北京市', '天津市', '上海市', '重庆市']:
return city
return ''
def extract_region_from_text(text, org=""):
"""从公告正文或公司名中提取地区(省份)"""
if not text and not org:
return ""
# 1. 从正文找项目/交货/施工地点
if text:
for pat in [
r"(?:项目|交货|送货|施工|到货|供货|工程|实施|服务)[地址地点]*[::]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
r"(?:地[址点]|所在地区)[::]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
r"位\s*于\s*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
r"(?:项目|工程)\s*(?:位于|所在地|所在)[:::\s]*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
]:
m = re.search(pat, text)
if m:
region = m.group(1)
for sp in SPECIAL_PROVS:
if sp in region:
return sp
return region[:2] # 取前两个字
# 2. 从全文扫描省份关键词
prov = extract_province_best(text)
if prov:
return prov
# 3. 从公司名推断
if org:
org_prov = extract_province_best(org)
if org_prov:
return org_prov
return ""
def extract_region_detail_from_text(text):
"""从公告正文中提取详细收货/交货地址"""
if not text:
return ""
for pat in [
r"(?:交货|收货|送货|项目|施工|到货|供货|工程|实施|服务)[地址地点]*[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
r"(?:地[址点])[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
r"建设地点[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
r"项目地点[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
]:
m = re.search(pat, text)
if m:
addr = m.group(1).strip()
if len(addr) >= 6:
return addr
return ""
def extract_amount_from_text(text): def extract_amount_from_text(text):
"""从公告正文中提取金额(万元)""" """从公告正文中提取金额(万元)"""
if not text: if not text:
@ -367,6 +537,8 @@ def extract_amounts_for_existing(state, output_dir):
for notice_id, info in state.items(): for notice_id, info in state.items():
if notice_id == "lastRun": if notice_id == "lastRun":
continue continue
if not isinstance(info, dict):
continue
if info.get("amount"): if info.get("amount"):
continue continue
pending.append(notice_id) pending.append(notice_id)
@ -396,12 +568,27 @@ def extract_amounts_for_existing(state, output_dir):
await page.close() await page.close()
amount = extract_amount_from_text(text) amount = extract_amount_from_text(text)
deadline = extract_deadline_from_text(text)
region = extract_region_from_text(text, info.get("org", ""))
region_detail = extract_region_detail_from_text(text)
updated = False
if amount: if amount:
state[notice_id]["amount"] = amount state[notice_id]["amount"] = amount
updated = True
if deadline:
state[notice_id]["deadline"] = deadline
updated = True
if region:
state[notice_id]["region"] = region
updated = True
if region_detail:
state[notice_id]["region_detail"] = region_detail
updated = True
if updated:
success += 1 success += 1
print(f"✅ {amount}") print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
else: else:
print("⏭️ 未找到金额") print("⏭️ 未找到金额/截止日期/地区")
except Exception as e: except Exception as e:
print(f"❌ {e}") print(f"❌ {e}")
@ -472,6 +659,15 @@ def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file
amount = extract_amount_from_text(page_text) amount = extract_amount_from_text(page_text)
if amount and state is not None and notice_id in state: if amount and state is not None and notice_id in state:
state[notice_id]["amount"] = amount state[notice_id]["amount"] = amount
deadline = extract_deadline_from_text(page_text)
if deadline and state is not None and notice_id in state:
state[notice_id]["deadline"] = deadline
region = extract_region_from_text(page_text, notice.get("org", ""))
if region and state is not None and notice_id in state:
state[notice_id]["region"] = region
region_detail = extract_region_detail_from_text(page_text)
if region_detail and state is not None and notice_id in state:
state[notice_id]["region_detail"] = region_detail
# 保存 HTML 源码到 html/ 目录(供 AI 阅读) # 保存 HTML 源码到 html/ 目录(供 AI 阅读)
html_source = await page.content() html_source = await page.content()