Compare commits
6 Commits
672a4b41b2
...
8d8945f92b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d8945f92b | ||
|
|
44609be8c7 | ||
|
|
0a423af4d2 | ||
|
|
ae72966766 | ||
|
|
bd213eb5ec | ||
|
|
49f922444a |
3
.gitignore
vendored
3
.gitignore
vendored
@ -41,6 +41,9 @@ pnpm-debug.log*
|
|||||||
# OS
|
# OS
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
|
|
||||||
|
# Uploads (不上传图片文件)
|
||||||
|
apps/api/public/uploads/
|
||||||
|
|
||||||
# Backup files (防误提交)
|
# Backup files (防误提交)
|
||||||
*.bak
|
*.bak
|
||||||
*.tmp
|
*.tmp
|
||||||
|
|||||||
42
AGENTS.md
42
AGENTS.md
@ -2,6 +2,48 @@
|
|||||||
|
|
||||||
语言:中文。所有对话、注释、文档、提交信息均使用中文。
|
语言:中文。所有对话、注释、文档、提交信息均使用中文。
|
||||||
|
|
||||||
|
## 🤝 协作流程标准(避免理解不一致造成返工)
|
||||||
|
|
||||||
|
**所有具体问题的修改,必须按这个流程执行:**
|
||||||
|
|
||||||
|
```
|
||||||
|
1. 用户描述需求(业务场景、看到的问题、截图、日志)
|
||||||
|
↓
|
||||||
|
2. 协作者反问 7 个技术维度(场景/触发/重要程度/数据/边界/样式/部署)
|
||||||
|
↓
|
||||||
|
3. 对齐范围:用户确认 + 协作者列技术方案
|
||||||
|
↓
|
||||||
|
4. 实施 + 验证
|
||||||
|
↓
|
||||||
|
5. 用户看效果,反馈或继续下一项
|
||||||
|
```
|
||||||
|
|
||||||
|
### 协作者必须问的 7 个技术维度
|
||||||
|
|
||||||
|
| 维度 | 问什么 |
|
||||||
|
|------|--------|
|
||||||
|
| **场景** | 哪个页面/哪个模块?用户路径是什么? |
|
||||||
|
| **触发条件** | 什么时候出现?手动还是自动?频率? |
|
||||||
|
| **重要程度** | 阻塞核心流程 / 体验优化 / 锦上添花? |
|
||||||
|
| **数据** | 影响多少数据?要不要数据迁移? |
|
||||||
|
| **边界** | 空数据/异常情况怎么处理? |
|
||||||
|
| **样式** | 改动大吗?要不要出截图对比? |
|
||||||
|
| **生产部署** | 改完要不要立即部署?还是攒一起? |
|
||||||
|
|
||||||
|
### 原则
|
||||||
|
|
||||||
|
- **不擅自开干**:未对齐范围前不做实现
|
||||||
|
- **不预设方案**:用户描述 ≠ 协作者的解决方案,先问清楚
|
||||||
|
- **不省略验证**:改完必须跑端到端测试,不只是"代码看起来对"
|
||||||
|
- **不静默改大动作**:涉及数据库结构、生产配置、依赖升级等要明确告知
|
||||||
|
|
||||||
|
### 例外(不需要问的简单场景)
|
||||||
|
|
||||||
|
- 拼写错误、明显笔误
|
||||||
|
- 单文件 typo 修复
|
||||||
|
- 用户明确说"按你判断做"
|
||||||
|
- 紧急 bug 修复(事后补问)
|
||||||
|
|
||||||
## 项目结构
|
## 项目结构
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
@ -18,6 +18,8 @@ interface FetchResult {
|
|||||||
noticeId: string
|
noticeId: string
|
||||||
detailUrl: string
|
detailUrl: string
|
||||||
amount?: string
|
amount?: string
|
||||||
|
region?: string
|
||||||
|
regionDetail?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
@ -109,7 +111,7 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
let results: FetchResult[] = []
|
let results: FetchResult[] = []
|
||||||
|
|
||||||
if (config.type === 'bid_powerchina') {
|
if (config.type === 'bid_powerchina') {
|
||||||
results = await this.runPowerChina()
|
results = await this.runPowerChina(config)
|
||||||
} else {
|
} else {
|
||||||
results = await this.runGenericScript(config)
|
results = await this.runGenericScript(config)
|
||||||
}
|
}
|
||||||
@ -129,6 +131,8 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
detailUrl: r.detailUrl,
|
detailUrl: r.detailUrl,
|
||||||
publishTime: r.publishTime || null,
|
publishTime: r.publishTime || null,
|
||||||
...(r.amount ? { amount: r.amount } : {}),
|
...(r.amount ? { amount: r.amount } : {}),
|
||||||
|
...(r.region ? { region: r.region } : {}),
|
||||||
|
...(r.regionDetail ? { regionDetail: r.regionDetail } : {}),
|
||||||
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
||||||
},
|
},
|
||||||
create: {
|
create: {
|
||||||
@ -143,7 +147,8 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
status: '待复核',
|
status: '待复核',
|
||||||
level: 'C',
|
level: 'C',
|
||||||
category: '',
|
category: '',
|
||||||
region: '',
|
region: r.region || '',
|
||||||
|
regionDetail: r.regionDetail || '',
|
||||||
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@ -172,49 +177,92 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async runPowerChina(): Promise<FetchResult[]> {
|
private async runPowerChina(config: { keywords?: string | null }): Promise<FetchResult[]> {
|
||||||
const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js')
|
const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js')
|
||||||
if (!fs.existsSync(scriptPath)) {
|
if (!fs.existsSync(scriptPath)) {
|
||||||
throw new Error('中电建抓取脚本不存在')
|
throw new Error('中电建抓取脚本不存在')
|
||||||
}
|
}
|
||||||
|
|
||||||
execSync(`node "${scriptPath}"`, {
|
let keywords: string[]
|
||||||
|
try {
|
||||||
|
keywords = config.keywords ? JSON.parse(config.keywords) : []
|
||||||
|
} catch {
|
||||||
|
keywords = []
|
||||||
|
}
|
||||||
|
if (keywords.length === 0) keywords = ['']
|
||||||
|
|
||||||
|
const today = new Date().toISOString().split('T')[0]
|
||||||
|
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
|
||||||
|
const seen = new Set<string>()
|
||||||
|
const allResults: FetchResult[] = []
|
||||||
|
|
||||||
|
for (const kw of keywords) {
|
||||||
|
const kwFlag = kw ? ` --keyword "${kw}"` : ''
|
||||||
|
try {
|
||||||
|
execSync(`node "${scriptPath}"${kwFlag}`, {
|
||||||
encoding: 'utf-8',
|
encoding: 'utf-8',
|
||||||
maxBuffer: 10 * 1024 * 1024,
|
maxBuffer: 10 * 1024 * 1024,
|
||||||
timeout: 180000,
|
timeout: 180000,
|
||||||
})
|
})
|
||||||
|
} catch {
|
||||||
|
// 单个 keyword 失败不影响其他 keyword
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
const today = new Date().toISOString().split('T')[0]
|
const statePath = path.join(baseDir, 'downloaded.json')
|
||||||
const statePath = path.join(
|
if (!fs.existsSync(statePath)) continue
|
||||||
process.env.HOME || '',
|
|
||||||
'desktop/招标公告',
|
|
||||||
today,
|
|
||||||
'downloaded.json',
|
|
||||||
)
|
|
||||||
|
|
||||||
if (!fs.existsSync(statePath)) return []
|
|
||||||
|
|
||||||
|
try {
|
||||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||||
const downloaded = state.downloaded || {}
|
const downloaded = state.downloaded || {}
|
||||||
const results: FetchResult[] = []
|
|
||||||
|
const outputDir = path.dirname(statePath)
|
||||||
|
const pdfDir = path.join(outputDir, 'pdf')
|
||||||
|
|
||||||
for (const [key, v] of Object.entries(downloaded)) {
|
for (const [key, v] of Object.entries(downloaded)) {
|
||||||
const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string }
|
const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string; publishTime?: string; path?: string }
|
||||||
if (info.skipped || info.failed) continue
|
if (info.skipped || info.failed) continue
|
||||||
const [systemId] = key.split('_')
|
const [systemId] = key.split('_')
|
||||||
|
if (seen.has(systemId)) continue
|
||||||
|
seen.add(systemId)
|
||||||
|
|
||||||
const realId = info.id || systemId
|
const realId = info.id || systemId
|
||||||
results.push({
|
|
||||||
|
let region = ''
|
||||||
|
let regionDetail = ''
|
||||||
|
if (info.path && fs.existsSync(pdfDir)) {
|
||||||
|
const pdfPath = path.join(pdfDir, path.basename(info.path))
|
||||||
|
if (fs.existsSync(pdfPath)) {
|
||||||
|
try {
|
||||||
|
const extractor = path.join(SCRIPTS_DIR, 'extract_address_from_pdf.py')
|
||||||
|
const title = info.title || ''
|
||||||
|
const cmd = `python3 "${extractor}" "${pdfPath}" "${title}"`
|
||||||
|
const out = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim()
|
||||||
|
const parsed = JSON.parse(out)
|
||||||
|
region = parsed.region || ''
|
||||||
|
regionDetail = parsed.regionDetail || ''
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
allResults.push({
|
||||||
source: '中电建',
|
source: '中电建',
|
||||||
title: info.title || '',
|
title: info.title || '',
|
||||||
organization: '中电建',
|
organization: '中电建',
|
||||||
publishTime: state.lastRun || '',
|
publishTime: info.publishTime || state.lastRun || '',
|
||||||
deadline: info.deadline || '',
|
deadline: info.deadline || '',
|
||||||
noticeId: systemId,
|
noticeId: systemId,
|
||||||
detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`,
|
detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`,
|
||||||
|
region,
|
||||||
|
regionDetail,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
} catch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return results
|
return allResults
|
||||||
}
|
}
|
||||||
|
|
||||||
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
|
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
|
||||||
@ -248,10 +296,13 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
|
|
||||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||||
const results: FetchResult[] = []
|
const results: FetchResult[] = []
|
||||||
|
const todayStr = new Date().toISOString().substring(0, 10)
|
||||||
|
|
||||||
for (const [noticeId, v] of Object.entries(state)) {
|
for (const [noticeId, v] of Object.entries(state)) {
|
||||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string }
|
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
|
||||||
if (noticeId === 'lastRun') continue
|
if (noticeId === 'lastRun') continue
|
||||||
|
// 只处理本次新增的记录(downloaded_at 包含今天日期),历史记录已入库无需重新处理
|
||||||
|
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
|
||||||
const leadAmount = info.amount || ''
|
const leadAmount = info.amount || ''
|
||||||
results.push({
|
results.push({
|
||||||
source,
|
source,
|
||||||
@ -262,6 +313,8 @@ export class CrawlerService implements OnModuleInit {
|
|||||||
noticeId: info.id || noticeId,
|
noticeId: info.id || noticeId,
|
||||||
detailUrl: info.detail_url || '',
|
detailUrl: info.detail_url || '',
|
||||||
amount: leadAmount,
|
amount: leadAmount,
|
||||||
|
region: info.region || '',
|
||||||
|
regionDetail: info.region_detail || '',
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -373,11 +373,11 @@ export class IntegrationService implements OnModuleDestroy {
|
|||||||
* 解析 MCP tool 返回的 content 字段
|
* 解析 MCP tool 返回的 content 字段
|
||||||
* MCP 协议格式: { content: [{ type: 'text', text: '...' }], isError?: boolean }
|
* MCP 协议格式: { content: [{ type: 'text', text: '...' }], isError?: boolean }
|
||||||
*/
|
*/
|
||||||
private parseToolResult<T>(result: any): T | null {
|
private parseToolResult<T>(result: any): T {
|
||||||
try {
|
if (!result) throw new Error('MCP 工具返回结果为空')
|
||||||
if (!result) return null
|
|
||||||
if (result.isError) {
|
if (result.isError) {
|
||||||
throw new Error(typeof result.content?.[0]?.text === 'string' ? result.content[0].text : 'Tool 调用返回错误')
|
const errMsg = typeof result.content?.[0]?.text === 'string' ? result.content[0].text : 'Tool 调用返回错误'
|
||||||
|
throw new Error(errMsg)
|
||||||
}
|
}
|
||||||
if (result.content && Array.isArray(result.content) && result.content.length > 0) {
|
if (result.content && Array.isArray(result.content) && result.content.length > 0) {
|
||||||
const item = result.content[0]
|
const item = result.content[0]
|
||||||
@ -390,10 +390,6 @@ export class IntegrationService implements OnModuleDestroy {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return result as T
|
return result as T
|
||||||
} catch (e: any) {
|
|
||||||
this.logger.error(`解析 MCP tool 结果失败: ${e.message}`)
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
getClient(integrationId: string): Client | null {
|
getClient(integrationId: string): Client | null {
|
||||||
|
|||||||
@ -14,8 +14,9 @@ export class LeadController {
|
|||||||
@Query('source') source?: string,
|
@Query('source') source?: string,
|
||||||
@Query('category') category?: string,
|
@Query('category') category?: string,
|
||||||
@Query('assignedTo') assignedTo?: string,
|
@Query('assignedTo') assignedTo?: string,
|
||||||
|
@Query('timeRange') timeRange?: string,
|
||||||
) {
|
) {
|
||||||
return this.leadService.findAll({ status, level, search, star, source, category, assignedTo })
|
return this.leadService.findAll({ status, level, search, star, source, category, assignedTo, timeRange })
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get('stats')
|
@Get('stats')
|
||||||
|
|||||||
@ -2,6 +2,25 @@ import { Injectable, NotFoundException, Logger, ServiceUnavailableException } fr
|
|||||||
import { PrismaService } from '../prisma/prisma.service'
|
import { PrismaService } from '../prisma/prisma.service'
|
||||||
import { IntegrationService } from '../integration/integration.service'
|
import { IntegrationService } from '../integration/integration.service'
|
||||||
|
|
||||||
|
// 品类 → 标题关键词映射(筛选时按标题模糊匹配,不走 category 字段)
|
||||||
|
const CATEGORY_KEYWORDS: Record<string, string[]> = {
|
||||||
|
'防火门': ['防火门', '防火卷帘', '防火窗', '钢质防火', '木质防火'],
|
||||||
|
'配电箱': ['配电箱', '配电柜', '配电盘', '照明配电箱', '动力配电箱', '电表箱'],
|
||||||
|
'电线电缆': ['电线', '电缆', '导线', '电力线', '控制电缆', '光缆', '架空线'],
|
||||||
|
'阀门': ['阀门', '闸阀', '球阀', '蝶阀', '截止阀', '止回阀', '调节阀'],
|
||||||
|
'钢材': ['钢材', '钢管', '钢板', '钢筋', '型钢', '槽钢', '角钢', '工字钢', 'H型钢', '螺纹钢'],
|
||||||
|
'涂料': ['涂料', '油漆', '防腐漆', '防火涂料', '乳胶漆', '防水涂料'],
|
||||||
|
'混凝土': ['混凝土', '商混', '商品砼', '水泥', '砂浆', '砼'],
|
||||||
|
'变压器': ['变压器', '箱变', '干式变', '油浸变', '电力变压器'],
|
||||||
|
'水泵': ['水泵', '离心泵', '潜污泵', '消防泵', '排污泵', '循环泵'],
|
||||||
|
'风机': ['风机', '通风机', '排风机', '轴流风机', '离心风机'],
|
||||||
|
'管材': ['管材', '管道', 'PVC管', 'PE管', '镀锌管', '无缝管', '螺旋管', '给水管'],
|
||||||
|
'开关': ['开关', '断路器', '接触器', '继电器', '熔断器', '隔离开关'],
|
||||||
|
'灯具': ['灯具', '照明灯', '路灯', '防爆灯', 'LED灯', '太阳能灯'],
|
||||||
|
'仪表': ['仪表', '仪器', '传感器', '变送器', '流量计', '压力表', '温度计'],
|
||||||
|
'电机': ['电机', '电动机', '减速机', '减速器', '变频电机'],
|
||||||
|
}
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class LeadService {
|
export class LeadService {
|
||||||
private readonly logger = new Logger(LeadService.name)
|
private readonly logger = new Logger(LeadService.name)
|
||||||
@ -11,7 +30,7 @@ export class LeadService {
|
|||||||
private integrationService: IntegrationService,
|
private integrationService: IntegrationService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string }) {
|
async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string; timeRange?: string }) {
|
||||||
const where: any = {}
|
const where: any = {}
|
||||||
if (params?.status) where.status = params.status
|
if (params?.status) where.status = params.status
|
||||||
if (params?.level) where.level = params.level
|
if (params?.level) where.level = params.level
|
||||||
@ -28,14 +47,44 @@ export class LeadService {
|
|||||||
else if (!isNaN(starVal)) where.star = starVal
|
else if (!isNaN(starVal)) where.star = starVal
|
||||||
}
|
}
|
||||||
if (params?.source) where.source = params.source
|
if (params?.source) where.source = params.source
|
||||||
if (params?.category) where.category = params.category
|
if (params?.category) {
|
||||||
|
const keywords = CATEGORY_KEYWORDS[params.category]
|
||||||
|
if (keywords) {
|
||||||
|
where.OR = keywords.map(kw => ({ title: { contains: kw } }))
|
||||||
|
}
|
||||||
|
}
|
||||||
if (params?.assignedTo) {
|
if (params?.assignedTo) {
|
||||||
where.assignedTo = params.assignedTo === '未分配' ? null : params.assignedTo
|
where.assignedTo = params.assignedTo === '未分配' ? null : params.assignedTo
|
||||||
}
|
}
|
||||||
|
if (params?.timeRange && params.timeRange !== '全部时间') {
|
||||||
|
const now = new Date()
|
||||||
|
const pad = (n: number) => n.toString().padStart(2, '0')
|
||||||
|
const today = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`
|
||||||
|
if (params.timeRange === '今天') {
|
||||||
|
where.publishTime = { startsWith: today }
|
||||||
|
} else if (params.timeRange === '昨天') {
|
||||||
|
const y = new Date(now)
|
||||||
|
y.setDate(y.getDate() - 1)
|
||||||
|
const yesterday = `${y.getFullYear()}-${pad(y.getMonth() + 1)}-${pad(y.getDate())}`
|
||||||
|
where.publishTime = { startsWith: yesterday }
|
||||||
|
} else if (params.timeRange === '本周') {
|
||||||
|
const day = now.getDay()
|
||||||
|
const diff = day === 0 ? 6 : day - 1 // Monday
|
||||||
|
const monday = new Date(now)
|
||||||
|
monday.setDate(now.getDate() - diff)
|
||||||
|
const sunday = new Date(monday)
|
||||||
|
sunday.setDate(monday.getDate() + 6)
|
||||||
|
const m = `${monday.getFullYear()}-${pad(monday.getMonth() + 1)}-${pad(monday.getDate())}`
|
||||||
|
const s = `${sunday.getFullYear()}-${pad(sunday.getMonth() + 1)}-${pad(sunday.getDate())}`
|
||||||
|
where.publishTime = { gte: m, lte: s + ' 23:59:59' }
|
||||||
|
} else if (params.timeRange === '本月') {
|
||||||
|
where.publishTime = { startsWith: `${now.getFullYear()}-${pad(now.getMonth() + 1)}` }
|
||||||
|
}
|
||||||
|
}
|
||||||
return this.prisma.lead.findMany({
|
return this.prisma.lead.findMany({
|
||||||
where,
|
where,
|
||||||
orderBy: [
|
orderBy: [
|
||||||
{ level: 'asc' },
|
{ deadline: 'desc' },
|
||||||
{ createdAt: 'desc' },
|
{ createdAt: 'desc' },
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|||||||
@ -40,14 +40,6 @@ export class ProductController {
|
|||||||
return this.productService.batchPublish(body.ids)
|
return this.productService.batchPublish(body.ids)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* F002 P0-A: 批量上架商品到 B2B 平台 (MCP import_products)
|
|
||||||
*/
|
|
||||||
@Post('publish-to-direv')
|
|
||||||
publishToDirev(@Body() body: { ids: string[] }) {
|
|
||||||
return this.productService.publishToDirev(body.ids)
|
|
||||||
}
|
|
||||||
|
|
||||||
@Delete(':id')
|
@Delete(':id')
|
||||||
remove(@Param('id') id: string) {
|
remove(@Param('id') id: string) {
|
||||||
return this.productService.remove(id)
|
return this.productService.remove(id)
|
||||||
|
|||||||
@ -42,90 +42,87 @@ export class ProductService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async batchPublish(ids: string[]) {
|
async batchPublish(ids: string[]) {
|
||||||
return this.prisma.product.updateMany({
|
|
||||||
where: { id: { in: ids } },
|
|
||||||
data: { status: 'published' },
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async remove(id: string) {
|
|
||||||
return this.prisma.product.delete({ where: { id } })
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* F002 P0-A: 批量上架商品到 B2B 平台 (MCP import_products)
|
|
||||||
* 字段映射: Product → import_products schema
|
|
||||||
*/
|
|
||||||
async publishToDirev(ids: string[]) {
|
|
||||||
const b2bIntegration = await this.integrationService.findActiveMCP()
|
|
||||||
if (!b2bIntegration) {
|
|
||||||
throw new ServiceUnavailableException('未配置 B2B 集成')
|
|
||||||
}
|
|
||||||
|
|
||||||
const products = await this.prisma.product.findMany({
|
const products = await this.prisma.product.findMany({
|
||||||
where: { id: { in: ids } },
|
where: { id: { in: ids } },
|
||||||
})
|
})
|
||||||
|
if (products.length === 0) return { imported: 0, errors: [], ids: [] }
|
||||||
|
|
||||||
if (products.length === 0) {
|
const b2bIntegration = await this.integrationService.findActiveMCP()
|
||||||
throw new ServiceUnavailableException('未找到指定商品')
|
if (!b2bIntegration) {
|
||||||
|
throw new ServiceUnavailableException('未配置 B2B 平台 MCP 集成,请在系统配置中添加')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Category string → numeric ID mapping (B2B tb_good first_category_id)
|
|
||||||
const CATEGORY_MAP: Record<string, number> = {
|
const CATEGORY_MAP: Record<string, number> = {
|
||||||
'电缆': 28,
|
'电缆': 28, '电力电缆': 28, '配电箱': 29, '开关柜': 29,
|
||||||
'电力电缆': 28,
|
'变压器': 27, '电线': 28, '母线槽': 30, '桥架': 31,
|
||||||
'配电箱': 29,
|
|
||||||
'开关柜': 29,
|
|
||||||
'变压器': 27,
|
|
||||||
'电线': 28,
|
|
||||||
'母线槽': 30,
|
|
||||||
'桥架': 31,
|
|
||||||
}
|
}
|
||||||
const fallbackCategoryId = 1
|
const fallbackCategoryId = 1
|
||||||
|
|
||||||
const items = products.map(p => {
|
const mappedItems = products.map(p => {
|
||||||
const catId = CATEGORY_MAP[p.category] || fallbackCategoryId
|
const catId = CATEGORY_MAP[p.category] || fallbackCategoryId
|
||||||
const item: any = {
|
const item: any = {
|
||||||
name: p.name,
|
name: p.name, category_id: catId,
|
||||||
category_id: catId,
|
price: p.price > 0 ? p.price : 0.01,
|
||||||
price: p.price,
|
brand: p.brand || '',
|
||||||
brandName: p.brand || '',
|
unit: p.unit || '',
|
||||||
stock: p.quantity,
|
sku: p.spec || p.name || '',
|
||||||
}
|
}
|
||||||
if (p.spec) item.sku = p.spec
|
|
||||||
if (p.imageUrl) item.images = [p.imageUrl]
|
if (p.imageUrl) item.images = [p.imageUrl]
|
||||||
if (p.supplier) item.remark = `供应商:${p.supplier}`
|
if (p.supplier) item.remark = `供应商:${p.supplier}`
|
||||||
return item
|
return item
|
||||||
})
|
})
|
||||||
|
|
||||||
|
this.logger.log(`batchPublish: 发送 ${mappedItems.length} 个商品到 MCP import_products`)
|
||||||
const result = await this.integrationService.executeTool<any>(
|
const result = await this.integrationService.executeTool<any>(
|
||||||
b2bIntegration.id,
|
b2bIntegration.id, 'import_products',
|
||||||
'import_products',
|
{ products: mappedItems },
|
||||||
{ products: items },
|
|
||||||
{ retries: 2, retryDelayMs: 1000 },
|
{ retries: 2, retryDelayMs: 1000 },
|
||||||
)
|
)
|
||||||
|
|
||||||
|
this.logger.log(`batchPublish: MCP 返回 success=${result.success} data=${result.data ? JSON.stringify(result.data).slice(0, 300) : '(empty)'}`)
|
||||||
|
|
||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
const errorMsg = result.error || '未知错误'
|
throw new ServiceUnavailableException(`B2B 同步失败: ${result.error || '未知错误'}`)
|
||||||
this.logger.error(`商品上架失败 (ids=${ids.join(',')}): ${errorMsg}`)
|
|
||||||
throw new ServiceUnavailableException(`上架失败: ${errorMsg}`)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = result.data as any
|
// MCP import_products 返回格式: { code:0, msg:"成功", data: [{ status:"success"|"failed", data:{id:"xxx"}, sku, name, error? }] }
|
||||||
const payload = data.data || data
|
const items: Array<{ status: string; data?: any; sku?: string; name?: string; error?: string }> =
|
||||||
const imported = payload.imported || 0
|
(result.data as any)?.data || []
|
||||||
const errors = payload.errors || []
|
const imported = items.filter(i => i.status === 'success').length
|
||||||
this.logger.log(`商品上架成功: 导入 ${imported} 条, 失败 ${errors.length} 条`)
|
const b2bIds: string[] = items
|
||||||
|
.filter(i => i.status === 'success' && i.data?.id)
|
||||||
|
.map(i => String(i.data.id))
|
||||||
|
const errors: Array<{ sku: string; name: string; error: string }> = items
|
||||||
|
.filter(i => i.status === 'failed')
|
||||||
|
.map(i => ({ sku: i.sku || '', name: i.name || '', error: i.error || '未知错误' }))
|
||||||
|
if (errors.length > 0) {
|
||||||
|
this.logger.warn(`B2B import_products 导入失败详情: ${errors.map(e => `[${e.sku}] ${e.name}: ${e.error}`).join(' | ')}`)
|
||||||
|
}
|
||||||
|
this.logger.log(`B2B import_products 完成: 导入 ${imported} 条, 失败 ${errors.length} 条, B2B IDs: ${b2bIds.join(',')}`)
|
||||||
|
|
||||||
|
if (b2bIds.length > 0) {
|
||||||
|
this.logger.log(`batchPublish: 上架 ${b2bIds.length} 个商品到 B2B publish_products`)
|
||||||
|
const publishResult = await this.integrationService.executeTool<any>(
|
||||||
|
b2bIntegration.id, 'publish_products',
|
||||||
|
{ product_ids: b2bIds },
|
||||||
|
{ retries: 1, retryDelayMs: 1000 },
|
||||||
|
)
|
||||||
|
this.logger.log(`batchPublish: publish_products 返回 success=${publishResult.success}`)
|
||||||
|
if (!publishResult.success) {
|
||||||
|
throw new ServiceUnavailableException(`B2B 上架失败: ${publishResult.error || 'publish_products 返回失败'}`)
|
||||||
|
}
|
||||||
|
this.logger.log(`B2B 上架成功: ${b2bIds.length} 个商品`)
|
||||||
|
}
|
||||||
|
|
||||||
await this.prisma.product.updateMany({
|
await this.prisma.product.updateMany({
|
||||||
where: { id: { in: products.map(p => p.id) } },
|
where: { id: { in: ids } },
|
||||||
data: { status: 'published' },
|
data: { status: 'published' },
|
||||||
})
|
})
|
||||||
|
|
||||||
return {
|
return { imported, errors, ids: b2bIds }
|
||||||
imported,
|
}
|
||||||
errors,
|
|
||||||
ids: payload.ids || [],
|
async remove(id: string) {
|
||||||
}
|
return this.prisma.product.delete({ where: { id } })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1 +1 @@
|
|||||||
NEXT_PUBLIC_API_URL=
|
NEXT_PUBLIC_API_URL=/api
|
||||||
|
|||||||
@ -35,7 +35,7 @@ function mapLead(raw: any): Lead {
|
|||||||
id: raw.id,
|
id: raw.id,
|
||||||
title: raw.title,
|
title: raw.title,
|
||||||
star: raw.star || 0,
|
star: raw.star || 0,
|
||||||
level: raw.level || 'C',
|
level: raw.level ? raw.level + '级' : 'C级',
|
||||||
amount: raw.amount || '待确认',
|
amount: raw.amount || '待确认',
|
||||||
status: raw.status || '待复核',
|
status: raw.status || '待复核',
|
||||||
deadline: raw.deadline || '',
|
deadline: raw.deadline || '',
|
||||||
@ -58,9 +58,9 @@ function mapLead(raw: any): Lead {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const STATUS_OPTIONS = ['全部状态', '待复核', '待跟进', '跟进中', '已转询价', '已转协同', '已成交', '无效线索']
|
const STATUS_OPTIONS = ['全部状态', '待复核', '待跟进', '跟进中', '已转询价', '已转协同', '已成交', '无效线索']
|
||||||
const LEVEL_OPTIONS = ['全部级别', 'S级', 'A级', 'B级']
|
const LEVEL_OPTIONS = ['全部级别', 'S级', 'A级', 'B级', 'C级', 'D级']
|
||||||
const CAT_OPTIONS = ['全部品类', '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土']
|
const CAT_OPTIONS = ['全部品类', '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土', '变压器', '水泵', '风机', '管材', '开关', '灯具', '仪表', '电机']
|
||||||
const SOURCE_OPTIONS = ['全部来源', 'PowerChina 招采', '新矿集团', '发改委审批', '行业资讯']
|
const SOURCE_OPTIONS = ['全部来源', '中电建', '新矿集团', '中核集团']
|
||||||
const PERSON_OPTIONS = ['全部人员', '郑钊', '赵主任', '运营管理员', '未分配']
|
const PERSON_OPTIONS = ['全部人员', '郑钊', '赵主任', '运营管理员', '未分配']
|
||||||
|
|
||||||
function getStatusClass(status: string) {
|
function getStatusClass(status: string) {
|
||||||
@ -110,7 +110,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
|
|
||||||
const showToast = (msg: string) => { setToast(msg); setTimeout(() => setToast(''), 2500) }
|
const showToast = (msg: string) => { setToast(msg); setTimeout(() => setToast(''), 2500) }
|
||||||
|
|
||||||
const [fTime, setFTime] = useState('全部时间')
|
const [fTime, setFTime] = useState('全部日期')
|
||||||
const [fStar, setFStar] = useState('全部星级')
|
const [fStar, setFStar] = useState('全部星级')
|
||||||
const [fStatus, setFStatus] = useState('全部状态')
|
const [fStatus, setFStatus] = useState('全部状态')
|
||||||
const [fLevel, setFLevel] = useState('全部级别')
|
const [fLevel, setFLevel] = useState('全部级别')
|
||||||
@ -118,7 +118,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
const [fSource, setFSource] = useState('全部来源')
|
const [fSource, setFSource] = useState('全部来源')
|
||||||
const [fPerson, setFPerson] = useState('全部人员')
|
const [fPerson, setFPerson] = useState('全部人员')
|
||||||
|
|
||||||
const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string) => {
|
const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string, timeRange?: string) => {
|
||||||
setLoading(true)
|
setLoading(true)
|
||||||
const params = new URLSearchParams()
|
const params = new URLSearchParams()
|
||||||
if (star && star !== '全部星级') {
|
if (star && star !== '全部星级') {
|
||||||
@ -128,11 +128,12 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
else if (star === '2星及以下') params.set('star', '2星及以下')
|
else if (star === '2星及以下') params.set('star', '2星及以下')
|
||||||
}
|
}
|
||||||
if (status && status !== '全部状态') params.set('status', status)
|
if (status && status !== '全部状态') params.set('status', status)
|
||||||
if (level && level !== '全部级别') params.set('level', level)
|
if (level && level !== '全部级别') params.set('level', level.replace('级', ''))
|
||||||
if (cat && cat !== '全部品类') params.set('category', cat)
|
if (cat && cat !== '全部品类') params.set('category', cat)
|
||||||
if (source && source !== '全部来源') params.set('source', source)
|
if (source && source !== '全部来源') params.set('source', source)
|
||||||
if (person && person !== '全部人员' && person !== '未分配') params.set('assignedTo', person)
|
if (person && person !== '全部人员' && person !== '未分配') params.set('assignedTo', person)
|
||||||
if (person === '未分配') params.set('assignedTo', '未分配')
|
if (person === '未分配') params.set('assignedTo', '未分配')
|
||||||
|
if (timeRange && timeRange !== '全部日期') params.set('timeRange', timeRange)
|
||||||
try {
|
try {
|
||||||
const qs = params.toString()
|
const qs = params.toString()
|
||||||
const data: any[] = await fetch(`${API}${qs ? '?' + qs : ''}`).then(r => r.json())
|
const data: any[] = await fetch(`${API}${qs ? '?' + qs : ''}`).then(r => r.json())
|
||||||
@ -144,10 +145,10 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
setLoading(false)
|
setLoading(false)
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson])
|
useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime])
|
||||||
|
|
||||||
const refreshLeads = async () => {
|
const refreshLeads = async () => {
|
||||||
await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson)
|
await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
const totalPages = Math.ceil(filtered.length / pageSize)
|
const totalPages = Math.ceil(filtered.length / pageSize)
|
||||||
@ -392,7 +393,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
<div className="bg-white p-4 border border-border-subtle rounded-lg shadow-sm flex flex-wrap items-center gap-4">
|
<div className="bg-white p-4 border border-border-subtle rounded-lg shadow-sm flex flex-wrap items-center gap-4">
|
||||||
<div className="flex flex-wrap items-center gap-4">
|
<div className="flex flex-wrap items-center gap-4">
|
||||||
{[
|
{[
|
||||||
{ label: '时间范围', value: fTime, set: setFTime, opts: ['全部时间', '今天', '昨天', '本周', '本月'] },
|
{ label: '公告日期', value: fTime, set: setFTime, opts: ['全部日期', '今天', '昨天', '本周', '本月'] },
|
||||||
{ label: '星级', value: fStar, set: setFStar, opts: ['全部星级', '5星', '4星', '3星', '2星及以下'] },
|
{ label: '星级', value: fStar, set: setFStar, opts: ['全部星级', '5星', '4星', '3星', '2星及以下'] },
|
||||||
{ label: '状态', value: fStatus, set: setFStatus, opts: STATUS_OPTIONS },
|
{ label: '状态', value: fStatus, set: setFStatus, opts: STATUS_OPTIONS },
|
||||||
{ label: '甲方级别', value: fLevel, set: setFLevel, opts: LEVEL_OPTIONS },
|
{ label: '甲方级别', value: fLevel, set: setFLevel, opts: LEVEL_OPTIONS },
|
||||||
@ -445,7 +446,6 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">星级</th>
|
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">星级</th>
|
||||||
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">甲方级别</th>
|
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">甲方级别</th>
|
||||||
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase text-right whitespace-nowrap">金额(万)</th>
|
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase text-right whitespace-nowrap">金额(万)</th>
|
||||||
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">数量</th>
|
|
||||||
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">状态</th>
|
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">状态</th>
|
||||||
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">投标截止</th>
|
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">投标截止</th>
|
||||||
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">公告日期</th>
|
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">公告日期</th>
|
||||||
@ -454,9 +454,9 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
</thead>
|
</thead>
|
||||||
<tbody className="divide-y divide-border-subtle">
|
<tbody className="divide-y divide-border-subtle">
|
||||||
{loading ? (
|
{loading ? (
|
||||||
<tr><td colSpan={11} className="p-8 text-center text-sm text-on-surface-variant">加载中...</td></tr>
|
<tr><td colSpan={10} className="p-8 text-center text-sm text-on-surface-variant">加载中...</td></tr>
|
||||||
) : pageLeads.length === 0 ? (
|
) : pageLeads.length === 0 ? (
|
||||||
<tr><td colSpan={11} className="p-8 text-center text-sm text-on-surface-variant">暂无数据</td></tr>
|
<tr><td colSpan={10} className="p-8 text-center text-sm text-on-surface-variant">暂无数据</td></tr>
|
||||||
) : pageLeads.map(lead => {
|
) : pageLeads.map(lead => {
|
||||||
const stars = '★'.repeat(lead.star) + '☆'.repeat(5 - lead.star)
|
const stars = '★'.repeat(lead.star) + '☆'.repeat(5 - lead.star)
|
||||||
return (
|
return (
|
||||||
@ -476,7 +476,6 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
|||||||
<td className="p-4 text-right whitespace-nowrap">
|
<td className="p-4 text-right whitespace-nowrap">
|
||||||
<span className={lead.amount === '待确认' ? 'text-[10px] text-slate-400' : 'text-[11px] font-medium'}>{lead.amount}</span>
|
<span className={lead.amount === '待确认' ? 'text-[10px] text-slate-400' : 'text-[11px] font-medium'}>{lead.amount}</span>
|
||||||
</td>
|
</td>
|
||||||
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.quantity ? `${lead.quantity}${lead.unit || ''}` : '-'}</td>
|
|
||||||
<td className="p-4 whitespace-nowrap"><span className={`px-2 py-0.5 text-[10px] font-medium rounded-full ${getStatusClass(lead.status)}`}>{lead.status}</span></td>
|
<td className="p-4 whitespace-nowrap"><span className={`px-2 py-0.5 text-[10px] font-medium rounded-full ${getStatusClass(lead.status)}`}>{lead.status}</span></td>
|
||||||
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.deadline ? lead.deadline.slice(5, 10) : '-'}</td>
|
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.deadline ? lead.deadline.slice(5, 10) : '-'}</td>
|
||||||
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.publishTime ? lead.publishTime.slice(0, 10) : '-'}</td>
|
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.publishTime ? lead.publishTime.slice(0, 10) : '-'}</td>
|
||||||
|
|||||||
@ -172,21 +172,8 @@ export default function ProductPage() {
|
|||||||
const batchSync = async () => {
|
const batchSync = async () => {
|
||||||
if (selectedIds.size === 0) return
|
if (selectedIds.size === 0) return
|
||||||
showToast(`正在同步 ${selectedIds.size} 个商品...`)
|
showToast(`正在同步 ${selectedIds.size} 个商品...`)
|
||||||
await fetch(`${API}/batch-publish`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: { 'Content-Type': 'application/json' },
|
|
||||||
body: JSON.stringify({ ids: Array.from(selectedIds) }),
|
|
||||||
})
|
|
||||||
setSelectedIds(new Set())
|
|
||||||
await loadProducts()
|
|
||||||
showToast('批量同步完成')
|
|
||||||
}
|
|
||||||
|
|
||||||
const batchPublishToDirev = async () => {
|
|
||||||
if (selectedIds.size === 0) return
|
|
||||||
showToast(`正在上架 ${selectedIds.size} 个商品到 B2B 平台...`)
|
|
||||||
try {
|
try {
|
||||||
const res = await fetch(`${API}/publish-to-direv`, {
|
const res = await fetch(`${API}/batch-publish`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ ids: Array.from(selectedIds) }),
|
body: JSON.stringify({ ids: Array.from(selectedIds) }),
|
||||||
@ -195,9 +182,9 @@ export default function ProductPage() {
|
|||||||
if (!res.ok) throw new Error(data.message || data.error || '请求失败')
|
if (!res.ok) throw new Error(data.message || data.error || '请求失败')
|
||||||
setSelectedIds(new Set())
|
setSelectedIds(new Set())
|
||||||
await loadProducts()
|
await loadProducts()
|
||||||
showToast(`上架成功: ${data.imported || 0} 条${data.errors?.length ? `, 失败 ${data.errors.length} 条` : ''}`)
|
showToast(`同步成功: ${data.imported || 0} 条导入 B2B${data.errors?.length ? `, 失败 ${data.errors.length} 条` : ''}`)
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
showToast(`上架失败: ${e.message}`)
|
showToast(`同步失败: ${e.message}`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -472,11 +459,6 @@ export default function ProductPage() {
|
|||||||
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" /></svg>
|
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" /></svg>
|
||||||
批量同步
|
批量同步
|
||||||
</button>
|
</button>
|
||||||
<button disabled={selectedIds.size === 0} onClick={batchPublishToDirev}
|
|
||||||
className="px-3 py-1.5 text-[10px] font-medium bg-emerald-600 text-white rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:opacity-90">
|
|
||||||
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M5 13l4 4L19 7" /></svg>
|
|
||||||
上架 B2B
|
|
||||||
</button>
|
|
||||||
<button disabled={selectedIds.size === 0} onClick={batchGenerate}
|
<button disabled={selectedIds.size === 0} onClick={batchGenerate}
|
||||||
className="px-3 py-1.5 text-[10px] font-medium bg-white border border-purple-200 text-purple-700 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:bg-purple-50 transition-colors">
|
className="px-3 py-1.5 text-[10px] font-medium bg-white border border-purple-200 text-purple-700 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:bg-purple-50 transition-colors">
|
||||||
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" /></svg>
|
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" /></svg>
|
||||||
|
|||||||
@ -799,9 +799,13 @@ export default function RuleCenterPage() {
|
|||||||
<td className="p-3">
|
<td className="p-3">
|
||||||
<button
|
<button
|
||||||
onClick={() => toggleCrawlerStatus(c.id, c.status)}
|
onClick={() => toggleCrawlerStatus(c.id, c.status)}
|
||||||
className={`px-2 py-0.5 text-[10px] font-medium rounded-full border ${getStatusStyle(c.status)}`}
|
className={`px-3 py-1 text-[11px] font-semibold rounded-md transition-all cursor-pointer ${
|
||||||
|
c.status === 'active'
|
||||||
|
? 'bg-amber-50 text-amber-700 hover:bg-amber-100 active:bg-amber-200'
|
||||||
|
: 'bg-emerald-50 text-emerald-700 hover:bg-emerald-100 active:bg-emerald-200'
|
||||||
|
}`}
|
||||||
>
|
>
|
||||||
{{ active: '暂停', paused: '启动', error: '重启' }[c.status] || c.status}
|
{{ active: '点击暂停', paused: '点击启动', error: '重启' }[c.status] || c.status}
|
||||||
</button>
|
</button>
|
||||||
</td>
|
</td>
|
||||||
<td className="p-3 text-xs text-on-surface-variant">
|
<td className="p-3 text-xs text-on-surface-variant">
|
||||||
|
|||||||
@ -80,7 +80,7 @@ sshpass -p 'Jubianshuzi@2026' ssh ubuntu@172.81.241.109 "
|
|||||||
|
|
||||||
# 4. 验证
|
# 4. 验证
|
||||||
curl -s -o /dev/null -w 'API: %{http_code}\n' http://172.81.241.109:4000/api/health
|
curl -s -o /dev/null -w 'API: %{http_code}\n' http://172.81.241.109:4000/api/health
|
||||||
curl -s -o /dev/null -w 'Web: %{http_code}\n' https://zoupeng.cloud
|
curl -s -o /dev/null -w 'Web: %{http_code}\n' https://direv.cloud
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@ -105,6 +105,8 @@ curl -s -o /dev/null -w 'Web: %{http_code}\n' https://zoupeng.cloud
|
|||||||
| 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — |
|
| 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — |
|
||||||
| 2026-06-23 | **密钥再次重置 + 司库系统重发**:devbox 傍晚又把密钥重置(hash 前缀 `m3hRN → diPRva`,update_date 18:18:22)。用 `reset-secret` 端点再次自助重置拿到新密钥 `fcb53684845648e2a7fc1082f5245180cfa4eea1f5404176`。**用户反馈现象**:选了 `司库系统升级完善项目` 点一键发布,本地显示"发布成功"但 B2B 后台查不到。**根因**:MCP 鉴权失败后代码降级为本地假订单号(X+时间戳),让用户误以为成功。 | scripts/rotate-mcp-key.sh |
|
| 2026-06-23 | **密钥再次重置 + 司库系统重发**:devbox 傍晚又把密钥重置(hash 前缀 `m3hRN → diPRva`,update_date 18:18:22)。用 `reset-secret` 端点再次自助重置拿到新密钥 `fcb53684845648e2a7fc1082f5245180cfa4eea1f5404176`。**用户反馈现象**:选了 `司库系统升级完善项目` 点一键发布,本地显示"发布成功"但 B2B 后台查不到。**根因**:MCP 鉴权失败后代码降级为本地假订单号(X+时间戳),让用户误以为成功。 | scripts/rotate-mcp-key.sh |
|
||||||
| 2026-06-23 | **彻底移除降级本地模式 + MCP 失败弹警告**:(1) 后端 `publishToDirev` 不再写本地假 `inquiryNo=X...`;MCP 鉴权/调用失败时抛 `ServiceUnavailableException` 返回 503 + 详细错误信息;Lead 状态完全不动。(2) 前端 `doPublish` 收集每条失败原因(fetch `res.json().message`),失败时弹新 Modal(红框 + 错误列表 + 「去 MCP 配置」/「我知道了」按钮)。**端到端验证**:把密钥改成 `invalid_test_key_for_qa` → publish 返回 HTTP 503 + JSON 含 `appSecret 校验失败` + 引导文案;Lead 表 publish/inquiryNo/followLogs 全部不变。 | apps/api/src/lead/lead.service.ts, apps/web/src/components/InquiryPage.tsx |
|
| 2026-06-23 | **彻底移除降级本地模式 + MCP 失败弹警告**:(1) 后端 `publishToDirev` 不再写本地假 `inquiryNo=X...`;MCP 鉴权/调用失败时抛 `ServiceUnavailableException` 返回 503 + 详细错误信息;Lead 状态完全不动。(2) 前端 `doPublish` 收集每条失败原因(fetch `res.json().message`),失败时弹新 Modal(红框 + 错误列表 + 「去 MCP 配置」/「我知道了」按钮)。**端到端验证**:把密钥改成 `invalid_test_key_for_qa` → publish 返回 HTTP 503 + JSON 含 `appSecret 校验失败` + 引导文案;Lead 表 publish/inquiryNo/followLogs 全部不变。 | apps/api/src/lead/lead.service.ts, apps/web/src/components/InquiryPage.tsx |
|
||||||
|
| 2026-06-25 | **生产部署(commit 49f9224)**:本地全量构建后 rsync 到 `/opt/pingtai/`;生产跑 `npx prisma db push + generate`;生产 `next build`(有 lockfile 警告但构建成功);`pm2 restart pingtai-api pingtai-web --update-env`。**手工补齐生产 db 缺列**(prisma db push 路径错):真生产 db 是 `/data/pingtai/dev.db`(不是 `/opt/pingtai/apps/api/prisma/dev.db`),缺 `Lead.regionDetail/quantity/unit/attachments/notes/notesExtractedAt/notesSource` + `Integration.companyUserId` 共 8 列。手工 `ALTER TABLE` 全部补齐,95 条 Lead 数据完整保留。**生产验证**:API + Web online 13m+;lead 95 条、product 75 条、crawler 3 条、inquiry 3 条都正常返回。**新建 Integration 配置**(生产 db 之前为空):在生产创建 MCP 集成 id=751ab09c-e6ab-4c97-bd2d-2c317df32893,appId=app_9dd07cb4e0e24fccbecd。**遗留问题**:生产服务器 172.81.241.109 → devbox 100.111.84.16:3002 网络不通(ping 100% 丢包,curl 5s 超时)。MCP publish_rfq 端到端在生产跑不通(生产 connect 集成返回 `fetch failed`),publish 接口在生产会返回 503 + "未配置 B2B 集成"或"网络不通",这是网络层面的限制不是代码问题。 | apps/api/src/lead/lead.service.ts, docs/本地开发工作流.md |
|
||||||
|
| 2026-06-25 | **主域名切换:弃用 zoupeng.cloud,改用 direv.cloud**。**根因**:`zoupeng.cloud` DNS 解析到 `222.137.3.123`(旧/无效 IP,ping 通但 443 不通),而 `direv.cloud` DNS 正确解析到 `172.81.241.109`(实际生产服务器 IP)。**改动**:文档第 83 行 `https://zoupeng.cloud` → `https://direv.cloud`,rsync 同步到生产。**长期建议**:去腾讯云 DNSPod 控制台把 `zoupeng.cloud` 的 A 记录改到 `172.81.241.109`;生产 nginx 加 `zoupeng.cloud` 的 server block(同 direv.cloud),否则 `zoupeng.cloud` 永远打不开。AGENTS.md 里"主 `https://zoupeng.cloud`、备 `https://direv.cloud`"是错的,应反过来。 | docs/本地开发工作流.md |
|
||||||
| 2026-06-22 | **devbox F002 升级完成 + 平台助手二次适配**:devbox 已把 publish_rfq schema 升级(W29.7, 补 11 商务字段),Java 后端 `InquiryServiceImpl.publish()` 也补全字段映射。平台助手这边把新参数加进 `publishToDirev` 的 `publishArgs`:deadline(直接传 Lead.deadline YYYY-MM-DD)、deliveryDate(自动 deadline+30 天)、shippingAddress(优先 Lead.regionDetail, fallback region)、contactName/Phone(从 Lead.contact/phone)、inquiryParty("郑州巨变")、freightFlag("含运费")、procurementType("定量采购")、invoiceType("增值税专票")、paymentMethod("货到付款") | apps/api/src/lead/lead.service.ts |
|
| 2026-06-22 | **devbox F002 升级完成 + 平台助手二次适配**:devbox 已把 publish_rfq schema 升级(W29.7, 补 11 商务字段),Java 后端 `InquiryServiceImpl.publish()` 也补全字段映射。平台助手这边把新参数加进 `publishToDirev` 的 `publishArgs`:deadline(直接传 Lead.deadline YYYY-MM-DD)、deliveryDate(自动 deadline+30 天)、shippingAddress(优先 Lead.regionDetail, fallback region)、contactName/Phone(从 Lead.contact/phone)、inquiryParty("郑州巨变")、freightFlag("含运费")、procurementType("定量采购")、invoiceType("增值税专票")、paymentMethod("货到付款") | apps/api/src/lead/lead.service.ts |
|
||||||
| 2026-06-22 | **devbox 升级导致密钥重置**:`app_9dd07cb4e0e24fccbecd` 的 appSecret 被重置(MySQL 里 hash 前缀从 `$2a$10$iR3Rl` 变 `$2a$10$m3hRN`)。用 `POST /sys/openapi/app/reset-secret/{id}` 端点自助重置拿到新密钥 `d02dc00c6fbe4a34a5bf013134146c3f2759148c61a341d2`,本地集成密钥已轮换 | scripts/rotate-mcp-key.sh |
|
| 2026-06-22 | **devbox 升级导致密钥重置**:`app_9dd07cb4e0e24fccbecd` 的 appSecret 被重置(MySQL 里 hash 前缀从 `$2a$10$iR3Rl` 变 `$2a$10$m3hRN`)。用 `POST /sys/openapi/app/reset-secret/{id}` 端点自助重置拿到新密钥 `d02dc00c6fbe4a34a5bf013134146c3f2759148c61a341d2`,本地集成密钥已轮换 | scripts/rotate-mcp-key.sh |
|
||||||
| 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — |
|
| 2026-06-22 | **端到端验证新字段入库**:rfqId=2068990783236042754,orderId=RFQ20260622173337369,MySQL 里 11 个商务字段都正确填充(purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls)。**唯一未入库:`payment_method` 字段为 NULL**,原因:devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表,没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功(注意日志可能有 `付款方式查询失败: ...` 警告)| — |
|
||||||
|
|||||||
132
scripts/extract_address_from_pdf.py
Normal file
132
scripts/extract_address_from_pdf.py
Normal file
@ -0,0 +1,132 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
从招标公告 PDF 中提取地址信息。
|
||||||
|
用法: python3 extract_address_from_pdf.py <pdf文件路径> [公告标题]
|
||||||
|
输出: JSON {"region": "省/市", "regionDetail": "详细地址"}
|
||||||
|
"""
|
||||||
|
import fitz
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# 全国省份/直辖市/自治区名称(含简称、别称)
|
||||||
|
PROVINCES = [
|
||||||
|
'北京', '天津', '上海', '重庆',
|
||||||
|
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||||
|
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||||
|
'河南', '湖北', '湖南', '广东', '海南',
|
||||||
|
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||||
|
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||||
|
]
|
||||||
|
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
|
||||||
|
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||||
|
|
||||||
|
|
||||||
|
def extract_province_best(text):
|
||||||
|
"""从任意文本中提取最可能的省份(先关键词匹配,避免正则在多字省份上截断)"""
|
||||||
|
if not text:
|
||||||
|
return ''
|
||||||
|
text_clean = text.replace(' ', '')
|
||||||
|
# 1. 特殊多字省份(优先,避免"新疆维吾尔自治区"被截)
|
||||||
|
for prov in SPECIAL_PROVS:
|
||||||
|
if prov in text_clean:
|
||||||
|
return prov
|
||||||
|
# 2. 直辖市
|
||||||
|
for prov in ['北京', '天津', '上海', '重庆']:
|
||||||
|
if prov in text_clean:
|
||||||
|
return prov
|
||||||
|
# 3. "XX省"(如"陕西省"、"广东省")
|
||||||
|
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
|
||||||
|
if m:
|
||||||
|
return m.group(1)[:2]
|
||||||
|
# 4. 两字省份关键词
|
||||||
|
for prov in TWO_CHAR_PROVS:
|
||||||
|
if prov in text_clean:
|
||||||
|
return prov
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
def extract_city_from_text(text):
|
||||||
|
"""从文本中提取城市名(XX市)"""
|
||||||
|
if not text:
|
||||||
|
return ''
|
||||||
|
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
|
||||||
|
if m:
|
||||||
|
city = m.group(1)
|
||||||
|
if city not in ['北京市', '天津市', '上海市', '重庆市']:
|
||||||
|
return city
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
def extract_address(pdf_path: str, title: str = '') -> dict:
|
||||||
|
pdf_text = ''
|
||||||
|
try:
|
||||||
|
doc = fitz.open(pdf_path)
|
||||||
|
for page in doc:
|
||||||
|
pdf_text += page.get_text()
|
||||||
|
doc.close()
|
||||||
|
except Exception:
|
||||||
|
pdf_text = ''
|
||||||
|
|
||||||
|
result = {'region': '', 'regionDetail': ''}
|
||||||
|
|
||||||
|
# ── 1. 从PDF正文中提取详细地址 ──
|
||||||
|
addr_patterns = [
|
||||||
|
r'建设地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'项目地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'交货地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'施工地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'项目地址[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'供货地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'到货地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'送货地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'工程地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'实施地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'服务地点[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'收货地址[::]\s*(.+?)(?:\n|$)',
|
||||||
|
r'所在地[::]\s*(.+?)(?:\n|$)',
|
||||||
|
]
|
||||||
|
|
||||||
|
addr = ''
|
||||||
|
for p in addr_patterns:
|
||||||
|
m = re.search(p, pdf_text)
|
||||||
|
if m:
|
||||||
|
addr = m.group(1).strip()
|
||||||
|
if len(addr) >= 4:
|
||||||
|
break
|
||||||
|
addr = ''
|
||||||
|
|
||||||
|
if addr:
|
||||||
|
result['regionDetail'] = addr
|
||||||
|
prov = extract_province_best(addr)
|
||||||
|
if prov:
|
||||||
|
result['region'] = prov
|
||||||
|
else:
|
||||||
|
city = extract_city_from_text(addr)
|
||||||
|
if city:
|
||||||
|
result['region'] = city
|
||||||
|
|
||||||
|
# ── 2. 正文没提取到region,从整个PDF文本中搜索 ──
|
||||||
|
if not result['region'] and pdf_text:
|
||||||
|
prov = extract_province_best(pdf_text)
|
||||||
|
if prov:
|
||||||
|
result['region'] = prov
|
||||||
|
|
||||||
|
# ── 3. 还没提取到,从标题中搜索 ──
|
||||||
|
if not result['region'] and title:
|
||||||
|
prov = extract_province_best(title)
|
||||||
|
if prov:
|
||||||
|
result['region'] = prov
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print(json.dumps({'region': '', 'regionDetail': ''}))
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
pdf_path = sys.argv[1]
|
||||||
|
title = sys.argv[2] if len(sys.argv) > 2 else ''
|
||||||
|
result = extract_address(pdf_path, title)
|
||||||
|
print(json.dumps(result, ensure_ascii=False))
|
||||||
@ -107,7 +107,7 @@ def parse_list_page(html, type_label):
|
|||||||
"detail_url": BASE_URL + a_tag.get("href", ""),
|
"detail_url": BASE_URL + a_tag.get("href", ""),
|
||||||
"notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "",
|
"notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "",
|
||||||
"type": type_label,
|
"type": type_label,
|
||||||
"publish_date": "",
|
"publish_time": "",
|
||||||
"deadline": "",
|
"deadline": "",
|
||||||
"status": "",
|
"status": "",
|
||||||
}
|
}
|
||||||
@ -120,7 +120,7 @@ def parse_list_page(html, type_label):
|
|||||||
if "Gray" in cls:
|
if "Gray" in cls:
|
||||||
# 发布日期
|
# 发布日期
|
||||||
if re.match(r"\d{4}-\d{2}-\d{2}", text):
|
if re.match(r"\d{4}-\d{2}-\d{2}", text):
|
||||||
notice["publish_date"] = text
|
notice["publish_time"] = text
|
||||||
elif "Green" in cls:
|
elif "Green" in cls:
|
||||||
# 正在报名
|
# 正在报名
|
||||||
notice["status"] = text.strip("[]")
|
notice["status"] = text.strip("[]")
|
||||||
@ -132,7 +132,7 @@ def parse_list_page(html, type_label):
|
|||||||
em_tag = span.find("em", class_="Red")
|
em_tag = span.find("em", class_="Red")
|
||||||
if em_tag:
|
if em_tag:
|
||||||
em_text = em_tag.get_text(strip=True)
|
em_text = em_tag.get_text(strip=True)
|
||||||
dl_match = re.search(r"文件下载截止[::](.+)", em_text)
|
dl_match = re.search(r"文件(?:发售|下载)截止[::](.+)", em_text)
|
||||||
if dl_match:
|
if dl_match:
|
||||||
notice["deadline"] = dl_match.group(1).strip()
|
notice["deadline"] = dl_match.group(1).strip()
|
||||||
|
|
||||||
@ -173,6 +173,39 @@ def save_state(state_path, state):
|
|||||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
|
||||||
|
def load_accumulated_state(output_base, state_filename, today_str):
|
||||||
|
"""加载历史状态实现跨天去重:合并所有历史状态文件中的已下载记录"""
|
||||||
|
accumulated = {}
|
||||||
|
last_run_date = None
|
||||||
|
state_dir = os.path.join(output_base, today_str)
|
||||||
|
today_state = load_state(os.path.join(state_dir, state_filename))
|
||||||
|
if today_state:
|
||||||
|
for k, v in today_state.items():
|
||||||
|
if k != "lastRun":
|
||||||
|
accumulated[k] = v
|
||||||
|
last_run = today_state.get("lastRun")
|
||||||
|
if last_run:
|
||||||
|
last_run_date = last_run[:10]
|
||||||
|
try:
|
||||||
|
for i in range(1, 31):
|
||||||
|
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
|
path = os.path.join(output_base, d, state_filename)
|
||||||
|
if os.path.exists(path):
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
prev = json.load(f)
|
||||||
|
except:
|
||||||
|
continue
|
||||||
|
for k, v in prev.items():
|
||||||
|
if k != "lastRun" and k not in accumulated:
|
||||||
|
accumulated[k] = v
|
||||||
|
if k == "lastRun" and not last_run_date:
|
||||||
|
last_run_date = str(v)[:10]
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
return accumulated, last_run_date
|
||||||
|
|
||||||
|
|
||||||
def generate_markdown(notices, output_dir, date_range, keyword=""):
|
def generate_markdown(notices, output_dir, date_range, keyword=""):
|
||||||
"""生成公告清单 Markdown"""
|
"""生成公告清单 Markdown"""
|
||||||
lines = [
|
lines = [
|
||||||
@ -195,7 +228,7 @@ def generate_markdown(notices, output_dir, date_range, keyword=""):
|
|||||||
deadline_display = n['deadline'] if n['deadline'] else '-'
|
deadline_display = n['deadline'] if n['deadline'] else '-'
|
||||||
lines.extend([
|
lines.extend([
|
||||||
f"## {i}. {n['title']}",
|
f"## {i}. {n['title']}",
|
||||||
f"- **发布日期**: {n['publish_date']}",
|
f"- **发布日期**: {n['publish_time']}",
|
||||||
f"- **公告类型**: {n['type']}",
|
f"- **公告类型**: {n['type']}",
|
||||||
f"- **状态**: {status_display}",
|
f"- **状态**: {status_display}",
|
||||||
f"- **文件下载截止**: {deadline_display}",
|
f"- **文件下载截止**: {deadline_display}",
|
||||||
@ -225,15 +258,21 @@ def run():
|
|||||||
parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
|
parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
|
||||||
parser.add_argument("--force", action="store_true", help="强制重新抓取")
|
parser.add_argument("--force", action="store_true", help="强制重新抓取")
|
||||||
parser.add_argument("--keyword", default="", help="关键词过滤(匹配标题)")
|
parser.add_argument("--keyword", default="", help="关键词过滤(匹配标题)")
|
||||||
parser.add_argument("--max-pages", type=int, default=0, help="最大翻页数(0=不限制)")
|
parser.add_argument("--max-pages", type=int, default=10, help="最大翻页数(0=不限制,默认10页≈160条)")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
today = date.today()
|
today = date.today()
|
||||||
|
today_str = today.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
# 加载历史已下载记录(跨天去重)
|
||||||
|
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "cnncecp_downloaded.json", today_str)
|
||||||
|
|
||||||
if args.from_date:
|
if args.from_date:
|
||||||
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
||||||
|
elif not args.force and last_run_date:
|
||||||
|
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
|
||||||
else:
|
else:
|
||||||
date_from = today - timedelta(days=7)
|
date_from = today
|
||||||
|
|
||||||
if args.to_date:
|
if args.to_date:
|
||||||
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
||||||
@ -247,12 +286,12 @@ def run():
|
|||||||
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
||||||
|
|
||||||
# 输出目录
|
# 输出目录
|
||||||
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
|
output_dir = os.path.join(OUTPUT_BASE, today_str)
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
# 状态文件
|
# 状态文件(使用累积状态做去重)
|
||||||
state_file = os.path.join(output_dir, "cnncecp_downloaded.json")
|
state_file = os.path.join(output_dir, "cnncecp_downloaded.json")
|
||||||
state = load_state(state_file)
|
state = accumulated_state
|
||||||
|
|
||||||
all_notices = []
|
all_notices = []
|
||||||
new_count = 0
|
new_count = 0
|
||||||
@ -306,7 +345,7 @@ def run():
|
|||||||
type_total += 1
|
type_total += 1
|
||||||
|
|
||||||
# 日期过滤
|
# 日期过滤
|
||||||
if not match_date_range(notice["publish_date"], date_from, date_to):
|
if not match_date_range(notice.get("publish_time", ""), date_from, date_to):
|
||||||
type_filtered += 1
|
type_filtered += 1
|
||||||
filtered_count += 1
|
filtered_count += 1
|
||||||
continue
|
continue
|
||||||
@ -329,7 +368,7 @@ def run():
|
|||||||
state[notice_id] = {
|
state[notice_id] = {
|
||||||
"title": notice["title"],
|
"title": notice["title"],
|
||||||
"type": notice["type"],
|
"type": notice["type"],
|
||||||
"publish_date": notice["publish_date"],
|
"publish_time": notice["publish_time"],
|
||||||
"deadline": notice["deadline"],
|
"deadline": notice["deadline"],
|
||||||
"status": notice["status"],
|
"status": notice["status"],
|
||||||
"detail_url": notice["detail_url"],
|
"detail_url": notice["detail_url"],
|
||||||
@ -352,7 +391,8 @@ def run():
|
|||||||
print(f" ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}")
|
print(f" ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
# 保存状态
|
# 保存状态(含 lastRun 时间戳,用于下次增量爬取)
|
||||||
|
state["lastRun"] = datetime.now().isoformat()
|
||||||
save_state(state_file, state)
|
save_state(state_file, state)
|
||||||
|
|
||||||
# 生成清单
|
# 生成清单
|
||||||
|
|||||||
@ -25,9 +25,14 @@ const http = require('http');
|
|||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
|
|
||||||
// ========== 配置区 ==========
|
// ========== 配置区 ==========
|
||||||
const KEYWORD = '水电十一局';
|
|
||||||
|
// 支持命令行传入 --keyword(例如: node fetch-v1.2.js --keyword 电缆)
|
||||||
|
const args = process.argv.slice(2)
|
||||||
|
const keywordIdx = args.indexOf('--keyword')
|
||||||
|
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
|
||||||
|
|
||||||
// 公告类型:可按需增减。支持的类型值见 API 返回的 announcementType 字段
|
// 公告类型:可按需增减。支持的类型值见 API 返回的 announcementType 字段
|
||||||
const TYPES = ['招采公告', '资格预审公告', '采购预告', '中标结果公示'];
|
const TYPES = ['招采公告'];
|
||||||
// 可选:留空则从上次抓取日期继续;有值则从指定日期开始
|
// 可选:留空则从上次抓取日期继续;有值则从指定日期开始
|
||||||
const START_DATE = '';
|
const START_DATE = '';
|
||||||
const END_DATE = new Date().toISOString().split('T')[0]; // 今天
|
const END_DATE = new Date().toISOString().split('T')[0]; // 今天
|
||||||
@ -220,6 +225,54 @@ async function runWithConcurrency(tasks, limit) {
|
|||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------- 地址提取 ----------
|
||||||
|
|
||||||
|
const PROVINCES = [
|
||||||
|
'北京', '天津', '上海', '重庆',
|
||||||
|
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||||
|
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||||
|
'河南', '湖北', '湖南', '广东', '海南',
|
||||||
|
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||||
|
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||||
|
]
|
||||||
|
const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
|
||||||
|
const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||||
|
|
||||||
|
function extractRegionFromTitle(title) {
|
||||||
|
if (!title) return ''
|
||||||
|
// 匹配 "XX省"、"XX市"(直辖市)、"XX自治区"
|
||||||
|
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
|
||||||
|
if (m) {
|
||||||
|
const region = m[1]
|
||||||
|
if (region.includes('新疆')) return '新疆'
|
||||||
|
if (region.includes('广西')) return '广西'
|
||||||
|
if (region.includes('内蒙古')) return '内蒙古'
|
||||||
|
if (region.includes('宁夏')) return '宁夏'
|
||||||
|
if (region.includes('西藏')) return '西藏'
|
||||||
|
if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
|
||||||
|
if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
// 直接匹配省份关键词
|
||||||
|
for (const prov of SPECIAL_PROVS) {
|
||||||
|
if (title.includes(prov)) return prov
|
||||||
|
}
|
||||||
|
for (const prov of TWO_CHAR_PROVS) {
|
||||||
|
if (title.includes(prov)) return prov
|
||||||
|
}
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractCityFromTitle(title) {
|
||||||
|
if (!title) return ''
|
||||||
|
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
|
||||||
|
if (m) {
|
||||||
|
const city = m[1]
|
||||||
|
if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
|
||||||
|
}
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
// ---------- 生成公告清单 ----------
|
// ---------- 生成公告清单 ----------
|
||||||
|
|
||||||
function buildAttachmentName(r, index) {
|
function buildAttachmentName(r, index) {
|
||||||
@ -332,9 +385,11 @@ async function main() {
|
|||||||
|
|
||||||
log(`共获取 ${stats.total} 条公告(分 ${page} 页)`);
|
log(`共获取 ${stats.total} 条公告(分 ${page} 页)`);
|
||||||
|
|
||||||
// 过滤目标公告类型(由 TYPES 配置定义)
|
// 过滤目标公告类型(由 TYPES 配置定义),且仅取货物类
|
||||||
const caigou = allAnnouncements.filter(r => TYPES.includes(r.announcementType));
|
const caigou = allAnnouncements.filter(r =>
|
||||||
log(`其中 ${TYPES.join('、')}: ${caigou.length} 条`);
|
TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
|
||||||
|
);
|
||||||
|
log(`其中 ${TYPES.join('、')}(货物类): ${caigou.length} 条`);
|
||||||
|
|
||||||
// 生成公告清单(所有本次抓到的招采公告,无论是否已下载)
|
// 生成公告清单(所有本次抓到的招采公告,无论是否已下载)
|
||||||
const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
|
const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
|
||||||
@ -363,7 +418,7 @@ async function main() {
|
|||||||
if (!picUrl) {
|
if (!picUrl) {
|
||||||
stats.skipNoAttachment++;
|
stats.skipNoAttachment++;
|
||||||
log(`跳过(无附件): ${r.title}`, 'WARN');
|
log(`跳过(无附件): ${r.title}`, 'WARN');
|
||||||
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', time: new Date().toISOString() };
|
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -376,15 +431,20 @@ async function main() {
|
|||||||
|
|
||||||
tasks.push(async () => {
|
tasks.push(async () => {
|
||||||
log(`下载 [${shortName}]: ${fname}`);
|
log(`下载 [${shortName}]: ${fname}`);
|
||||||
|
|
||||||
|
// 从标题中提取省份(详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取)
|
||||||
|
const region = extractRegionFromTitle(r.title) || ''
|
||||||
|
const regionDetail = extractCityFromTitle(r.title) || ''
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await downloadFile(picUrl, fpath);
|
await downloadFile(picUrl, fpath);
|
||||||
const size = fs.statSync(fpath).size;
|
const size = fs.statSync(fpath).size;
|
||||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', path: `pdf/${fname}`, size, time: new Date().toISOString() };
|
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
|
||||||
stats.success++;
|
stats.success++;
|
||||||
log(` ✅ ${fname} (${size} bytes)`);
|
log(` ✅ ${fname} (${size} bytes) region=${region || '-'} addr=${regionDetail || '-'}`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
stats.failed++;
|
stats.failed++;
|
||||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', failed: true, error: err.message, time: new Date().toISOString() };
|
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
|
||||||
log(` ❌ ${fname}: ${err.message}`, 'ERROR');
|
log(` ❌ ${fname}: ${err.message}`, 'ERROR');
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@ -150,6 +150,41 @@ def save_state(state_path, state):
|
|||||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
|
||||||
|
def load_accumulated_state(output_base, state_filename, today_str):
|
||||||
|
"""加载历史状态实现跨天去重:合并所有历史状态文件中的已下载记录"""
|
||||||
|
accumulated = {}
|
||||||
|
last_run_date = None
|
||||||
|
state_dir = os.path.join(output_base, today_str)
|
||||||
|
# 先加载今日状态
|
||||||
|
today_state = load_state(os.path.join(state_dir, state_filename))
|
||||||
|
if today_state:
|
||||||
|
for k, v in today_state.items():
|
||||||
|
if k != "lastRun":
|
||||||
|
accumulated[k] = v
|
||||||
|
last_run = today_state.get("lastRun")
|
||||||
|
if last_run:
|
||||||
|
last_run_date = last_run[:10]
|
||||||
|
# 扫描历史目录(最近 30 天),合并之前下载的记录
|
||||||
|
try:
|
||||||
|
for i in range(1, 31):
|
||||||
|
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
|
path = os.path.join(output_base, d, state_filename)
|
||||||
|
if os.path.exists(path):
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
prev = json.load(f)
|
||||||
|
except:
|
||||||
|
continue
|
||||||
|
for k, v in prev.items():
|
||||||
|
if k != "lastRun" and k not in accumulated:
|
||||||
|
accumulated[k] = v
|
||||||
|
if k == "lastRun" and not last_run_date:
|
||||||
|
last_run_date = str(v)[:10]
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
return accumulated, last_run_date
|
||||||
|
|
||||||
|
|
||||||
def generate_markdown(notices, output_dir, date_range):
|
def generate_markdown(notices, output_dir, date_range):
|
||||||
"""生成公告清单 Markdown"""
|
"""生成公告清单 Markdown"""
|
||||||
lines = [
|
lines = [
|
||||||
@ -195,10 +230,16 @@ def run():
|
|||||||
|
|
||||||
today = date.today()
|
today = date.today()
|
||||||
|
|
||||||
|
# 加载历史已下载记录(跨天去重)
|
||||||
|
today_str = today.strftime("%Y-%m-%d")
|
||||||
|
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
|
||||||
|
|
||||||
if args.from_date:
|
if args.from_date:
|
||||||
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
||||||
|
elif not args.force and last_run_date:
|
||||||
|
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
|
||||||
else:
|
else:
|
||||||
date_from = today - timedelta(days=7) # 默认抓取最近 7 天
|
date_from = today - timedelta(days=7)
|
||||||
|
|
||||||
if args.to_date:
|
if args.to_date:
|
||||||
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
||||||
@ -212,7 +253,7 @@ def run():
|
|||||||
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
||||||
|
|
||||||
# 输出目录
|
# 输出目录
|
||||||
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
|
output_dir = os.path.join(OUTPUT_BASE, today_str)
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
# 子目录:html(源码) + pdf(转存)
|
# 子目录:html(源码) + pdf(转存)
|
||||||
@ -221,9 +262,9 @@ def run():
|
|||||||
os.makedirs(html_dir, exist_ok=True)
|
os.makedirs(html_dir, exist_ok=True)
|
||||||
os.makedirs(pdf_dir, exist_ok=True)
|
os.makedirs(pdf_dir, exist_ok=True)
|
||||||
|
|
||||||
# 状态文件
|
# 状态文件(使用累积状态做去重)
|
||||||
state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
|
state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
|
||||||
state = load_state(state_file)
|
state = accumulated_state
|
||||||
|
|
||||||
all_notices = []
|
all_notices = []
|
||||||
new_count = 0
|
new_count = 0
|
||||||
@ -268,6 +309,8 @@ def run():
|
|||||||
type_skip += 1
|
type_skip += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# 从标题/组织名初步提取地区(详情页提取时会补充/覆盖)
|
||||||
|
title_region = extract_region_from_text(notice["title"], notice["org"])
|
||||||
all_notices.append(notice)
|
all_notices.append(notice)
|
||||||
state[notice_id] = {
|
state[notice_id] = {
|
||||||
"title": notice["title"],
|
"title": notice["title"],
|
||||||
@ -276,6 +319,7 @@ def run():
|
|||||||
"type": notice["type"],
|
"type": notice["type"],
|
||||||
"detail_url": notice["detail_url"],
|
"detail_url": notice["detail_url"],
|
||||||
"downloaded_at": datetime.now().isoformat(),
|
"downloaded_at": datetime.now().isoformat(),
|
||||||
|
"region": title_region,
|
||||||
}
|
}
|
||||||
new_count += 1
|
new_count += 1
|
||||||
type_new += 1
|
type_new += 1
|
||||||
@ -283,7 +327,8 @@ def run():
|
|||||||
print(f" 新增: {type_new} | 跳过: {type_skip}")
|
print(f" 新增: {type_new} | 跳过: {type_skip}")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
# 保存状态
|
# 保存状态(含 lastRun 时间戳,用于下次增量爬取)
|
||||||
|
state["lastRun"] = datetime.now().isoformat()
|
||||||
save_state(state_file, state)
|
save_state(state_file, state)
|
||||||
|
|
||||||
# 为缺少金额的记录从详情页提取
|
# 为缺少金额的记录从详情页提取
|
||||||
@ -296,7 +341,9 @@ def run():
|
|||||||
# 详情页转存 PDF + HTML(新公告 + 已有状态中缺文件的)
|
# 详情页转存 PDF + HTML(新公告 + 已有状态中缺文件的)
|
||||||
all_pending = list(all_notices)
|
all_pending = list(all_notices)
|
||||||
if not args.force:
|
if not args.force:
|
||||||
for notice_id, info in state.items():
|
for notice_id, info in list(state.items()):
|
||||||
|
if not isinstance(info, dict):
|
||||||
|
continue
|
||||||
pdf_key = info.get("pdf", "")
|
pdf_key = info.get("pdf", "")
|
||||||
html_key = info.get("html", "")
|
html_key = info.get("html", "")
|
||||||
pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
|
pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
|
||||||
@ -323,6 +370,129 @@ def run():
|
|||||||
print(f"📁 保存目录: {output_dir}")
|
print(f"📁 保存目录: {output_dir}")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_deadline_from_text(text):
|
||||||
|
"""从公告正文中提取截止日期(采购文件获取截止时间)"""
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
# 模式1: "起至2026年6月22日19:00(北京时间)止" → 取截止日期
|
||||||
|
m = re.search(r"起至(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2}):(\d{2})", text)
|
||||||
|
if m:
|
||||||
|
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||||
|
# 模式2: "获取时间:...至2026-06-22 19:00"
|
||||||
|
m = re.search(r"至\s*(\d{4})-(\d{1,2})-(\d{1,2})\s+\d{1,2}:\d{2}", text)
|
||||||
|
if m:
|
||||||
|
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||||
|
# 模式3: "截止时间:2026年6月22日" 或 "截止日期:2026-06-22"
|
||||||
|
for pat in [
|
||||||
|
r"截止[时间日期]*[::]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
|
||||||
|
r"截止[时间日期]*[::]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
|
||||||
|
r"递交[文件]*截止[时间日期]*[::]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
|
||||||
|
r"递交[文件]*截止[时间日期]*[::]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
|
||||||
|
]:
|
||||||
|
m = re.search(pat, text)
|
||||||
|
if m:
|
||||||
|
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
# 全国省份/直辖市/自治区
|
||||||
|
PROVINCES = [
|
||||||
|
'北京', '天津', '上海', '重庆',
|
||||||
|
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||||
|
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||||
|
'河南', '湖北', '湖南', '广东', '海南',
|
||||||
|
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||||
|
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||||
|
]
|
||||||
|
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
|
||||||
|
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||||
|
|
||||||
|
|
||||||
|
def extract_province_best(text):
|
||||||
|
"""从任意文本中提取最可能的省份(先关键词匹配,避免正则在多字省份上截断)"""
|
||||||
|
if not text:
|
||||||
|
return ''
|
||||||
|
text_clean = text.replace(' ', '')
|
||||||
|
# 1. 特殊多字省份(优先,避免"新疆维吾尔自治区"被截)
|
||||||
|
for sp in SPECIAL_PROVS:
|
||||||
|
if sp in text_clean:
|
||||||
|
return sp
|
||||||
|
# 2. 直辖市
|
||||||
|
for dc in ['北京', '天津', '上海', '重庆']:
|
||||||
|
if dc in text_clean:
|
||||||
|
return dc
|
||||||
|
# 3. "XX省"(如"陕西省"、"广东省")
|
||||||
|
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
|
||||||
|
if m:
|
||||||
|
return m.group(1)[:2]
|
||||||
|
# 4. 两字省份关键词
|
||||||
|
for p in TWO_CHAR_PROVS:
|
||||||
|
if p in text_clean:
|
||||||
|
return p
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
def extract_city_from_text(text):
|
||||||
|
"""从文本中提取城市名(XX市)"""
|
||||||
|
if not text:
|
||||||
|
return ''
|
||||||
|
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
|
||||||
|
if m:
|
||||||
|
city = m.group(1)
|
||||||
|
if city not in ['北京市', '天津市', '上海市', '重庆市']:
|
||||||
|
return city
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
def extract_region_from_text(text, org=""):
|
||||||
|
"""从公告正文或公司名中提取地区(省份)"""
|
||||||
|
if not text and not org:
|
||||||
|
return ""
|
||||||
|
# 1. 从正文找项目/交货/施工地点
|
||||||
|
if text:
|
||||||
|
for pat in [
|
||||||
|
r"(?:项目|交货|送货|施工|到货|供货|工程|实施|服务)[地址地点]*[::]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
|
||||||
|
r"(?:地[址点]|所在地区)[::]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
|
||||||
|
r"位\s*于\s*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
|
||||||
|
r"(?:项目|工程)\s*(?:位于|所在地|所在)[:::\s]*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
|
||||||
|
]:
|
||||||
|
m = re.search(pat, text)
|
||||||
|
if m:
|
||||||
|
region = m.group(1)
|
||||||
|
for sp in SPECIAL_PROVS:
|
||||||
|
if sp in region:
|
||||||
|
return sp
|
||||||
|
return region[:2] # 取前两个字
|
||||||
|
# 2. 从全文扫描省份关键词
|
||||||
|
prov = extract_province_best(text)
|
||||||
|
if prov:
|
||||||
|
return prov
|
||||||
|
# 3. 从公司名推断
|
||||||
|
if org:
|
||||||
|
org_prov = extract_province_best(org)
|
||||||
|
if org_prov:
|
||||||
|
return org_prov
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def extract_region_detail_from_text(text):
|
||||||
|
"""从公告正文中提取详细收货/交货地址"""
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
for pat in [
|
||||||
|
r"(?:交货|收货|送货|项目|施工|到货|供货|工程|实施|服务)[地址地点]*[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
|
||||||
|
r"(?:地[址点])[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
|
||||||
|
r"建设地点[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
|
||||||
|
r"项目地点[::]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
|
||||||
|
]:
|
||||||
|
m = re.search(pat, text)
|
||||||
|
if m:
|
||||||
|
addr = m.group(1).strip()
|
||||||
|
if len(addr) >= 6:
|
||||||
|
return addr
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
def extract_amount_from_text(text):
|
def extract_amount_from_text(text):
|
||||||
"""从公告正文中提取金额(万元)"""
|
"""从公告正文中提取金额(万元)"""
|
||||||
if not text:
|
if not text:
|
||||||
@ -367,6 +537,8 @@ def extract_amounts_for_existing(state, output_dir):
|
|||||||
for notice_id, info in state.items():
|
for notice_id, info in state.items():
|
||||||
if notice_id == "lastRun":
|
if notice_id == "lastRun":
|
||||||
continue
|
continue
|
||||||
|
if not isinstance(info, dict):
|
||||||
|
continue
|
||||||
if info.get("amount"):
|
if info.get("amount"):
|
||||||
continue
|
continue
|
||||||
pending.append(notice_id)
|
pending.append(notice_id)
|
||||||
@ -396,12 +568,27 @@ def extract_amounts_for_existing(state, output_dir):
|
|||||||
await page.close()
|
await page.close()
|
||||||
|
|
||||||
amount = extract_amount_from_text(text)
|
amount = extract_amount_from_text(text)
|
||||||
|
deadline = extract_deadline_from_text(text)
|
||||||
|
region = extract_region_from_text(text, info.get("org", ""))
|
||||||
|
region_detail = extract_region_detail_from_text(text)
|
||||||
|
updated = False
|
||||||
if amount:
|
if amount:
|
||||||
state[notice_id]["amount"] = amount
|
state[notice_id]["amount"] = amount
|
||||||
|
updated = True
|
||||||
|
if deadline:
|
||||||
|
state[notice_id]["deadline"] = deadline
|
||||||
|
updated = True
|
||||||
|
if region:
|
||||||
|
state[notice_id]["region"] = region
|
||||||
|
updated = True
|
||||||
|
if region_detail:
|
||||||
|
state[notice_id]["region_detail"] = region_detail
|
||||||
|
updated = True
|
||||||
|
if updated:
|
||||||
success += 1
|
success += 1
|
||||||
print(f"✅ {amount}")
|
print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
|
||||||
else:
|
else:
|
||||||
print("⏭️ 未找到金额")
|
print("⏭️ 未找到金额/截止日期/地区")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"❌ {e}")
|
print(f"❌ {e}")
|
||||||
|
|
||||||
@ -472,6 +659,15 @@ def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file
|
|||||||
amount = extract_amount_from_text(page_text)
|
amount = extract_amount_from_text(page_text)
|
||||||
if amount and state is not None and notice_id in state:
|
if amount and state is not None and notice_id in state:
|
||||||
state[notice_id]["amount"] = amount
|
state[notice_id]["amount"] = amount
|
||||||
|
deadline = extract_deadline_from_text(page_text)
|
||||||
|
if deadline and state is not None and notice_id in state:
|
||||||
|
state[notice_id]["deadline"] = deadline
|
||||||
|
region = extract_region_from_text(page_text, notice.get("org", ""))
|
||||||
|
if region and state is not None and notice_id in state:
|
||||||
|
state[notice_id]["region"] = region
|
||||||
|
region_detail = extract_region_detail_from_text(page_text)
|
||||||
|
if region_detail and state is not None and notice_id in state:
|
||||||
|
state[notice_id]["region_detail"] = region_detail
|
||||||
|
|
||||||
# 保存 HTML 源码到 html/ 目录(供 AI 阅读)
|
# 保存 HTML 源码到 html/ 目录(供 AI 阅读)
|
||||||
html_source = await page.content()
|
html_source = await page.content()
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user