Compare commits
6 Commits
672a4b41b2
...
8d8945f92b
| Author | SHA1 | Date | |
|---|---|---|---|
|
8d8945f92b | ||
|
|
44609be8c7 | ||
|
|
0a423af4d2 | ||
|
|
ae72966766 | ||
|
|
bd213eb5ec | ||
|
|
49f922444a |
3
.gitignore
vendored
3
.gitignore
vendored
@ -41,6 +41,9 @@ pnpm-debug.log*
|
||||
# OS
|
||||
Thumbs.db
|
||||
|
||||
# Uploads (不上传图片文件)
|
||||
apps/api/public/uploads/
|
||||
|
||||
# Backup files (防误提交)
|
||||
*.bak
|
||||
*.tmp
|
||||
|
||||
42
AGENTS.md
42
AGENTS.md
@ -2,6 +2,48 @@
|
||||
|
||||
语言：中文。所有对话、注释、文档、提交信息均使用中文。
|
||||
|
||||
## 🤝 协作流程标准（避免理解不一致造成返工）
|
||||
|
||||
**所有具体问题的修改，必须按这个流程执行：**
|
||||
|
||||
```
|
||||
1. 用户描述需求（业务场景、看到的问题、截图、日志）
|
||||
↓
|
||||
2. 协作者反问 7 个技术维度（场景/触发/重要程度/数据/边界/样式/部署）
|
||||
↓
|
||||
3. 对齐范围：用户确认 + 协作者列技术方案
|
||||
↓
|
||||
4. 实施 + 验证
|
||||
↓
|
||||
5. 用户看效果，反馈或继续下一项
|
||||
```
|
||||
|
||||
### 协作者必须问的 7 个技术维度
|
||||
|
||||
| 维度 | 问什么 |
|
||||
|------|--------|
|
||||
| **场景** | 哪个页面/哪个模块？用户路径是什么？ |
|
||||
| **触发条件** | 什么时候出现？手动还是自动？频率？ |
|
||||
| **重要程度** | 阻塞核心流程 / 体验优化 / 锦上添花？ |
|
||||
| **数据** | 影响多少数据？要不要数据迁移？ |
|
||||
| **边界** | 空数据/异常情况怎么处理？ |
|
||||
| **样式** | 改动大吗？要不要出截图对比？ |
|
||||
| **生产部署** | 改完要不要立即部署？还是攒一起？ |
|
||||
|
||||
### 原则
|
||||
|
||||
- **不擅自开干**：未对齐范围前不做实现
|
||||
- **不预设方案**：用户描述 ≠ 协作者的解决方案，先问清楚
|
||||
- **不省略验证**：改完必须跑端到端测试，不只是"代码看起来对"
|
||||
- **不静默改大动作**：涉及数据库结构、生产配置、依赖升级等要明确告知
|
||||
|
||||
### 例外（不需要问的简单场景）
|
||||
|
||||
- 拼写错误、明显笔误
|
||||
- 单文件 typo 修复
|
||||
- 用户明确说"按你判断做"
|
||||
- 紧急 bug 修复（事后补问）
|
||||
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
|
||||
@ -18,6 +18,8 @@ interface FetchResult {
|
||||
noticeId: string
|
||||
detailUrl: string
|
||||
amount?: string
|
||||
region?: string
|
||||
regionDetail?: string
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
@ -109,7 +111,7 @@ export class CrawlerService implements OnModuleInit {
|
||||
let results: FetchResult[] = []
|
||||
|
||||
if (config.type === 'bid_powerchina') {
|
||||
results = await this.runPowerChina()
|
||||
results = await this.runPowerChina(config)
|
||||
} else {
|
||||
results = await this.runGenericScript(config)
|
||||
}
|
||||
@ -129,6 +131,8 @@ export class CrawlerService implements OnModuleInit {
|
||||
detailUrl: r.detailUrl,
|
||||
publishTime: r.publishTime || null,
|
||||
...(r.amount ? { amount: r.amount } : {}),
|
||||
...(r.region ? { region: r.region } : {}),
|
||||
...(r.regionDetail ? { regionDetail: r.regionDetail } : {}),
|
||||
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
||||
},
|
||||
create: {
|
||||
@ -143,7 +147,8 @@ export class CrawlerService implements OnModuleInit {
|
||||
status: '待复核',
|
||||
level: 'C',
|
||||
category: '',
|
||||
region: '',
|
||||
region: r.region || '',
|
||||
regionDetail: r.regionDetail || '',
|
||||
summary: `来源: ${r.source}\n公告ID: ${r.noticeId}\n链接: ${r.detailUrl}`,
|
||||
},
|
||||
})
|
||||
@ -172,49 +177,92 @@ export class CrawlerService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
private async runPowerChina(): Promise<FetchResult[]> {
|
||||
private async runPowerChina(config: { keywords?: string | null }): Promise<FetchResult[]> {
|
||||
const scriptPath = path.join(TENDER_FETCHER_DIR, 'fetch-v1.2.js')
|
||||
if (!fs.existsSync(scriptPath)) {
|
||||
throw new Error('中电建抓取脚本不存在')
|
||||
}
|
||||
|
||||
execSync(`node "${scriptPath}"`, {
|
||||
let keywords: string[]
|
||||
try {
|
||||
keywords = config.keywords ? JSON.parse(config.keywords) : []
|
||||
} catch {
|
||||
keywords = []
|
||||
}
|
||||
if (keywords.length === 0) keywords = ['']
|
||||
|
||||
const today = new Date().toISOString().split('T')[0]
|
||||
const baseDir = path.join(process.env.HOME || '', 'desktop/招标公告', today)
|
||||
const seen = new Set<string>()
|
||||
const allResults: FetchResult[] = []
|
||||
|
||||
for (const kw of keywords) {
|
||||
const kwFlag = kw ? ` --keyword "${kw}"` : ''
|
||||
try {
|
||||
execSync(`node "${scriptPath}"${kwFlag}`, {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
timeout: 180000,
|
||||
})
|
||||
} catch {
|
||||
// 单个 keyword 失败不影响其他 keyword
|
||||
continue
|
||||
}
|
||||
|
||||
const today = new Date().toISOString().split('T')[0]
|
||||
const statePath = path.join(
|
||||
process.env.HOME || '',
|
||||
'desktop/招标公告',
|
||||
today,
|
||||
'downloaded.json',
|
||||
)
|
||||
|
||||
if (!fs.existsSync(statePath)) return []
|
||||
const statePath = path.join(baseDir, 'downloaded.json')
|
||||
if (!fs.existsSync(statePath)) continue
|
||||
|
||||
try {
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||
const downloaded = state.downloaded || {}
|
||||
const results: FetchResult[] = []
|
||||
|
||||
const outputDir = path.dirname(statePath)
|
||||
const pdfDir = path.join(outputDir, 'pdf')
|
||||
|
||||
for (const [key, v] of Object.entries(downloaded)) {
|
||||
const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string }
|
||||
const info = v as { title?: string; skipped?: boolean; failed?: boolean; id?: string; deadline?: string; publishTime?: string; path?: string }
|
||||
if (info.skipped || info.failed) continue
|
||||
const [systemId] = key.split('_')
|
||||
if (seen.has(systemId)) continue
|
||||
seen.add(systemId)
|
||||
|
||||
const realId = info.id || systemId
|
||||
results.push({
|
||||
|
||||
let region = ''
|
||||
let regionDetail = ''
|
||||
if (info.path && fs.existsSync(pdfDir)) {
|
||||
const pdfPath = path.join(pdfDir, path.basename(info.path))
|
||||
if (fs.existsSync(pdfPath)) {
|
||||
try {
|
||||
const extractor = path.join(SCRIPTS_DIR, 'extract_address_from_pdf.py')
|
||||
const title = info.title || ''
|
||||
const cmd = `python3 "${extractor}" "${pdfPath}" "${title}"`
|
||||
const out = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim()
|
||||
const parsed = JSON.parse(out)
|
||||
region = parsed.region || ''
|
||||
regionDetail = parsed.regionDetail || ''
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
allResults.push({
|
||||
source: '中电建',
|
||||
title: info.title || '',
|
||||
organization: '中电建',
|
||||
publishTime: state.lastRun || '',
|
||||
publishTime: info.publishTime || state.lastRun || '',
|
||||
deadline: info.deadline || '',
|
||||
noticeId: systemId,
|
||||
detailUrl: `https://bid.powerchina.cn/notice/detail?id=${realId}`,
|
||||
region,
|
||||
regionDetail,
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
return allResults
|
||||
}
|
||||
|
||||
private async runGenericScript(config: { type: string; sourceName?: string | null; scriptPath?: string | null }): Promise<FetchResult[]> {
|
||||
@ -248,10 +296,13 @@ export class CrawlerService implements OnModuleInit {
|
||||
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'))
|
||||
const results: FetchResult[] = []
|
||||
const todayStr = new Date().toISOString().substring(0, 10)
|
||||
|
||||
for (const [noticeId, v] of Object.entries(state)) {
|
||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string }
|
||||
const info = v as { title?: string; org?: string; publish_time?: string; detail_url?: string; id?: string; deadline?: string; amount?: string; region?: string; region_detail?: string; downloaded_at?: string }
|
||||
if (noticeId === 'lastRun') continue
|
||||
// 只处理本次新增的记录（downloaded_at 包含今天日期），历史记录已入库无需重新处理
|
||||
if (info.downloaded_at && !info.downloaded_at.startsWith(todayStr)) continue
|
||||
const leadAmount = info.amount || ''
|
||||
results.push({
|
||||
source,
|
||||
@ -262,6 +313,8 @@ export class CrawlerService implements OnModuleInit {
|
||||
noticeId: info.id || noticeId,
|
||||
detailUrl: info.detail_url || '',
|
||||
amount: leadAmount,
|
||||
region: info.region || '',
|
||||
regionDetail: info.region_detail || '',
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -373,11 +373,11 @@ export class IntegrationService implements OnModuleDestroy {
|
||||
* 解析 MCP tool 返回的 content 字段
|
||||
* MCP 协议格式: { content: [{ type: 'text', text: '...' }], isError?: boolean }
|
||||
*/
|
||||
private parseToolResult<T>(result: any): T | null {
|
||||
try {
|
||||
if (!result) return null
|
||||
private parseToolResult<T>(result: any): T {
|
||||
if (!result) throw new Error('MCP 工具返回结果为空')
|
||||
if (result.isError) {
|
||||
throw new Error(typeof result.content?.[0]?.text === 'string' ? result.content[0].text : 'Tool 调用返回错误')
|
||||
const errMsg = typeof result.content?.[0]?.text === 'string' ? result.content[0].text : 'Tool 调用返回错误'
|
||||
throw new Error(errMsg)
|
||||
}
|
||||
if (result.content && Array.isArray(result.content) && result.content.length > 0) {
|
||||
const item = result.content[0]
|
||||
@ -390,10 +390,6 @@ export class IntegrationService implements OnModuleDestroy {
|
||||
}
|
||||
}
|
||||
return result as T
|
||||
} catch (e: any) {
|
||||
this.logger.error(`解析 MCP tool 结果失败: ${e.message}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
getClient(integrationId: string): Client | null {
|
||||
|
||||
@ -14,8 +14,9 @@ export class LeadController {
|
||||
@Query('source') source?: string,
|
||||
@Query('category') category?: string,
|
||||
@Query('assignedTo') assignedTo?: string,
|
||||
@Query('timeRange') timeRange?: string,
|
||||
) {
|
||||
return this.leadService.findAll({ status, level, search, star, source, category, assignedTo })
|
||||
return this.leadService.findAll({ status, level, search, star, source, category, assignedTo, timeRange })
|
||||
}
|
||||
|
||||
@Get('stats')
|
||||
|
||||
@ -2,6 +2,25 @@ import { Injectable, NotFoundException, Logger, ServiceUnavailableException } fr
|
||||
import { PrismaService } from '../prisma/prisma.service'
|
||||
import { IntegrationService } from '../integration/integration.service'
|
||||
|
||||
// 品类 → 标题关键词映射（筛选时按标题模糊匹配，不走 category 字段）
|
||||
const CATEGORY_KEYWORDS: Record<string, string[]> = {
|
||||
'防火门': ['防火门', '防火卷帘', '防火窗', '钢质防火', '木质防火'],
|
||||
'配电箱': ['配电箱', '配电柜', '配电盘', '照明配电箱', '动力配电箱', '电表箱'],
|
||||
'电线电缆': ['电线', '电缆', '导线', '电力线', '控制电缆', '光缆', '架空线'],
|
||||
'阀门': ['阀门', '闸阀', '球阀', '蝶阀', '截止阀', '止回阀', '调节阀'],
|
||||
'钢材': ['钢材', '钢管', '钢板', '钢筋', '型钢', '槽钢', '角钢', '工字钢', 'H型钢', '螺纹钢'],
|
||||
'涂料': ['涂料', '油漆', '防腐漆', '防火涂料', '乳胶漆', '防水涂料'],
|
||||
'混凝土': ['混凝土', '商混', '商品砼', '水泥', '砂浆', '砼'],
|
||||
'变压器': ['变压器', '箱变', '干式变', '油浸变', '电力变压器'],
|
||||
'水泵': ['水泵', '离心泵', '潜污泵', '消防泵', '排污泵', '循环泵'],
|
||||
'风机': ['风机', '通风机', '排风机', '轴流风机', '离心风机'],
|
||||
'管材': ['管材', '管道', 'PVC管', 'PE管', '镀锌管', '无缝管', '螺旋管', '给水管'],
|
||||
'开关': ['开关', '断路器', '接触器', '继电器', '熔断器', '隔离开关'],
|
||||
'灯具': ['灯具', '照明灯', '路灯', '防爆灯', 'LED灯', '太阳能灯'],
|
||||
'仪表': ['仪表', '仪器', '传感器', '变送器', '流量计', '压力表', '温度计'],
|
||||
'电机': ['电机', '电动机', '减速机', '减速器', '变频电机'],
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class LeadService {
|
||||
private readonly logger = new Logger(LeadService.name)
|
||||
@ -11,7 +30,7 @@ export class LeadService {
|
||||
private integrationService: IntegrationService,
|
||||
) {}
|
||||
|
||||
async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string }) {
|
||||
async findAll(params?: { status?: string; level?: string; search?: string; star?: string; source?: string; category?: string; assignedTo?: string; timeRange?: string }) {
|
||||
const where: any = {}
|
||||
if (params?.status) where.status = params.status
|
||||
if (params?.level) where.level = params.level
|
||||
@ -28,14 +47,44 @@ export class LeadService {
|
||||
else if (!isNaN(starVal)) where.star = starVal
|
||||
}
|
||||
if (params?.source) where.source = params.source
|
||||
if (params?.category) where.category = params.category
|
||||
if (params?.category) {
|
||||
const keywords = CATEGORY_KEYWORDS[params.category]
|
||||
if (keywords) {
|
||||
where.OR = keywords.map(kw => ({ title: { contains: kw } }))
|
||||
}
|
||||
}
|
||||
if (params?.assignedTo) {
|
||||
where.assignedTo = params.assignedTo === '未分配' ? null : params.assignedTo
|
||||
}
|
||||
if (params?.timeRange && params.timeRange !== '全部时间') {
|
||||
const now = new Date()
|
||||
const pad = (n: number) => n.toString().padStart(2, '0')
|
||||
const today = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`
|
||||
if (params.timeRange === '今天') {
|
||||
where.publishTime = { startsWith: today }
|
||||
} else if (params.timeRange === '昨天') {
|
||||
const y = new Date(now)
|
||||
y.setDate(y.getDate() - 1)
|
||||
const yesterday = `${y.getFullYear()}-${pad(y.getMonth() + 1)}-${pad(y.getDate())}`
|
||||
where.publishTime = { startsWith: yesterday }
|
||||
} else if (params.timeRange === '本周') {
|
||||
const day = now.getDay()
|
||||
const diff = day === 0 ? 6 : day - 1 // Monday
|
||||
const monday = new Date(now)
|
||||
monday.setDate(now.getDate() - diff)
|
||||
const sunday = new Date(monday)
|
||||
sunday.setDate(monday.getDate() + 6)
|
||||
const m = `${monday.getFullYear()}-${pad(monday.getMonth() + 1)}-${pad(monday.getDate())}`
|
||||
const s = `${sunday.getFullYear()}-${pad(sunday.getMonth() + 1)}-${pad(sunday.getDate())}`
|
||||
where.publishTime = { gte: m, lte: s + ' 23:59:59' }
|
||||
} else if (params.timeRange === '本月') {
|
||||
where.publishTime = { startsWith: `${now.getFullYear()}-${pad(now.getMonth() + 1)}` }
|
||||
}
|
||||
}
|
||||
return this.prisma.lead.findMany({
|
||||
where,
|
||||
orderBy: [
|
||||
{ level: 'asc' },
|
||||
{ deadline: 'desc' },
|
||||
{ createdAt: 'desc' },
|
||||
],
|
||||
})
|
||||
|
||||
@ -40,14 +40,6 @@ export class ProductController {
|
||||
return this.productService.batchPublish(body.ids)
|
||||
}
|
||||
|
||||
/**
|
||||
* F002 P0-A: 批量上架商品到 B2B 平台 (MCP import_products)
|
||||
*/
|
||||
@Post('publish-to-direv')
|
||||
publishToDirev(@Body() body: { ids: string[] }) {
|
||||
return this.productService.publishToDirev(body.ids)
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
remove(@Param('id') id: string) {
|
||||
return this.productService.remove(id)
|
||||
|
||||
@ -42,90 +42,87 @@ export class ProductService {
|
||||
}
|
||||
|
||||
async batchPublish(ids: string[]) {
|
||||
return this.prisma.product.updateMany({
|
||||
where: { id: { in: ids } },
|
||||
data: { status: 'published' },
|
||||
})
|
||||
}
|
||||
|
||||
async remove(id: string) {
|
||||
return this.prisma.product.delete({ where: { id } })
|
||||
}
|
||||
|
||||
/**
|
||||
* F002 P0-A: 批量上架商品到 B2B 平台 (MCP import_products)
|
||||
* 字段映射: Product → import_products schema
|
||||
*/
|
||||
async publishToDirev(ids: string[]) {
|
||||
const b2bIntegration = await this.integrationService.findActiveMCP()
|
||||
if (!b2bIntegration) {
|
||||
throw new ServiceUnavailableException('未配置 B2B 集成')
|
||||
}
|
||||
|
||||
const products = await this.prisma.product.findMany({
|
||||
where: { id: { in: ids } },
|
||||
})
|
||||
if (products.length === 0) return { imported: 0, errors: [], ids: [] }
|
||||
|
||||
if (products.length === 0) {
|
||||
throw new ServiceUnavailableException('未找到指定商品')
|
||||
const b2bIntegration = await this.integrationService.findActiveMCP()
|
||||
if (!b2bIntegration) {
|
||||
throw new ServiceUnavailableException('未配置 B2B 平台 MCP 集成，请在系统配置中添加')
|
||||
}
|
||||
|
||||
// Category string → numeric ID mapping (B2B tb_good first_category_id)
|
||||
const CATEGORY_MAP: Record<string, number> = {
|
||||
'电缆': 28,
|
||||
'电力电缆': 28,
|
||||
'配电箱': 29,
|
||||
'开关柜': 29,
|
||||
'变压器': 27,
|
||||
'电线': 28,
|
||||
'母线槽': 30,
|
||||
'桥架': 31,
|
||||
'电缆': 28, '电力电缆': 28, '配电箱': 29, '开关柜': 29,
|
||||
'变压器': 27, '电线': 28, '母线槽': 30, '桥架': 31,
|
||||
}
|
||||
const fallbackCategoryId = 1
|
||||
|
||||
const items = products.map(p => {
|
||||
const mappedItems = products.map(p => {
|
||||
const catId = CATEGORY_MAP[p.category] || fallbackCategoryId
|
||||
const item: any = {
|
||||
name: p.name,
|
||||
category_id: catId,
|
||||
price: p.price,
|
||||
brandName: p.brand || '',
|
||||
stock: p.quantity,
|
||||
name: p.name, category_id: catId,
|
||||
price: p.price > 0 ? p.price : 0.01,
|
||||
brand: p.brand || '',
|
||||
unit: p.unit || '',
|
||||
sku: p.spec || p.name || '',
|
||||
}
|
||||
if (p.spec) item.sku = p.spec
|
||||
if (p.imageUrl) item.images = [p.imageUrl]
|
||||
if (p.supplier) item.remark = `供应商:${p.supplier}`
|
||||
return item
|
||||
})
|
||||
|
||||
this.logger.log(`batchPublish: 发送 ${mappedItems.length} 个商品到 MCP import_products`)
|
||||
const result = await this.integrationService.executeTool<any>(
|
||||
b2bIntegration.id,
|
||||
'import_products',
|
||||
{ products: items },
|
||||
b2bIntegration.id, 'import_products',
|
||||
{ products: mappedItems },
|
||||
{ retries: 2, retryDelayMs: 1000 },
|
||||
)
|
||||
|
||||
this.logger.log(`batchPublish: MCP 返回 success=${result.success} data=${result.data ? JSON.stringify(result.data).slice(0, 300) : '(empty)'}`)
|
||||
|
||||
if (!result.success) {
|
||||
const errorMsg = result.error || '未知错误'
|
||||
this.logger.error(`商品上架失败 (ids=${ids.join(',')}): ${errorMsg}`)
|
||||
throw new ServiceUnavailableException(`上架失败: ${errorMsg}`)
|
||||
throw new ServiceUnavailableException(`B2B 同步失败: ${result.error || '未知错误'}`)
|
||||
}
|
||||
|
||||
const data = result.data as any
|
||||
const payload = data.data || data
|
||||
const imported = payload.imported || 0
|
||||
const errors = payload.errors || []
|
||||
this.logger.log(`商品上架成功: 导入 ${imported} 条, 失败 ${errors.length} 条`)
|
||||
// MCP import_products 返回格式: { code:0, msg:"成功", data: [{ status:"success"|"failed", data:{id:"xxx"}, sku, name, error? }] }
|
||||
const items: Array<{ status: string; data?: any; sku?: string; name?: string; error?: string }> =
|
||||
(result.data as any)?.data || []
|
||||
const imported = items.filter(i => i.status === 'success').length
|
||||
const b2bIds: string[] = items
|
||||
.filter(i => i.status === 'success' && i.data?.id)
|
||||
.map(i => String(i.data.id))
|
||||
const errors: Array<{ sku: string; name: string; error: string }> = items
|
||||
.filter(i => i.status === 'failed')
|
||||
.map(i => ({ sku: i.sku || '', name: i.name || '', error: i.error || '未知错误' }))
|
||||
if (errors.length > 0) {
|
||||
this.logger.warn(`B2B import_products 导入失败详情: ${errors.map(e => `[${e.sku}] ${e.name}: ${e.error}`).join(' | ')}`)
|
||||
}
|
||||
this.logger.log(`B2B import_products 完成: 导入 ${imported} 条, 失败 ${errors.length} 条, B2B IDs: ${b2bIds.join(',')}`)
|
||||
|
||||
if (b2bIds.length > 0) {
|
||||
this.logger.log(`batchPublish: 上架 ${b2bIds.length} 个商品到 B2B publish_products`)
|
||||
const publishResult = await this.integrationService.executeTool<any>(
|
||||
b2bIntegration.id, 'publish_products',
|
||||
{ product_ids: b2bIds },
|
||||
{ retries: 1, retryDelayMs: 1000 },
|
||||
)
|
||||
this.logger.log(`batchPublish: publish_products 返回 success=${publishResult.success}`)
|
||||
if (!publishResult.success) {
|
||||
throw new ServiceUnavailableException(`B2B 上架失败: ${publishResult.error || 'publish_products 返回失败'}`)
|
||||
}
|
||||
this.logger.log(`B2B 上架成功: ${b2bIds.length} 个商品`)
|
||||
}
|
||||
|
||||
await this.prisma.product.updateMany({
|
||||
where: { id: { in: products.map(p => p.id) } },
|
||||
where: { id: { in: ids } },
|
||||
data: { status: 'published' },
|
||||
})
|
||||
|
||||
return {
|
||||
imported,
|
||||
errors,
|
||||
ids: payload.ids || [],
|
||||
return { imported, errors, ids: b2bIds }
|
||||
}
|
||||
|
||||
async remove(id: string) {
|
||||
return this.prisma.product.delete({ where: { id } })
|
||||
}
|
||||
}
|
||||
|
||||
@ -1 +1 @@
|
||||
NEXT_PUBLIC_API_URL=
|
||||
NEXT_PUBLIC_API_URL=/api
|
||||
|
||||
@ -35,7 +35,7 @@ function mapLead(raw: any): Lead {
|
||||
id: raw.id,
|
||||
title: raw.title,
|
||||
star: raw.star || 0,
|
||||
level: raw.level || 'C',
|
||||
level: raw.level ? raw.level + '级' : 'C级',
|
||||
amount: raw.amount || '待确认',
|
||||
status: raw.status || '待复核',
|
||||
deadline: raw.deadline || '',
|
||||
@ -58,9 +58,9 @@ function mapLead(raw: any): Lead {
|
||||
}
|
||||
|
||||
const STATUS_OPTIONS = ['全部状态', '待复核', '待跟进', '跟进中', '已转询价', '已转协同', '已成交', '无效线索']
|
||||
const LEVEL_OPTIONS = ['全部级别', 'S级', 'A级', 'B级']
|
||||
const CAT_OPTIONS = ['全部品类', '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土']
|
||||
const SOURCE_OPTIONS = ['全部来源', 'PowerChina 招采', '新矿集团', '发改委审批', '行业资讯']
|
||||
const LEVEL_OPTIONS = ['全部级别', 'S级', 'A级', 'B级', 'C级', 'D级']
|
||||
const CAT_OPTIONS = ['全部品类', '防火门', '配电箱', '电线电缆', '阀门', '钢材', '涂料', '混凝土', '变压器', '水泵', '风机', '管材', '开关', '灯具', '仪表', '电机']
|
||||
const SOURCE_OPTIONS = ['全部来源', '中电建', '新矿集团', '中核集团']
|
||||
const PERSON_OPTIONS = ['全部人员', '郑钊', '赵主任', '运营管理员', '未分配']
|
||||
|
||||
function getStatusClass(status: string) {
|
||||
@ -110,7 +110,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
|
||||
const showToast = (msg: string) => { setToast(msg); setTimeout(() => setToast(''), 2500) }
|
||||
|
||||
const [fTime, setFTime] = useState('全部时间')
|
||||
const [fTime, setFTime] = useState('全部日期')
|
||||
const [fStar, setFStar] = useState('全部星级')
|
||||
const [fStatus, setFStatus] = useState('全部状态')
|
||||
const [fLevel, setFLevel] = useState('全部级别')
|
||||
@ -118,7 +118,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
const [fSource, setFSource] = useState('全部来源')
|
||||
const [fPerson, setFPerson] = useState('全部人员')
|
||||
|
||||
const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string) => {
|
||||
const loadLeads = useCallback(async (star?: string, status?: string, level?: string, cat?: string, source?: string, person?: string, timeRange?: string) => {
|
||||
setLoading(true)
|
||||
const params = new URLSearchParams()
|
||||
if (star && star !== '全部星级') {
|
||||
@ -128,11 +128,12 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
else if (star === '2星及以下') params.set('star', '2星及以下')
|
||||
}
|
||||
if (status && status !== '全部状态') params.set('status', status)
|
||||
if (level && level !== '全部级别') params.set('level', level)
|
||||
if (level && level !== '全部级别') params.set('level', level.replace('级', ''))
|
||||
if (cat && cat !== '全部品类') params.set('category', cat)
|
||||
if (source && source !== '全部来源') params.set('source', source)
|
||||
if (person && person !== '全部人员' && person !== '未分配') params.set('assignedTo', person)
|
||||
if (person === '未分配') params.set('assignedTo', '未分配')
|
||||
if (timeRange && timeRange !== '全部日期') params.set('timeRange', timeRange)
|
||||
try {
|
||||
const qs = params.toString()
|
||||
const data: any[] = await fetch(`${API}${qs ? '?' + qs : ''}`).then(r => r.json())
|
||||
@ -144,10 +145,10 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
setLoading(false)
|
||||
}, [])
|
||||
|
||||
useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson])
|
||||
useEffect(() => { loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime) }, [loadLeads, fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime])
|
||||
|
||||
const refreshLeads = async () => {
|
||||
await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson)
|
||||
await loadLeads(fStar, fStatus, fLevel, fCat, fSource, fPerson, fTime)
|
||||
}
|
||||
|
||||
const totalPages = Math.ceil(filtered.length / pageSize)
|
||||
@ -392,7 +393,7 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
<div className="bg-white p-4 border border-border-subtle rounded-lg shadow-sm flex flex-wrap items-center gap-4">
|
||||
<div className="flex flex-wrap items-center gap-4">
|
||||
{[
|
||||
{ label: '时间范围', value: fTime, set: setFTime, opts: ['全部时间', '今天', '昨天', '本周', '本月'] },
|
||||
{ label: '公告日期', value: fTime, set: setFTime, opts: ['全部日期', '今天', '昨天', '本周', '本月'] },
|
||||
{ label: '星级', value: fStar, set: setFStar, opts: ['全部星级', '5星', '4星', '3星', '2星及以下'] },
|
||||
{ label: '状态', value: fStatus, set: setFStatus, opts: STATUS_OPTIONS },
|
||||
{ label: '甲方级别', value: fLevel, set: setFLevel, opts: LEVEL_OPTIONS },
|
||||
@ -445,7 +446,6 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">星级</th>
|
||||
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">甲方级别</th>
|
||||
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase text-right whitespace-nowrap">金额(万)</th>
|
||||
<th className="p-4 w-20 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">数量</th>
|
||||
<th className="p-4 w-16 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">状态</th>
|
||||
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">投标截止</th>
|
||||
<th className="p-4 w-24 text-[10px] font-bold tracking-wider text-on-surface-variant uppercase whitespace-nowrap">公告日期</th>
|
||||
@ -454,9 +454,9 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
</thead>
|
||||
<tbody className="divide-y divide-border-subtle">
|
||||
{loading ? (
|
||||
<tr><td colSpan={11} className="p-8 text-center text-sm text-on-surface-variant">加载中...</td></tr>
|
||||
<tr><td colSpan={10} className="p-8 text-center text-sm text-on-surface-variant">加载中...</td></tr>
|
||||
) : pageLeads.length === 0 ? (
|
||||
<tr><td colSpan={11} className="p-8 text-center text-sm text-on-surface-variant">暂无数据</td></tr>
|
||||
<tr><td colSpan={10} className="p-8 text-center text-sm text-on-surface-variant">暂无数据</td></tr>
|
||||
) : pageLeads.map(lead => {
|
||||
const stars = '★'.repeat(lead.star) + '☆'.repeat(5 - lead.star)
|
||||
return (
|
||||
@ -476,7 +476,6 @@ export default function InquiryPage({ onLeadDetail }: { onLeadDetail?: (id: stri
|
||||
<td className="p-4 text-right whitespace-nowrap">
|
||||
<span className={lead.amount === '待确认' ? 'text-[10px] text-slate-400' : 'text-[11px] font-medium'}>{lead.amount}</span>
|
||||
</td>
|
||||
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.quantity ? `${lead.quantity}${lead.unit || ''}` : '-'}</td>
|
||||
<td className="p-4 whitespace-nowrap"><span className={`px-2 py-0.5 text-[10px] font-medium rounded-full ${getStatusClass(lead.status)}`}>{lead.status}</span></td>
|
||||
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.deadline ? lead.deadline.slice(5, 10) : '-'}</td>
|
||||
<td className="p-4 text-[10px] text-slate-500 whitespace-nowrap">{lead.publishTime ? lead.publishTime.slice(0, 10) : '-'}</td>
|
||||
|
||||
@ -172,21 +172,8 @@ export default function ProductPage() {
|
||||
const batchSync = async () => {
|
||||
if (selectedIds.size === 0) return
|
||||
showToast(`正在同步 ${selectedIds.size} 个商品...`)
|
||||
await fetch(`${API}/batch-publish`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: Array.from(selectedIds) }),
|
||||
})
|
||||
setSelectedIds(new Set())
|
||||
await loadProducts()
|
||||
showToast('批量同步完成')
|
||||
}
|
||||
|
||||
const batchPublishToDirev = async () => {
|
||||
if (selectedIds.size === 0) return
|
||||
showToast(`正在上架 ${selectedIds.size} 个商品到 B2B 平台...`)
|
||||
try {
|
||||
const res = await fetch(`${API}/publish-to-direv`, {
|
||||
const res = await fetch(`${API}/batch-publish`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: Array.from(selectedIds) }),
|
||||
@ -195,9 +182,9 @@ export default function ProductPage() {
|
||||
if (!res.ok) throw new Error(data.message || data.error || '请求失败')
|
||||
setSelectedIds(new Set())
|
||||
await loadProducts()
|
||||
showToast(`上架成功: ${data.imported || 0} 条${data.errors?.length ? `, 失败 ${data.errors.length} 条` : ''}`)
|
||||
showToast(`同步成功: ${data.imported || 0} 条导入 B2B${data.errors?.length ? `, 失败 ${data.errors.length} 条` : ''}`)
|
||||
} catch (e: any) {
|
||||
showToast(`上架失败: ${e.message}`)
|
||||
showToast(`同步失败: ${e.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
@ -472,11 +459,6 @@ export default function ProductPage() {
|
||||
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" /></svg>
|
||||
批量同步
|
||||
</button>
|
||||
<button disabled={selectedIds.size === 0} onClick={batchPublishToDirev}
|
||||
className="px-3 py-1.5 text-[10px] font-medium bg-emerald-600 text-white rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:opacity-90">
|
||||
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M5 13l4 4L19 7" /></svg>
|
||||
上架 B2B
|
||||
</button>
|
||||
<button disabled={selectedIds.size === 0} onClick={batchGenerate}
|
||||
className="px-3 py-1.5 text-[10px] font-medium bg-white border border-purple-200 text-purple-700 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1 hover:bg-purple-50 transition-colors">
|
||||
<svg className="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" /></svg>
|
||||
|
||||
@ -799,9 +799,13 @@ export default function RuleCenterPage() {
|
||||
<td className="p-3">
|
||||
<button
|
||||
onClick={() => toggleCrawlerStatus(c.id, c.status)}
|
||||
className={`px-2 py-0.5 text-[10px] font-medium rounded-full border ${getStatusStyle(c.status)}`}
|
||||
className={`px-3 py-1 text-[11px] font-semibold rounded-md transition-all cursor-pointer ${
|
||||
c.status === 'active'
|
||||
? 'bg-amber-50 text-amber-700 hover:bg-amber-100 active:bg-amber-200'
|
||||
: 'bg-emerald-50 text-emerald-700 hover:bg-emerald-100 active:bg-emerald-200'
|
||||
}`}
|
||||
>
|
||||
{{ active: '暂停', paused: '启动', error: '重启' }[c.status] || c.status}
|
||||
{{ active: '点击暂停', paused: '点击启动', error: '重启' }[c.status] || c.status}
|
||||
</button>
|
||||
</td>
|
||||
<td className="p-3 text-xs text-on-surface-variant">
|
||||
|
||||
@ -80,7 +80,7 @@ sshpass -p 'Jubianshuzi@2026' ssh ubuntu@172.81.241.109 "
|
||||
|
||||
# 4. 验证
|
||||
curl -s -o /dev/null -w 'API: %{http_code}\n' http://172.81.241.109:4000/api/health
|
||||
curl -s -o /dev/null -w 'Web: %{http_code}\n' https://zoupeng.cloud
|
||||
curl -s -o /dev/null -w 'Web: %{http_code}\n' https://direv.cloud
|
||||
```
|
||||
|
||||
---
|
||||
@ -105,6 +105,8 @@ curl -s -o /dev/null -w 'Web: %{http_code}\n' https://zoupeng.cloud
|
||||
| 2026-06-22 | **端到端验证新字段入库**：rfqId=2068990783236042754，orderId=RFQ20260622173337369，MySQL 里 11 个商务字段都正确填充（purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls）。**唯一未入库：`payment_method` 字段为 NULL**，原因：devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表，没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功（注意日志可能有 `付款方式查询失败: ...` 警告）| — |
|
||||
| 2026-06-23 | **密钥再次重置 + 司库系统重发**：devbox 傍晚又把密钥重置（hash 前缀 `m3hRN → diPRva`，update_date 18:18:22）。用 `reset-secret` 端点再次自助重置拿到新密钥 `fcb53684845648e2a7fc1082f5245180cfa4eea1f5404176`。**用户反馈现象**：选了 `司库系统升级完善项目` 点一键发布，本地显示"发布成功"但 B2B 后台查不到。**根因**：MCP 鉴权失败后代码降级为本地假订单号（X+时间戳），让用户误以为成功。 | scripts/rotate-mcp-key.sh |
|
||||
| 2026-06-23 | **彻底移除降级本地模式 + MCP 失败弹警告**：(1) 后端 `publishToDirev` 不再写本地假 `inquiryNo=X...`；MCP 鉴权/调用失败时抛 `ServiceUnavailableException` 返回 503 + 详细错误信息；Lead 状态完全不动。(2) 前端 `doPublish` 收集每条失败原因（fetch `res.json().message`），失败时弹新 Modal（红框 + 错误列表 + 「去 MCP 配置」/「我知道了」按钮）。**端到端验证**：把密钥改成 `invalid_test_key_for_qa` → publish 返回 HTTP 503 + JSON 含 `appSecret 校验失败` + 引导文案；Lead 表 publish/inquiryNo/followLogs 全部不变。 | apps/api/src/lead/lead.service.ts, apps/web/src/components/InquiryPage.tsx |
|
||||
| 2026-06-25 | **生产部署（commit 49f9224）**：本地全量构建后 rsync 到 `/opt/pingtai/`；生产跑 `npx prisma db push + generate`；生产 `next build`（有 lockfile 警告但构建成功）；`pm2 restart pingtai-api pingtai-web --update-env`。**手工补齐生产 db 缺列**（prisma db push 路径错）：真生产 db 是 `/data/pingtai/dev.db`（不是 `/opt/pingtai/apps/api/prisma/dev.db`），缺 `Lead.regionDetail/quantity/unit/attachments/notes/notesExtractedAt/notesSource` + `Integration.companyUserId` 共 8 列。手工 `ALTER TABLE` 全部补齐，95 条 Lead 数据完整保留。**生产验证**：API + Web online 13m+；lead 95 条、product 75 条、crawler 3 条、inquiry 3 条都正常返回。**新建 Integration 配置**（生产 db 之前为空）：在生产创建 MCP 集成 id=751ab09c-e6ab-4c97-bd2d-2c317df32893，appId=app_9dd07cb4e0e24fccbecd。**遗留问题**：生产服务器 172.81.241.109 → devbox 100.111.84.16:3002 网络不通（ping 100% 丢包，curl 5s 超时）。MCP publish_rfq 端到端在生产跑不通（生产 connect 集成返回 `fetch failed`），publish 接口在生产会返回 503 + "未配置 B2B 集成"或"网络不通"，这是网络层面的限制不是代码问题。 | apps/api/src/lead/lead.service.ts, docs/本地开发工作流.md |
|
||||
| 2026-06-25 | **主域名切换：弃用 zoupeng.cloud，改用 direv.cloud**。**根因**：`zoupeng.cloud` DNS 解析到 `222.137.3.123`（旧/无效 IP，ping 通但 443 不通），而 `direv.cloud` DNS 正确解析到 `172.81.241.109`（实际生产服务器 IP）。**改动**：文档第 83 行 `https://zoupeng.cloud` → `https://direv.cloud`，rsync 同步到生产。**长期建议**：去腾讯云 DNSPod 控制台把 `zoupeng.cloud` 的 A 记录改到 `172.81.241.109`；生产 nginx 加 `zoupeng.cloud` 的 server block（同 direv.cloud），否则 `zoupeng.cloud` 永远打不开。AGENTS.md 里"主 `https://zoupeng.cloud`、备 `https://direv.cloud`"是错的，应反过来。 | docs/本地开发工作流.md |
|
||||
| 2026-06-22 | **devbox F002 升级完成 + 平台助手二次适配**：devbox 已把 publish_rfq schema 升级（W29.7, 补 11 商务字段），Java 后端 `InquiryServiceImpl.publish()` 也补全字段映射。平台助手这边把新参数加进 `publishToDirev` 的 `publishArgs`：deadline(直接传 Lead.deadline YYYY-MM-DD)、deliveryDate(自动 deadline+30 天)、shippingAddress(优先 Lead.regionDetail, fallback region)、contactName/Phone(从 Lead.contact/phone)、inquiryParty("郑州巨变")、freightFlag("含运费")、procurementType("定量采购")、invoiceType("增值税专票")、paymentMethod("货到付款") | apps/api/src/lead/lead.service.ts |
|
||||
| 2026-06-22 | **devbox 升级导致密钥重置**：`app_9dd07cb4e0e24fccbecd` 的 appSecret 被重置（MySQL 里 hash 前缀从 `$2a$10$iR3Rl` 变 `$2a$10$m3hRN`）。用 `POST /sys/openapi/app/reset-secret/{id}` 端点自助重置拿到新密钥 `d02dc00c6fbe4a34a5bf013134146c3f2759148c61a341d2`，本地集成密钥已轮换 | scripts/rotate-mcp-key.sh |
|
||||
| 2026-06-22 | **端到端验证新字段入库**：rfqId=2068990783236042754，orderId=RFQ20260622173337369，MySQL 里 11 个商务字段都正确填充（purchase_quantity=50, inquiry_valid_date=2026-08-31, deliver_date=2026-09-30, shipping_address, contact_name="郑钊", contact_phone=15532027557, inquiry_party="郑州巨变", freight_flag=1, invoice_type=2, procurement_type=1, urls）。**唯一未入库：`payment_method` 字段为 NULL**，原因：devbox 按企业 ID + 名称查 `tb_supplier_pay_type` 表，没找到 "货到付款" 这个字典项就会抛错但不影响 publish 成功（注意日志可能有 `付款方式查询失败: ...` 警告）| — |
|
||||
|
||||
132
scripts/extract_address_from_pdf.py
Normal file
132
scripts/extract_address_from_pdf.py
Normal file
@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
从招标公告 PDF 中提取地址信息。
|
||||
用法: python3 extract_address_from_pdf.py <pdf文件路径> [公告标题]
|
||||
输出: JSON {"region": "省/市", "regionDetail": "详细地址"}
|
||||
"""
|
||||
import fitz
|
||||
import re
|
||||
import json
|
||||
import sys
|
||||
|
||||
# 全国省份/直辖市/自治区名称（含简称、别称）
|
||||
PROVINCES = [
|
||||
'北京', '天津', '上海', '重庆',
|
||||
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||
'河南', '湖北', '湖南', '广东', '海南',
|
||||
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||
]
|
||||
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
|
||||
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||
|
||||
|
||||
def extract_province_best(text):
|
||||
"""从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）"""
|
||||
if not text:
|
||||
return ''
|
||||
text_clean = text.replace(' ', '')
|
||||
# 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截）
|
||||
for prov in SPECIAL_PROVS:
|
||||
if prov in text_clean:
|
||||
return prov
|
||||
# 2. 直辖市
|
||||
for prov in ['北京', '天津', '上海', '重庆']:
|
||||
if prov in text_clean:
|
||||
return prov
|
||||
# 3. "XX省"（如"陕西省"、"广东省"）
|
||||
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
|
||||
if m:
|
||||
return m.group(1)[:2]
|
||||
# 4. 两字省份关键词
|
||||
for prov in TWO_CHAR_PROVS:
|
||||
if prov in text_clean:
|
||||
return prov
|
||||
return ''
|
||||
|
||||
|
||||
def extract_city_from_text(text):
|
||||
"""从文本中提取城市名（XX市）"""
|
||||
if not text:
|
||||
return ''
|
||||
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
|
||||
if m:
|
||||
city = m.group(1)
|
||||
if city not in ['北京市', '天津市', '上海市', '重庆市']:
|
||||
return city
|
||||
return ''
|
||||
|
||||
|
||||
def extract_address(pdf_path: str, title: str = '') -> dict:
|
||||
pdf_text = ''
|
||||
try:
|
||||
doc = fitz.open(pdf_path)
|
||||
for page in doc:
|
||||
pdf_text += page.get_text()
|
||||
doc.close()
|
||||
except Exception:
|
||||
pdf_text = ''
|
||||
|
||||
result = {'region': '', 'regionDetail': ''}
|
||||
|
||||
# ── 1. 从PDF正文中提取详细地址 ──
|
||||
addr_patterns = [
|
||||
r'建设地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'项目地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'交货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'施工地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'项目地址[：:]\s*(.+?)(?:\n|$)',
|
||||
r'供货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'到货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'送货地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'工程地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'实施地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'服务地点[：:]\s*(.+?)(?:\n|$)',
|
||||
r'收货地址[：:]\s*(.+?)(?:\n|$)',
|
||||
r'所在地[：:]\s*(.+?)(?:\n|$)',
|
||||
]
|
||||
|
||||
addr = ''
|
||||
for p in addr_patterns:
|
||||
m = re.search(p, pdf_text)
|
||||
if m:
|
||||
addr = m.group(1).strip()
|
||||
if len(addr) >= 4:
|
||||
break
|
||||
addr = ''
|
||||
|
||||
if addr:
|
||||
result['regionDetail'] = addr
|
||||
prov = extract_province_best(addr)
|
||||
if prov:
|
||||
result['region'] = prov
|
||||
else:
|
||||
city = extract_city_from_text(addr)
|
||||
if city:
|
||||
result['region'] = city
|
||||
|
||||
# ── 2. 正文没提取到region，从整个PDF文本中搜索 ──
|
||||
if not result['region'] and pdf_text:
|
||||
prov = extract_province_best(pdf_text)
|
||||
if prov:
|
||||
result['region'] = prov
|
||||
|
||||
# ── 3. 还没提取到，从标题中搜索 ──
|
||||
if not result['region'] and title:
|
||||
prov = extract_province_best(title)
|
||||
if prov:
|
||||
result['region'] = prov
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print(json.dumps({'region': '', 'regionDetail': ''}))
|
||||
sys.exit(0)
|
||||
|
||||
pdf_path = sys.argv[1]
|
||||
title = sys.argv[2] if len(sys.argv) > 2 else ''
|
||||
result = extract_address(pdf_path, title)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
@ -107,7 +107,7 @@ def parse_list_page(html, type_label):
|
||||
"detail_url": BASE_URL + a_tag.get("href", ""),
|
||||
"notice_id": re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")).group(1) if re.search(r"/(\d+)\.jhtml", a_tag.get("href", "")) else "",
|
||||
"type": type_label,
|
||||
"publish_date": "",
|
||||
"publish_time": "",
|
||||
"deadline": "",
|
||||
"status": "",
|
||||
}
|
||||
@ -120,7 +120,7 @@ def parse_list_page(html, type_label):
|
||||
if "Gray" in cls:
|
||||
# 发布日期
|
||||
if re.match(r"\d{4}-\d{2}-\d{2}", text):
|
||||
notice["publish_date"] = text
|
||||
notice["publish_time"] = text
|
||||
elif "Green" in cls:
|
||||
# 正在报名
|
||||
notice["status"] = text.strip("[]")
|
||||
@ -132,7 +132,7 @@ def parse_list_page(html, type_label):
|
||||
em_tag = span.find("em", class_="Red")
|
||||
if em_tag:
|
||||
em_text = em_tag.get_text(strip=True)
|
||||
dl_match = re.search(r"文件下载截止[：:](.+)", em_text)
|
||||
dl_match = re.search(r"文件(?:发售|下载)截止[：:](.+)", em_text)
|
||||
if dl_match:
|
||||
notice["deadline"] = dl_match.group(1).strip()
|
||||
|
||||
@ -173,6 +173,39 @@ def save_state(state_path, state):
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def load_accumulated_state(output_base, state_filename, today_str):
|
||||
"""加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录"""
|
||||
accumulated = {}
|
||||
last_run_date = None
|
||||
state_dir = os.path.join(output_base, today_str)
|
||||
today_state = load_state(os.path.join(state_dir, state_filename))
|
||||
if today_state:
|
||||
for k, v in today_state.items():
|
||||
if k != "lastRun":
|
||||
accumulated[k] = v
|
||||
last_run = today_state.get("lastRun")
|
||||
if last_run:
|
||||
last_run_date = last_run[:10]
|
||||
try:
|
||||
for i in range(1, 31):
|
||||
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||
path = os.path.join(output_base, d, state_filename)
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
prev = json.load(f)
|
||||
except:
|
||||
continue
|
||||
for k, v in prev.items():
|
||||
if k != "lastRun" and k not in accumulated:
|
||||
accumulated[k] = v
|
||||
if k == "lastRun" and not last_run_date:
|
||||
last_run_date = str(v)[:10]
|
||||
except:
|
||||
pass
|
||||
return accumulated, last_run_date
|
||||
|
||||
|
||||
def generate_markdown(notices, output_dir, date_range, keyword=""):
|
||||
"""生成公告清单 Markdown"""
|
||||
lines = [
|
||||
@ -195,7 +228,7 @@ def generate_markdown(notices, output_dir, date_range, keyword=""):
|
||||
deadline_display = n['deadline'] if n['deadline'] else '-'
|
||||
lines.extend([
|
||||
f"## {i}. {n['title']}",
|
||||
f"- **发布日期**: {n['publish_date']}",
|
||||
f"- **发布日期**: {n['publish_time']}",
|
||||
f"- **公告类型**: {n['type']}",
|
||||
f"- **状态**: {status_display}",
|
||||
f"- **文件下载截止**: {deadline_display}",
|
||||
@ -225,15 +258,21 @@ def run():
|
||||
parser.add_argument("--all-types", action="store_true", help="抓取所有公告类型")
|
||||
parser.add_argument("--force", action="store_true", help="强制重新抓取")
|
||||
parser.add_argument("--keyword", default="", help="关键词过滤（匹配标题）")
|
||||
parser.add_argument("--max-pages", type=int, default=0, help="最大翻页数（0=不限制）")
|
||||
parser.add_argument("--max-pages", type=int, default=10, help="最大翻页数（0=不限制，默认10页≈160条）")
|
||||
args = parser.parse_args()
|
||||
|
||||
today = date.today()
|
||||
today_str = today.strftime("%Y-%m-%d")
|
||||
|
||||
# 加载历史已下载记录（跨天去重）
|
||||
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "cnncecp_downloaded.json", today_str)
|
||||
|
||||
if args.from_date:
|
||||
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
||||
elif not args.force and last_run_date:
|
||||
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
|
||||
else:
|
||||
date_from = today - timedelta(days=7)
|
||||
date_from = today
|
||||
|
||||
if args.to_date:
|
||||
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
||||
@ -247,12 +286,12 @@ def run():
|
||||
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
||||
|
||||
# 输出目录
|
||||
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
|
||||
output_dir = os.path.join(OUTPUT_BASE, today_str)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# 状态文件
|
||||
# 状态文件（使用累积状态做去重）
|
||||
state_file = os.path.join(output_dir, "cnncecp_downloaded.json")
|
||||
state = load_state(state_file)
|
||||
state = accumulated_state
|
||||
|
||||
all_notices = []
|
||||
new_count = 0
|
||||
@ -306,7 +345,7 @@ def run():
|
||||
type_total += 1
|
||||
|
||||
# 日期过滤
|
||||
if not match_date_range(notice["publish_date"], date_from, date_to):
|
||||
if not match_date_range(notice.get("publish_time", ""), date_from, date_to):
|
||||
type_filtered += 1
|
||||
filtered_count += 1
|
||||
continue
|
||||
@ -329,7 +368,7 @@ def run():
|
||||
state[notice_id] = {
|
||||
"title": notice["title"],
|
||||
"type": notice["type"],
|
||||
"publish_date": notice["publish_date"],
|
||||
"publish_time": notice["publish_time"],
|
||||
"deadline": notice["deadline"],
|
||||
"status": notice["status"],
|
||||
"detail_url": notice["detail_url"],
|
||||
@ -352,7 +391,8 @@ def run():
|
||||
print(f" ✅ {label} 完成: 新增 {type_new} | 跳过 {type_skip} | 过滤 {type_filtered}")
|
||||
print()
|
||||
|
||||
# 保存状态
|
||||
# 保存状态（含 lastRun 时间戳，用于下次增量爬取）
|
||||
state["lastRun"] = datetime.now().isoformat()
|
||||
save_state(state_file, state)
|
||||
|
||||
# 生成清单
|
||||
|
||||
@ -25,9 +25,14 @@ const http = require('http');
|
||||
const crypto = require('crypto');
|
||||
|
||||
// ========== 配置区 ==========
|
||||
const KEYWORD = '水电十一局';
|
||||
|
||||
// 支持命令行传入 --keyword（例如: node fetch-v1.2.js --keyword 电缆）
|
||||
const args = process.argv.slice(2)
|
||||
const keywordIdx = args.indexOf('--keyword')
|
||||
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
|
||||
|
||||
// 公告类型：可按需增减。支持的类型值见 API 返回的 announcementType 字段
|
||||
const TYPES = ['招采公告', '资格预审公告', '采购预告', '中标结果公示'];
|
||||
const TYPES = ['招采公告'];
|
||||
// 可选：留空则从上次抓取日期继续；有值则从指定日期开始
|
||||
const START_DATE = '';
|
||||
const END_DATE = new Date().toISOString().split('T')[0]; // 今天
|
||||
@ -220,6 +225,54 @@ async function runWithConcurrency(tasks, limit) {
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------- 地址提取 ----------
|
||||
|
||||
const PROVINCES = [
|
||||
'北京', '天津', '上海', '重庆',
|
||||
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||
'河南', '湖北', '湖南', '广东', '海南',
|
||||
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||
]
|
||||
const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
|
||||
const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||
|
||||
function extractRegionFromTitle(title) {
|
||||
if (!title) return ''
|
||||
// 匹配 "XX省"、"XX市"（直辖市）、"XX自治区"
|
||||
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
|
||||
if (m) {
|
||||
const region = m[1]
|
||||
if (region.includes('新疆')) return '新疆'
|
||||
if (region.includes('广西')) return '广西'
|
||||
if (region.includes('内蒙古')) return '内蒙古'
|
||||
if (region.includes('宁夏')) return '宁夏'
|
||||
if (region.includes('西藏')) return '西藏'
|
||||
if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
|
||||
if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
|
||||
return ''
|
||||
}
|
||||
// 直接匹配省份关键词
|
||||
for (const prov of SPECIAL_PROVS) {
|
||||
if (title.includes(prov)) return prov
|
||||
}
|
||||
for (const prov of TWO_CHAR_PROVS) {
|
||||
if (title.includes(prov)) return prov
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function extractCityFromTitle(title) {
|
||||
if (!title) return ''
|
||||
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
|
||||
if (m) {
|
||||
const city = m[1]
|
||||
if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
// ---------- 生成公告清单 ----------
|
||||
|
||||
function buildAttachmentName(r, index) {
|
||||
@ -332,9 +385,11 @@ async function main() {
|
||||
|
||||
log(`共获取 ${stats.total} 条公告（分 ${page} 页）`);
|
||||
|
||||
// 过滤目标公告类型（由 TYPES 配置定义）
|
||||
const caigou = allAnnouncements.filter(r => TYPES.includes(r.announcementType));
|
||||
log(`其中 ${TYPES.join('、')}: ${caigou.length} 条`);
|
||||
// 过滤目标公告类型（由 TYPES 配置定义），且仅取货物类
|
||||
const caigou = allAnnouncements.filter(r =>
|
||||
TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
|
||||
);
|
||||
log(`其中 ${TYPES.join('、')}（货物类）: ${caigou.length} 条`);
|
||||
|
||||
// 生成公告清单（所有本次抓到的招采公告，无论是否已下载）
|
||||
const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
|
||||
@ -363,7 +418,7 @@ async function main() {
|
||||
if (!picUrl) {
|
||||
stats.skipNoAttachment++;
|
||||
log(`跳过（无附件）: ${r.title}`, 'WARN');
|
||||
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', time: new Date().toISOString() };
|
||||
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -376,15 +431,20 @@ async function main() {
|
||||
|
||||
tasks.push(async () => {
|
||||
log(`下载 [${shortName}]: ${fname}`);
|
||||
|
||||
// 从标题中提取省份（详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取）
|
||||
const region = extractRegionFromTitle(r.title) || ''
|
||||
const regionDetail = extractCityFromTitle(r.title) || ''
|
||||
|
||||
try {
|
||||
await downloadFile(picUrl, fpath);
|
||||
const size = fs.statSync(fpath).size;
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', path: `pdf/${fname}`, size, time: new Date().toISOString() };
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
|
||||
stats.success++;
|
||||
log(` ✅ ${fname} (${size} bytes)`);
|
||||
log(` ✅ ${fname} (${size} bytes) region=${region || '-'} addr=${regionDetail || '-'}`);
|
||||
} catch (err) {
|
||||
stats.failed++;
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', failed: true, error: err.message, time: new Date().toISOString() };
|
||||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
|
||||
log(` ❌ ${fname}: ${err.message}`, 'ERROR');
|
||||
}
|
||||
});
|
||||
|
||||
@ -150,6 +150,41 @@ def save_state(state_path, state):
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def load_accumulated_state(output_base, state_filename, today_str):
|
||||
"""加载历史状态实现跨天去重：合并所有历史状态文件中的已下载记录"""
|
||||
accumulated = {}
|
||||
last_run_date = None
|
||||
state_dir = os.path.join(output_base, today_str)
|
||||
# 先加载今日状态
|
||||
today_state = load_state(os.path.join(state_dir, state_filename))
|
||||
if today_state:
|
||||
for k, v in today_state.items():
|
||||
if k != "lastRun":
|
||||
accumulated[k] = v
|
||||
last_run = today_state.get("lastRun")
|
||||
if last_run:
|
||||
last_run_date = last_run[:10]
|
||||
# 扫描历史目录（最近 30 天），合并之前下载的记录
|
||||
try:
|
||||
for i in range(1, 31):
|
||||
d = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||
path = os.path.join(output_base, d, state_filename)
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
prev = json.load(f)
|
||||
except:
|
||||
continue
|
||||
for k, v in prev.items():
|
||||
if k != "lastRun" and k not in accumulated:
|
||||
accumulated[k] = v
|
||||
if k == "lastRun" and not last_run_date:
|
||||
last_run_date = str(v)[:10]
|
||||
except:
|
||||
pass
|
||||
return accumulated, last_run_date
|
||||
|
||||
|
||||
def generate_markdown(notices, output_dir, date_range):
|
||||
"""生成公告清单 Markdown"""
|
||||
lines = [
|
||||
@ -195,10 +230,16 @@ def run():
|
||||
|
||||
today = date.today()
|
||||
|
||||
# 加载历史已下载记录（跨天去重）
|
||||
today_str = today.strftime("%Y-%m-%d")
|
||||
accumulated_state, last_run_date = load_accumulated_state(OUTPUT_BASE, "xkgroup_downloaded.json", today_str)
|
||||
|
||||
if args.from_date:
|
||||
date_from = datetime.strptime(args.from_date, "%Y-%m-%d").date()
|
||||
elif not args.force and last_run_date:
|
||||
date_from = datetime.strptime(last_run_date, "%Y-%m-%d").date()
|
||||
else:
|
||||
date_from = today - timedelta(days=7) # 默认抓取最近 7 天
|
||||
date_from = today - timedelta(days=7)
|
||||
|
||||
if args.to_date:
|
||||
date_to = datetime.strptime(args.to_date, "%Y-%m-%d").date()
|
||||
@ -212,7 +253,7 @@ def run():
|
||||
notice_types = list(NOTICE_TYPES.keys()) if args.all_types else DEFAULT_TYPES
|
||||
|
||||
# 输出目录
|
||||
output_dir = os.path.join(OUTPUT_BASE, today.strftime("%Y-%m-%d"))
|
||||
output_dir = os.path.join(OUTPUT_BASE, today_str)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# 子目录：html（源码） + pdf（转存）
|
||||
@ -221,9 +262,9 @@ def run():
|
||||
os.makedirs(html_dir, exist_ok=True)
|
||||
os.makedirs(pdf_dir, exist_ok=True)
|
||||
|
||||
# 状态文件
|
||||
# 状态文件（使用累积状态做去重）
|
||||
state_file = os.path.join(output_dir, "xkgroup_downloaded.json")
|
||||
state = load_state(state_file)
|
||||
state = accumulated_state
|
||||
|
||||
all_notices = []
|
||||
new_count = 0
|
||||
@ -268,6 +309,8 @@ def run():
|
||||
type_skip += 1
|
||||
continue
|
||||
|
||||
# 从标题/组织名初步提取地区（详情页提取时会补充/覆盖）
|
||||
title_region = extract_region_from_text(notice["title"], notice["org"])
|
||||
all_notices.append(notice)
|
||||
state[notice_id] = {
|
||||
"title": notice["title"],
|
||||
@ -276,6 +319,7 @@ def run():
|
||||
"type": notice["type"],
|
||||
"detail_url": notice["detail_url"],
|
||||
"downloaded_at": datetime.now().isoformat(),
|
||||
"region": title_region,
|
||||
}
|
||||
new_count += 1
|
||||
type_new += 1
|
||||
@ -283,7 +327,8 @@ def run():
|
||||
print(f" 新增: {type_new} | 跳过: {type_skip}")
|
||||
print()
|
||||
|
||||
# 保存状态
|
||||
# 保存状态（含 lastRun 时间戳，用于下次增量爬取）
|
||||
state["lastRun"] = datetime.now().isoformat()
|
||||
save_state(state_file, state)
|
||||
|
||||
# 为缺少金额的记录从详情页提取
|
||||
@ -296,7 +341,9 @@ def run():
|
||||
# 详情页转存 PDF + HTML（新公告 + 已有状态中缺文件的）
|
||||
all_pending = list(all_notices)
|
||||
if not args.force:
|
||||
for notice_id, info in state.items():
|
||||
for notice_id, info in list(state.items()):
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
pdf_key = info.get("pdf", "")
|
||||
html_key = info.get("html", "")
|
||||
pdf_ok = pdf_key and os.path.exists(os.path.join(output_dir, pdf_key))
|
||||
@ -323,6 +370,129 @@ def run():
|
||||
print(f"📁 保存目录: {output_dir}")
|
||||
|
||||
|
||||
def extract_deadline_from_text(text):
|
||||
"""从公告正文中提取截止日期（采购文件获取截止时间）"""
|
||||
if not text:
|
||||
return ""
|
||||
# 模式1: "起至2026年6月22日19:00（北京时间）止" → 取截止日期
|
||||
m = re.search(r"起至(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2}):(\d{2})", text)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||
# 模式2: "获取时间：...至2026-06-22 19:00"
|
||||
m = re.search(r"至\s*(\d{4})-(\d{1,2})-(\d{1,2})\s+\d{1,2}:\d{2}", text)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||
# 模式3: "截止时间：2026年6月22日" 或 "截止日期：2026-06-22"
|
||||
for pat in [
|
||||
r"截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
|
||||
r"截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
|
||||
r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日",
|
||||
r"递交[文件]*截止[时间日期]*[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})",
|
||||
]:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
|
||||
return ""
|
||||
|
||||
|
||||
# 全国省份/直辖市/自治区
|
||||
PROVINCES = [
|
||||
'北京', '天津', '上海', '重庆',
|
||||
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||||
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||||
'河南', '湖北', '湖南', '广东', '海南',
|
||||
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||||
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||||
]
|
||||
TWO_CHAR_PROVS = [p for p in PROVINCES if len(p) == 2]
|
||||
SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||||
|
||||
|
||||
def extract_province_best(text):
|
||||
"""从任意文本中提取最可能的省份（先关键词匹配，避免正则在多字省份上截断）"""
|
||||
if not text:
|
||||
return ''
|
||||
text_clean = text.replace(' ', '')
|
||||
# 1. 特殊多字省份（优先，避免"新疆维吾尔自治区"被截）
|
||||
for sp in SPECIAL_PROVS:
|
||||
if sp in text_clean:
|
||||
return sp
|
||||
# 2. 直辖市
|
||||
for dc in ['北京', '天津', '上海', '重庆']:
|
||||
if dc in text_clean:
|
||||
return dc
|
||||
# 3. "XX省"（如"陕西省"、"广东省"）
|
||||
m = re.search(r'([\u4e00-\u9fff]{2}省)', text)
|
||||
if m:
|
||||
return m.group(1)[:2]
|
||||
# 4. 两字省份关键词
|
||||
for p in TWO_CHAR_PROVS:
|
||||
if p in text_clean:
|
||||
return p
|
||||
return ''
|
||||
|
||||
|
||||
def extract_city_from_text(text):
|
||||
"""从文本中提取城市名（XX市）"""
|
||||
if not text:
|
||||
return ''
|
||||
m = re.search(r'([\u4e00-\u9fff]{2,4}(?:市))', text)
|
||||
if m:
|
||||
city = m.group(1)
|
||||
if city not in ['北京市', '天津市', '上海市', '重庆市']:
|
||||
return city
|
||||
return ''
|
||||
|
||||
|
||||
def extract_region_from_text(text, org=""):
|
||||
"""从公告正文或公司名中提取地区（省份）"""
|
||||
if not text and not org:
|
||||
return ""
|
||||
# 1. 从正文找项目/交货/施工地点
|
||||
if text:
|
||||
for pat in [
|
||||
r"(?:项目|交货|送货|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
|
||||
r"(?:地[址点]|所在地区)[：:]\s*([\u4e00-\u9fff]{2,8}(?:省|市|自治区))",
|
||||
r"位\s*于\s*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
|
||||
r"(?:项目|工程)\s*(?:位于|所在地|所在)[：:：\s]*([\u4e00-\u9fff]{2,4}(?:省|市|自治区))",
|
||||
]:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
region = m.group(1)
|
||||
for sp in SPECIAL_PROVS:
|
||||
if sp in region:
|
||||
return sp
|
||||
return region[:2] # 取前两个字
|
||||
# 2. 从全文扫描省份关键词
|
||||
prov = extract_province_best(text)
|
||||
if prov:
|
||||
return prov
|
||||
# 3. 从公司名推断
|
||||
if org:
|
||||
org_prov = extract_province_best(org)
|
||||
if org_prov:
|
||||
return org_prov
|
||||
return ""
|
||||
|
||||
|
||||
def extract_region_detail_from_text(text):
|
||||
"""从公告正文中提取详细收货/交货地址"""
|
||||
if not text:
|
||||
return ""
|
||||
for pat in [
|
||||
r"(?:交货|收货|送货|项目|施工|到货|供货|工程|实施|服务)[地址地点]*[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
|
||||
r"(?:地[址点])[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}(?:路|街|道|巷|号|村|镇|乡|区|县|市|大厦|楼|园|区|厂|公司|仓库))(?:\n|$)",
|
||||
r"建设地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
|
||||
r"项目地点[：:]\s*([\u4e00-\u9fff\d\-—\s]{6,60}?)(?:\n|$)",
|
||||
]:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
addr = m.group(1).strip()
|
||||
if len(addr) >= 6:
|
||||
return addr
|
||||
return ""
|
||||
|
||||
|
||||
def extract_amount_from_text(text):
|
||||
"""从公告正文中提取金额（万元）"""
|
||||
if not text:
|
||||
@ -367,6 +537,8 @@ def extract_amounts_for_existing(state, output_dir):
|
||||
for notice_id, info in state.items():
|
||||
if notice_id == "lastRun":
|
||||
continue
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
if info.get("amount"):
|
||||
continue
|
||||
pending.append(notice_id)
|
||||
@ -396,12 +568,27 @@ def extract_amounts_for_existing(state, output_dir):
|
||||
await page.close()
|
||||
|
||||
amount = extract_amount_from_text(text)
|
||||
deadline = extract_deadline_from_text(text)
|
||||
region = extract_region_from_text(text, info.get("org", ""))
|
||||
region_detail = extract_region_detail_from_text(text)
|
||||
updated = False
|
||||
if amount:
|
||||
state[notice_id]["amount"] = amount
|
||||
updated = True
|
||||
if deadline:
|
||||
state[notice_id]["deadline"] = deadline
|
||||
updated = True
|
||||
if region:
|
||||
state[notice_id]["region"] = region
|
||||
updated = True
|
||||
if region_detail:
|
||||
state[notice_id]["region_detail"] = region_detail
|
||||
updated = True
|
||||
if updated:
|
||||
success += 1
|
||||
print(f"✅ {amount}")
|
||||
print(f"✅ amount={amount or '-'} deadline={deadline or '-'} region={region or '-'} addr={region_detail or '-'}")
|
||||
else:
|
||||
print("⏭️ 未找到金额")
|
||||
print("⏭️ 未找到金额/截止日期/地区")
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
|
||||
@ -472,6 +659,15 @@ def download_pdfs(notices, output_dir, html_dir, pdf_dir, state=None, state_file
|
||||
amount = extract_amount_from_text(page_text)
|
||||
if amount and state is not None and notice_id in state:
|
||||
state[notice_id]["amount"] = amount
|
||||
deadline = extract_deadline_from_text(page_text)
|
||||
if deadline and state is not None and notice_id in state:
|
||||
state[notice_id]["deadline"] = deadline
|
||||
region = extract_region_from_text(page_text, notice.get("org", ""))
|
||||
if region and state is not None and notice_id in state:
|
||||
state[notice_id]["region"] = region
|
||||
region_detail = extract_region_detail_from_text(page_text)
|
||||
if region_detail and state is not None and notice_id in state:
|
||||
state[notice_id]["region_detail"] = region_detail
|
||||
|
||||
# 保存 HTML 源码到 html/ 目录（供 AI 阅读）
|
||||
html_source = await page.content()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user