platform-assistant/scripts/tender-fetcher/fetch-v1.2.js
Sisyphus eb15849168 feat: 询价标题AI压缩 + 采集器品类化 + 询价搜索与品类多选框
- TitleCompressorService: 发布前超长标题(>50字)调 MiniMax 压缩(保留公司名+物料+采购类型), JSON结构化输出对抗推理模型思考噪音, 失败硬截断兜底
- publishToDirev 集成标题压缩, 保留原50字预校验作为防御
- crawler resolveKeywords: 采集器配置优先按关联品类展开关键词库, 中电建每关键词独立状态文件
- InquiryPage: 新增关键词搜索(fSearch → GET /api/leads?search=)
- RuleCenterPage: 采集器配置由 keywords 改为品类多选框(categories JSON数组)
2026-09-30 09:34:58 +08:00

493 lines
18 KiB
JavaScript
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env node
/**
* 中电建阳光采购网 - 招采公告抓取脚本 v1.2
*
* v1.2 新增:
* - 支持多公告类型:招采公告、资格预审公告、采购预告、中标结果公示
* - 类型在配置区可自定义增减
*
* 改进点(继承v1.1):
* 1. 分页逻辑 - 自动遍历所有页
* 2. 去重机制 - 基于 systemId 持久化记录
* 3. 空附件处理 - pictureUrl 为空时跳过并告警
* 4. 稳定文件名 - 基于 systemId + 内容 hash
* 5. 并发下载 - 控制并发数,避免被限流
* 6. 失败重试 - 3次重试,间隔递增
* 7. 状态持久化 - downloaded.json 记录已下载文件
* 8. 差量更新 - 只抓自上次运行以来的新增公告
* 9. 详细报告 - 成功/失败/跳过数量统计
*/
const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');
const crypto = require('crypto');
// ========== 配置区 ==========
// 支持命令行传入 --keyword(例如: node fetch-v1.2.js --keyword 电缆)
const args = process.argv.slice(2)
const keywordIdx = args.indexOf('--keyword')
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
// 公告类型:与 API 返回的 announcementType 字段值严格一致
const TYPES = ['招采公告', '变更公告'];
// 可选:留空则从上次抓取日期继续;有值则从指定日期开始(支持环境变量覆盖)
const START_DATE = process.env.START_DATE || '';
const END_DATE = new Date().toISOString().split('T')[0]; // 今天
// 跨天首次运行(state 为空)时回溯 N 天, 避免错过前一天后半天发布的公告
const BACKFILL_DAYS = 3;
const OUTPUT_DIR = path.join(process.env.HOME, 'desktop/招标公告', END_DATE);
const PDF_DIR = path.join(OUTPUT_DIR, 'pdf'); // PDF 附件
const HTML_DIR = path.join(OUTPUT_DIR, 'html'); // HTML 转换(待安装转换技能)
// 每个关键词使用独立的状态文件,避免多关键词共用状态导致后续关键词跳过
const STATE_FILE = KEYWORD
? path.join(OUTPUT_DIR, `downloaded_${KEYWORD}.json`)
: path.join(OUTPUT_DIR, 'downloaded.json');
const LOG_FILE = path.join(OUTPUT_DIR, 'fetch.log'); // 运行日志
const CONCURRENCY = 3; // 并发下载数
const RETRY_TIMES = 3;
const RETRY_DELAY_BASE = 1000; // ms
// ==========================
// ---------- 工具函数 ----------
function log(msg, type = 'INFO') {
const ts = new Date().toISOString().replace('T', ' ').substring(0, 19);
const line = `[${ts}] [${type}] ${msg}`;
console.log(line);
fs.appendFileSync(LOG_FILE, line + '\n');
}
function loadState() {
try {
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
} catch {
return { lastRun: null, downloaded: {}, lastId: null };
}
}
function saveState(state) {
fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
}
function md5(str) {
return crypto.createHash('md5').update(str).digest('hex').substring(0, 8);
}
function sanitizeFilename(name, maxLen = 50) {
return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_').replace(/\s+/g, '_').substring(0, maxLen);
}
function fileExtFromUrl(url) {
if (!url) return null;
const m = url.match(/fileType=(\d+)/);
const typeMap = { 2: '.pdf', 3: '.pdf', 4: '.pdf', 6: '.pdf', 7: '.zip' };
return m ? (typeMap[m[1]] || '.pdf') : (path.extname(new URL(url).pathname) || '.pdf');
}
// ---------- HTTP 请求 ----------
function request(url, options = {}, retries = RETRY_TIMES) {
return new Promise((resolve, reject) => {
const parsedUrl = new URL(url);
const protocol = parsedUrl.protocol === 'https:' ? https : http;
const reqOptions = {
hostname: parsedUrl.hostname,
path: parsedUrl.pathname + parsedUrl.search,
method: options.method || 'GET',
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'zh-CN,zh;q=0.9',
...options.headers,
},
timeout: 15000,
};
const req = protocol.request(reqOptions, (res) => {
const chunks = [];
res.on('data', chunk => chunks.push(chunk));
res.on('end', () => {
const data = Buffer.concat(chunks).toString('utf8');
if (res.statusCode === 301 || res.statusCode === 302) {
const location = res.headers.location;
if (location.startsWith('http')) {
request(location, options, retries).then(resolve).catch(reject);
} else {
const base = `${parsedUrl.protocol}//${parsedUrl.host}`;
request(base + location, options, retries).then(resolve).catch(reject);
}
return;
}
if (res.statusCode !== 200) {
reject(new Error(`HTTP ${res.statusCode}`));
return;
}
try {
resolve(JSON.parse(data));
} catch {
resolve(data);
}
});
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
if (options.body) req.write(options.body);
req.end();
}).catch(err => {
if (retries > 0) {
const delay = RETRY_DELAY_BASE * (RETRY_TIMES - retries + 1);
return new Promise(r => setTimeout(r, delay))
.then(() => request(url, options, retries - 1));
}
throw err;
});
}
// ---------- API 调用 ----------
async function fetchAnnouncements(keyword, startDate, endDate, page = 1, pageSize = 50) {
const postData = JSON.stringify({
keyWords: keyword,
pageNum: page,
pageSize,
publishStartTime: startDate ? `${startDate} 00:00:00` : '',
publishEndTime: `${endDate} 23:59:59`,
});
return request('https://bid.powerchina.cn/newcbs/recpro-newmember/BidAnnouncementSummary/list', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Origin': 'https://bid.powerchina.cn',
'Referer': 'https://bid.powerchina.cn/search',
},
body: postData,
});
}
// ---------- 下载文件 ----------
async function downloadFile(fileUrl, destPath, retries = RETRY_TIMES) {
return new Promise((resolve, reject) => {
const parsedUrl = new URL(fileUrl);
const protocol = parsedUrl.protocol === 'https:' ? https : http;
const file = fs.createWriteStream(destPath);
const req = protocol.request({
hostname: parsedUrl.hostname,
path: parsedUrl.pathname + parsedUrl.search,
method: 'GET',
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Referer': 'https://bid.powerchina.cn/',
},
timeout: 30000,
}, (res) => {
if (res.statusCode === 301 || res.statusCode === 302) {
file.close();
downloadFile(res.headers.location.startsWith('http')
? res.headers.location
: `${parsedUrl.protocol}//${parsedUrl.host}${res.headers.location}`,
destPath, retries
).then(resolve).catch(reject);
return;
}
if (res.statusCode !== 200) {
file.close();
reject(new Error(`Download HTTP ${res.statusCode}`));
return;
}
res.pipe(file);
file.on('finish', () => { file.close(); resolve(); });
});
req.on('error', (err) => { file.close(); reject(err); });
req.on('timeout', () => { req.destroy(); file.close(); reject(new Error('Download timeout')); });
req.end();
}).catch(err => {
if (retries > 0) {
const delay = RETRY_DELAY_BASE * (RETRY_TIMES - retries + 1);
return new Promise(r => setTimeout(r, delay))
.then(() => downloadFile(fileUrl, destPath, retries - 1));
}
throw err;
});
}
// ---------- 并发控制 ----------
async function runWithConcurrency(tasks, limit) {
const results = [];
for (let i = 0; i < tasks.length; i += limit) {
const batch = tasks.slice(i, i + limit);
const batchResults = await Promise.allSettled(batch.map(t => t()));
results.push(...batchResults);
}
return results;
}
// ---------- 地址提取 ----------
const PROVINCES = [
'北京', '天津', '上海', '重庆',
'河北', '山西', '辽宁', '吉林', '黑龙江',
'江苏', '浙江', '安徽', '福建', '江西', '山东',
'河南', '湖北', '湖南', '广东', '海南',
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
]
const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
function extractRegionFromTitle(title) {
if (!title) return ''
// 匹配 "XX省"、"XX市"(直辖市)、"XX自治区"
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
if (m) {
const region = m[1]
if (region.includes('新疆')) return '新疆'
if (region.includes('广西')) return '广西'
if (region.includes('内蒙古')) return '内蒙古'
if (region.includes('宁夏')) return '宁夏'
if (region.includes('西藏')) return '西藏'
if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
return ''
}
// 直接匹配省份关键词
for (const prov of SPECIAL_PROVS) {
if (title.includes(prov)) return prov
}
for (const prov of TWO_CHAR_PROVS) {
if (title.includes(prov)) return prov
}
return ''
}
function extractCityFromTitle(title) {
if (!title) return ''
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
if (m) {
const city = m[1]
if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
}
return ''
}
// ---------- 生成公告清单 ----------
function buildAttachmentName(r, index) {
const ext = fileExtFromUrl(r.pictureUrl);
const dateStr = (r.publishTime || '').substring(0, 10) || END_DATE;
const shortTitle = sanitizeFilename(r.title, 40);
return `${dateStr}_${index}_${shortTitle}${ext}`;
}
function formatDate(dt) {
if (!dt) return '-';
// 兼容 "2026-04-13T18:55:00" 和 "2026-04-13 18:55" 两种格式
const normalized = dt.includes('T') ? dt.replace('T', ' ').substring(0, 16) : dt.substring(0, 16);
return normalized;
}
function generateAnnouncementList(announcements, sinceDate, endDate) {
const lines = [];
const count = announcements.length;
const today = new Date().toISOString().substring(0, 10);
// 标题
// 标题:去掉日期中的无效前导零(如 04→4),让范围更易读
const fmtDate = d => d.replace(/-0(\d)/g, '-$1');
const rangeLabel = sinceDate === END_DATE ? '当天' : `${fmtDate(sinceDate)}至${fmtDate(END_DATE)}`;
lines.push(`# ${KEYWORD} 近${rangeLabel}采购公告`);
lines.push('');
lines.push(`**筛选条件**:${KEYWORD} | ${sinceDate} ~ ${endDate} | ${TYPES.join('、')}`);
lines.push('');
lines.push(`**公告总数**:${count}条`);
lines.push('');
lines.push(`> 数据来源:中电建阳光采购网 https://bid.powerchina.cn`);
lines.push(`> 下载时间:${today}`);
lines.push('');
lines.push('---');
lines.push('');
announcements.forEach((r, i) => {
const n = i + 1;
const attachmentName = buildAttachmentName(r, n);
const deadlineReg = formatDate(r.registrationDeadline);
const deadlineSub = formatDate(r.submissionDeadline);
const bidTime = formatDate(r.bidOpenTime);
const publishTime = formatDate(r.publishTime);
lines.push(`## ${n}. ${r.title}`);
lines.push(`- **发布日期**:\`${publishTime}\``);
lines.push(`- **公告ID**:\`${r.id}\`(systemId: \`${r.systemId}\`)`);
lines.push(`- **详情链接**:https://bid.powerchina.cn/notice/detail?id=${r.id}`);
lines.push(`- **采购分类**:${r.titleTypeName || '-'}`);
lines.push(`- **报名截止**:\`${deadlineReg}\``);
lines.push(`- **投标截止**:\`${deadlineSub}\``);
lines.push(`- **开标时间**:\`${bidTime}\``);
lines.push(`- **附件**:\`pdf/${attachmentName}\``);
lines.push('');
});
return lines.join('\n');
}
// ---------- 主流程 ----------
async function main() {
// 初始化
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
fs.mkdirSync(PDF_DIR, { recursive: true });
fs.mkdirSync(HTML_DIR, { recursive: true });
if (!fs.existsSync(LOG_FILE)) fs.writeFileSync(LOG_FILE, '');
const state = loadState();
const stats = { total: 0, new: 0, skipDuplicate: 0, skipNoAttachment: 0, success: 0, failed: 0, retried: 0 };
log(`========== 开始抓取: ${KEYWORD} ==========`);
// 确定起始日期:START_DATE 为空时退缩到上次运行的前 1 天(重叠 24 小时,覆盖 cron 间隔内漏抓的公告)
// state.downloaded 通过 systemId 去重,重叠不会造成重复入库
let sinceDate = START_DATE || (state.lastRun ? state.lastRun.split('T')[0] : null) || (() => {
const d = new Date(Date.now() - BACKFILL_DAYS * 86400 * 1000);
return d.toISOString().split('T')[0];
})();
// 增加 1 天回溯,覆盖 lastRun 之后但在 cron 间隔内发布的公告
if (!START_DATE && state.lastRun) {
const yesterday = new Date(Date.now() - 86400 * 1000).toISOString().split('T')[0];
sinceDate = yesterday;
}
if (START_DATE) {
log(`指定日期范围: ${START_DATE} ~ ${END_DATE}`);
} else if (state.lastRun) {
log(`增量模式(含 1 天回溯): ${sinceDate} ~ ${END_DATE}`);
} else if (sinceDate !== END_DATE) {
log(`跨天补抓: ${sinceDate} ~ ${END_DATE}`);
} else {
log(`首次运行: ${sinceDate} ~ ${END_DATE}`);
}
// 分页获取所有公告
let page = 1;
let hasMore = true;
let allAnnouncements = [];
while (hasMore) {
log(`获取第 ${page} 页...`);
let data;
try {
data = await fetchAnnouncements(KEYWORD, sinceDate, END_DATE, page, 50);
} catch (err) {
log(`第 ${page} 页请求失败: ${err.message}`, 'ERROR');
break;
}
const rows = Array.isArray(data) ? data : (data.rows || []);
allAnnouncements.push(...rows);
stats.total += rows.length;
if (rows.length < 50 || !data.total || allAnnouncements.length >= data.total) {
hasMore = false;
} else {
page++;
await new Promise(r => setTimeout(r, 500)); // 防止请求过快
}
}
log(`共获取 ${stats.total} 条公告(分 ${page} 页)`);
// 过滤目标公告类型(由 TYPES 配置定义),且仅取货物类
const caigou = allAnnouncements.filter(r =>
TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
);
log(`其中 ${TYPES.join('、')}(货物类): ${caigou.length} 条`);
// 生成公告清单(所有本次抓到的招采公告,无论是否已下载)
const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
const listFileName = `公告清单_${KEYWORD}_${sinceDate}-${END_DATE}.md`;
const listPath = path.join(OUTPUT_DIR, listFileName);
fs.writeFileSync(listPath, listMd, 'utf8');
log(`📄 公告清单已生成: ${listFileName}`);
// 构建下载任务(只下载未记录的)
const tasks = [];
for (const r of caigou) {
const systemId = r.systemId;
if (!systemId) {
stats.skipNoAttachment++;
log(`跳过(无systemId): ${r.title}`, 'WARN');
continue;
}
const key = `${systemId}_${r.announcementType}`;
if (state.downloaded[key]) {
stats.skipDuplicate++;
continue;
}
const picUrl = r.pictureUrl;
if (!picUrl) {
stats.skipNoAttachment++;
log(`跳过(无附件): ${r.title}`, 'WARN');
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
continue;
}
const ext = fileExtFromUrl(picUrl);
const dateStr = (r.publishTime || '').substring(0, 10) || END_DATE;
const shortTitle = sanitizeFilename(r.title, 40);
const fname = `${dateStr}_${systemId.substring(0, 8)}_${shortTitle}${ext}`;
const fpath = path.join(PDF_DIR, fname);
const shortName = sanitizeFilename(r.title, 20);
tasks.push(async () => {
log(`下载 [${shortName}]: ${fname}`);
// 从标题中提取省份(详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取)
const region = extractRegionFromTitle(r.title) || ''
const regionDetail = extractCityFromTitle(r.title) || ''
try {
await downloadFile(picUrl, fpath);
const size = fs.statSync(fpath).size;
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
stats.success++;
log(` ✅ ${fname} (${size} bytes) region=${region || '-'} addr=${regionDetail || '-'}`);
} catch (err) {
stats.failed++;
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
log(` ❌ ${fname}: ${err.message}`, 'ERROR');
}
});
}
// 并发执行
if (tasks.length > 0) {
log(`开始下载 ${tasks.length} 个附件(并发数: ${CONCURRENCY})...`);
await runWithConcurrency(tasks, CONCURRENCY);
}
// 更新状态
state.lastRun = new Date().toISOString();
state.lastKeyword = KEYWORD;
saveState(state);
// 输出统计
log(`========== 完成 ==========`);
log(`总计公告: ${stats.total} | ${TYPES.join('、')}: ${caigou.length}`);
log(`新增下载: ${stats.success} | 重复跳过: ${stats.skipDuplicate} | 无附件跳过: ${stats.skipNoAttachment} | 下载失败: ${stats.failed}`);
log(`状态已保存: ${STATE_FILE}`);
console.log('\n========== 汇总 ==========');
console.log(`✅ 成功: ${stats.success} | ❌ 失败: ${stats.failed} | ⏩ 重复: ${stats.skipDuplicate} | ⚠️ 无附件: ${stats.skipNoAttachment}`);
console.log(`📁 保存目录: ${OUTPUT_DIR}`);
console.log(`📄 公告清单: ${listFileName}`);
}
main().catch(err => { console.error('Fatal:', err); process.exit(1); });