bd213eb5ec
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
476 lines
17 KiB
JavaScript
Executable File
476 lines
17 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
||
/**
|
||
* 中电建阳光采购网 - 招采公告抓取脚本 v1.2
|
||
*
|
||
* v1.2 新增：
|
||
* - 支持多公告类型：招采公告、资格预审公告、采购预告、中标结果公示
|
||
* - 类型在配置区可自定义增减
|
||
*
|
||
* 改进点（继承v1.1）：
|
||
* 1. 分页逻辑 - 自动遍历所有页
|
||
* 2. 去重机制 - 基于 systemId 持久化记录
|
||
* 3. 空附件处理 - pictureUrl 为空时跳过并告警
|
||
* 4. 稳定文件名 - 基于 systemId + 内容 hash
|
||
* 5. 并发下载 - 控制并发数，避免被限流
|
||
* 6. 失败重试 - 3次重试，间隔递增
|
||
* 7. 状态持久化 - downloaded.json 记录已下载文件
|
||
* 8. 差量更新 - 只抓自上次运行以来的新增公告
|
||
* 9. 详细报告 - 成功/失败/跳过数量统计
|
||
*/
|
||
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const https = require('https');
|
||
const http = require('http');
|
||
const crypto = require('crypto');
|
||
|
||
// ========== 配置区 ==========
|
||
|
||
// 支持命令行传入 --keyword（例如: node fetch-v1.2.js --keyword 电缆）
|
||
const args = process.argv.slice(2)
|
||
const keywordIdx = args.indexOf('--keyword')
|
||
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''
|
||
|
||
// 公告类型：可按需增减。支持的类型值见 API 返回的 announcementType 字段
|
||
const TYPES = ['招采公告'];
|
||
// 可选：留空则从上次抓取日期继续；有值则从指定日期开始
|
||
const START_DATE = '';
|
||
const END_DATE = new Date().toISOString().split('T')[0]; // 今天
|
||
const OUTPUT_DIR = path.join(process.env.HOME, 'desktop/招标公告', END_DATE);
|
||
const PDF_DIR = path.join(OUTPUT_DIR, 'pdf'); // PDF 附件
|
||
const HTML_DIR = path.join(OUTPUT_DIR, 'html'); // HTML 转换（待安装转换技能）
|
||
const STATE_FILE = path.join(OUTPUT_DIR, 'downloaded.json'); // 已下载记录
|
||
const LOG_FILE = path.join(OUTPUT_DIR, 'fetch.log'); // 运行日志
|
||
const CONCURRENCY = 3; // 并发下载数
|
||
const RETRY_TIMES = 3;
|
||
const RETRY_DELAY_BASE = 1000; // ms
|
||
// ==========================
|
||
|
||
// ---------- 工具函数 ----------
|
||
|
||
function log(msg, type = 'INFO') {
|
||
const ts = new Date().toISOString().replace('T', ' ').substring(0, 19);
|
||
const line = `[${ts}] [${type}] ${msg}`;
|
||
console.log(line);
|
||
fs.appendFileSync(LOG_FILE, line + '\n');
|
||
}
|
||
|
||
function loadState() {
|
||
try {
|
||
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
|
||
} catch {
|
||
return { lastRun: null, downloaded: {}, lastId: null };
|
||
}
|
||
}
|
||
|
||
function saveState(state) {
|
||
fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
|
||
}
|
||
|
||
function md5(str) {
|
||
return crypto.createHash('md5').update(str).digest('hex').substring(0, 8);
|
||
}
|
||
|
||
function sanitizeFilename(name, maxLen = 50) {
|
||
return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_').replace(/\s+/g, '_').substring(0, maxLen);
|
||
}
|
||
|
||
function fileExtFromUrl(url) {
|
||
if (!url) return null;
|
||
const m = url.match(/fileType=(\d+)/);
|
||
const typeMap = { 2: '.pdf', 3: '.pdf', 4: '.pdf', 6: '.pdf', 7: '.zip' };
|
||
return m ? (typeMap[m[1]] || '.pdf') : (path.extname(new URL(url).pathname) || '.pdf');
|
||
}
|
||
|
||
// ---------- HTTP 请求 ----------
|
||
|
||
function request(url, options = {}, retries = RETRY_TIMES) {
|
||
return new Promise((resolve, reject) => {
|
||
const parsedUrl = new URL(url);
|
||
const protocol = parsedUrl.protocol === 'https:' ? https : http;
|
||
const reqOptions = {
|
||
hostname: parsedUrl.hostname,
|
||
path: parsedUrl.pathname + parsedUrl.search,
|
||
method: options.method || 'GET',
|
||
headers: {
|
||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Accept': 'application/json, text/plain, */*',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||
...options.headers,
|
||
},
|
||
timeout: 15000,
|
||
};
|
||
|
||
const req = protocol.request(reqOptions, (res) => {
|
||
let data = '';
|
||
res.on('data', chunk => data += chunk);
|
||
res.on('end', () => {
|
||
if (res.statusCode === 301 || res.statusCode === 302) {
|
||
const location = res.headers.location;
|
||
if (location.startsWith('http')) {
|
||
request(location, options, retries).then(resolve).catch(reject);
|
||
} else {
|
||
const base = `${parsedUrl.protocol}//${parsedUrl.host}`;
|
||
request(base + location, options, retries).then(resolve).catch(reject);
|
||
}
|
||
return;
|
||
}
|
||
if (res.statusCode !== 200) {
|
||
reject(new Error(`HTTP ${res.statusCode}`));
|
||
return;
|
||
}
|
||
try {
|
||
resolve(JSON.parse(data));
|
||
} catch {
|
||
resolve(data);
|
||
}
|
||
});
|
||
});
|
||
|
||
req.on('error', reject);
|
||
req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
|
||
|
||
if (options.body) req.write(options.body);
|
||
req.end();
|
||
}).catch(err => {
|
||
if (retries > 0) {
|
||
const delay = RETRY_DELAY_BASE * (RETRY_TIMES - retries + 1);
|
||
return new Promise(r => setTimeout(r, delay))
|
||
.then(() => request(url, options, retries - 1));
|
||
}
|
||
throw err;
|
||
});
|
||
}
|
||
|
||
// ---------- API 调用 ----------
|
||
|
||
async function fetchAnnouncements(keyword, startDate, endDate, page = 1, pageSize = 50) {
|
||
const postData = JSON.stringify({
|
||
keyWords: keyword,
|
||
pageNum: page,
|
||
pageSize,
|
||
publishStartTime: startDate ? `${startDate} 00:00:00` : '',
|
||
publishEndTime: `${endDate} 23:59:59`,
|
||
});
|
||
|
||
return request('https://bid.powerchina.cn/newcbs/recpro-newmember/BidAnnouncementSummary/list', {
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json',
|
||
'Origin': 'https://bid.powerchina.cn',
|
||
'Referer': 'https://bid.powerchina.cn/search',
|
||
},
|
||
body: postData,
|
||
});
|
||
}
|
||
|
||
// ---------- 下载文件 ----------
|
||
|
||
async function downloadFile(fileUrl, destPath, retries = RETRY_TIMES) {
|
||
return new Promise((resolve, reject) => {
|
||
const parsedUrl = new URL(fileUrl);
|
||
const protocol = parsedUrl.protocol === 'https:' ? https : http;
|
||
const file = fs.createWriteStream(destPath);
|
||
|
||
const req = protocol.request({
|
||
hostname: parsedUrl.hostname,
|
||
path: parsedUrl.pathname + parsedUrl.search,
|
||
method: 'GET',
|
||
headers: {
|
||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||
'Referer': 'https://bid.powerchina.cn/',
|
||
},
|
||
timeout: 30000,
|
||
}, (res) => {
|
||
if (res.statusCode === 301 || res.statusCode === 302) {
|
||
file.close();
|
||
downloadFile(res.headers.location.startsWith('http')
|
||
? res.headers.location
|
||
: `${parsedUrl.protocol}//${parsedUrl.host}${res.headers.location}`,
|
||
destPath, retries
|
||
).then(resolve).catch(reject);
|
||
return;
|
||
}
|
||
if (res.statusCode !== 200) {
|
||
file.close();
|
||
reject(new Error(`Download HTTP ${res.statusCode}`));
|
||
return;
|
||
}
|
||
res.pipe(file);
|
||
file.on('finish', () => { file.close(); resolve(); });
|
||
});
|
||
|
||
req.on('error', (err) => { file.close(); reject(err); });
|
||
req.on('timeout', () => { req.destroy(); file.close(); reject(new Error('Download timeout')); });
|
||
req.end();
|
||
}).catch(err => {
|
||
if (retries > 0) {
|
||
const delay = RETRY_DELAY_BASE * (RETRY_TIMES - retries + 1);
|
||
return new Promise(r => setTimeout(r, delay))
|
||
.then(() => downloadFile(fileUrl, destPath, retries - 1));
|
||
}
|
||
throw err;
|
||
});
|
||
}
|
||
|
||
// ---------- 并发控制 ----------
|
||
|
||
async function runWithConcurrency(tasks, limit) {
|
||
const results = [];
|
||
for (let i = 0; i < tasks.length; i += limit) {
|
||
const batch = tasks.slice(i, i + limit);
|
||
const batchResults = await Promise.allSettled(batch.map(t => t()));
|
||
results.push(...batchResults);
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ---------- 地址提取 ----------
|
||
|
||
const PROVINCES = [
|
||
'北京', '天津', '上海', '重庆',
|
||
'河北', '山西', '辽宁', '吉林', '黑龙江',
|
||
'江苏', '浙江', '安徽', '福建', '江西', '山东',
|
||
'河南', '湖北', '湖南', '广东', '海南',
|
||
'四川', '贵州', '云南', '陕西', '甘肃', '青海',
|
||
'台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
|
||
]
|
||
const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
|
||
const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']
|
||
|
||
function extractRegionFromTitle(title) {
|
||
if (!title) return ''
|
||
// 匹配 "XX省"、"XX市"（直辖市）、"XX自治区"
|
||
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
|
||
if (m) {
|
||
const region = m[1]
|
||
if (region.includes('新疆')) return '新疆'
|
||
if (region.includes('广西')) return '广西'
|
||
if (region.includes('内蒙古')) return '内蒙古'
|
||
if (region.includes('宁夏')) return '宁夏'
|
||
if (region.includes('西藏')) return '西藏'
|
||
if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
|
||
if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
|
||
return ''
|
||
}
|
||
// 直接匹配省份关键词
|
||
for (const prov of SPECIAL_PROVS) {
|
||
if (title.includes(prov)) return prov
|
||
}
|
||
for (const prov of TWO_CHAR_PROVS) {
|
||
if (title.includes(prov)) return prov
|
||
}
|
||
return ''
|
||
}
|
||
|
||
function extractCityFromTitle(title) {
|
||
if (!title) return ''
|
||
const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
|
||
if (m) {
|
||
const city = m[1]
|
||
if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
|
||
}
|
||
return ''
|
||
}
|
||
|
||
// ---------- 生成公告清单 ----------
|
||
|
||
function buildAttachmentName(r, index) {
|
||
const ext = fileExtFromUrl(r.pictureUrl);
|
||
const dateStr = (r.publishTime || '').substring(0, 10) || END_DATE;
|
||
const shortTitle = sanitizeFilename(r.title, 40);
|
||
return `${dateStr}_${index}_${shortTitle}${ext}`;
|
||
}
|
||
|
||
function formatDate(dt) {
|
||
if (!dt) return '-';
|
||
// 兼容 "2026-04-13T18:55:00" 和 "2026-04-13 18:55" 两种格式
|
||
const normalized = dt.includes('T') ? dt.replace('T', ' ').substring(0, 16) : dt.substring(0, 16);
|
||
return normalized;
|
||
}
|
||
|
||
function generateAnnouncementList(announcements, sinceDate, endDate) {
|
||
const lines = [];
|
||
const count = announcements.length;
|
||
const today = new Date().toISOString().substring(0, 10);
|
||
|
||
// 标题
|
||
// 标题：去掉日期中的无效前导零（如 04→4），让范围更易读
|
||
const fmtDate = d => d.replace(/-0(\d)/g, '-$1');
|
||
const rangeLabel = sinceDate === END_DATE ? '当天' : `${fmtDate(sinceDate)}至${fmtDate(END_DATE)}`;
|
||
lines.push(`# ${KEYWORD} 近${rangeLabel}采购公告`);
|
||
lines.push('');
|
||
lines.push(`**筛选条件**：${KEYWORD} | ${sinceDate} ~ ${endDate} | ${TYPES.join('、')}`);
|
||
lines.push('');
|
||
lines.push(`**公告总数**：${count}条`);
|
||
lines.push('');
|
||
lines.push(`> 数据来源：中电建阳光采购网 https://bid.powerchina.cn`);
|
||
lines.push(`> 下载时间：${today}`);
|
||
lines.push('');
|
||
lines.push('---');
|
||
lines.push('');
|
||
|
||
announcements.forEach((r, i) => {
|
||
const n = i + 1;
|
||
const attachmentName = buildAttachmentName(r, n);
|
||
const deadlineReg = formatDate(r.registrationDeadline);
|
||
const deadlineSub = formatDate(r.submissionDeadline);
|
||
const bidTime = formatDate(r.bidOpenTime);
|
||
const publishTime = formatDate(r.publishTime);
|
||
|
||
lines.push(`## ${n}. ${r.title}`);
|
||
lines.push(`- **发布日期**：\`${publishTime}\``);
|
||
lines.push(`- **公告ID**：\`${r.id}\`（systemId: \`${r.systemId}\`）`);
|
||
lines.push(`- **详情链接**：https://bid.powerchina.cn/notice/detail?id=${r.id}`);
|
||
lines.push(`- **采购分类**：${r.titleTypeName || '-'}`);
|
||
lines.push(`- **报名截止**：\`${deadlineReg}\``);
|
||
lines.push(`- **投标截止**：\`${deadlineSub}\``);
|
||
lines.push(`- **开标时间**：\`${bidTime}\``);
|
||
lines.push(`- **附件**：\`pdf/${attachmentName}\``);
|
||
lines.push('');
|
||
});
|
||
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// ---------- 主流程 ----------
|
||
|
||
async function main() {
|
||
// 初始化
|
||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||
fs.mkdirSync(PDF_DIR, { recursive: true });
|
||
fs.mkdirSync(HTML_DIR, { recursive: true });
|
||
if (!fs.existsSync(LOG_FILE)) fs.writeFileSync(LOG_FILE, '');
|
||
const state = loadState();
|
||
const stats = { total: 0, new: 0, skipDuplicate: 0, skipNoAttachment: 0, success: 0, failed: 0, retried: 0 };
|
||
|
||
log(`========== 开始抓取: ${KEYWORD} ==========`);
|
||
|
||
// 确定起始日期（差量：上次运行日期 或 配置的 START_DATE）
|
||
const sinceDate = START_DATE || (state.lastRun ? state.lastRun.split('T')[0] : null) || END_DATE;
|
||
if (START_DATE) {
|
||
log(`指定日期范围: ${START_DATE} ~ ${END_DATE}`);
|
||
} else if (state.lastRun) {
|
||
log(`差量模式: 自 ${state.lastRun.substring(0, 10)} 以来新增`);
|
||
} else {
|
||
log(`首次运行: ${sinceDate} ~ ${END_DATE}`);
|
||
}
|
||
|
||
// 分页获取所有公告
|
||
let page = 1;
|
||
let hasMore = true;
|
||
let allAnnouncements = [];
|
||
|
||
while (hasMore) {
|
||
log(`获取第 ${page} 页...`);
|
||
let data;
|
||
try {
|
||
data = await fetchAnnouncements(KEYWORD, sinceDate, END_DATE, page, 50);
|
||
} catch (err) {
|
||
log(`第 ${page} 页请求失败: ${err.message}`, 'ERROR');
|
||
break;
|
||
}
|
||
|
||
const rows = Array.isArray(data) ? data : (data.rows || []);
|
||
allAnnouncements.push(...rows);
|
||
stats.total += rows.length;
|
||
|
||
if (rows.length < 50 || !data.total || allAnnouncements.length >= data.total) {
|
||
hasMore = false;
|
||
} else {
|
||
page++;
|
||
await new Promise(r => setTimeout(r, 500)); // 防止请求过快
|
||
}
|
||
}
|
||
|
||
log(`共获取 ${stats.total} 条公告（分 ${page} 页）`);
|
||
|
||
// 过滤目标公告类型（由 TYPES 配置定义），且仅取货物类
|
||
const caigou = allAnnouncements.filter(r =>
|
||
TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
|
||
);
|
||
log(`其中 ${TYPES.join('、')}（货物类）: ${caigou.length} 条`);
|
||
|
||
// 生成公告清单（所有本次抓到的招采公告，无论是否已下载）
|
||
const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
|
||
const listFileName = `公告清单_${KEYWORD}_${sinceDate}-${END_DATE}.md`;
|
||
const listPath = path.join(OUTPUT_DIR, listFileName);
|
||
fs.writeFileSync(listPath, listMd, 'utf8');
|
||
log(`📄 公告清单已生成: ${listFileName}`);
|
||
|
||
// 构建下载任务（只下载未记录的）
|
||
const tasks = [];
|
||
for (const r of caigou) {
|
||
const systemId = r.systemId;
|
||
if (!systemId) {
|
||
stats.skipNoAttachment++;
|
||
log(`跳过（无systemId）: ${r.title}`, 'WARN');
|
||
continue;
|
||
}
|
||
|
||
const key = `${systemId}_${r.announcementType}`;
|
||
if (state.downloaded[key]) {
|
||
stats.skipDuplicate++;
|
||
continue;
|
||
}
|
||
|
||
const picUrl = r.pictureUrl;
|
||
if (!picUrl) {
|
||
stats.skipNoAttachment++;
|
||
log(`跳过（无附件）: ${r.title}`, 'WARN');
|
||
state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
|
||
continue;
|
||
}
|
||
|
||
const ext = fileExtFromUrl(picUrl);
|
||
const dateStr = (r.publishTime || '').substring(0, 10) || END_DATE;
|
||
const shortTitle = sanitizeFilename(r.title, 40);
|
||
const fname = `${dateStr}_${systemId.substring(0, 8)}_${shortTitle}${ext}`;
|
||
const fpath = path.join(PDF_DIR, fname);
|
||
const shortName = sanitizeFilename(r.title, 20);
|
||
|
||
tasks.push(async () => {
|
||
log(`下载 [${shortName}]: ${fname}`);
|
||
|
||
// 从标题中提取省份（详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取）
|
||
const region = extractRegionFromTitle(r.title) || ''
|
||
const regionDetail = extractCityFromTitle(r.title) || ''
|
||
|
||
try {
|
||
await downloadFile(picUrl, fpath);
|
||
const size = fs.statSync(fpath).size;
|
||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
|
||
stats.success++;
|
||
log(` ✅ ${fname} (${size} bytes) region=${region || '-'} addr=${regionDetail || '-'}`);
|
||
} catch (err) {
|
||
stats.failed++;
|
||
state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
|
||
log(` ❌ ${fname}: ${err.message}`, 'ERROR');
|
||
}
|
||
});
|
||
}
|
||
|
||
// 并发执行
|
||
if (tasks.length > 0) {
|
||
log(`开始下载 ${tasks.length} 个附件（并发数: ${CONCURRENCY}）...`);
|
||
await runWithConcurrency(tasks, CONCURRENCY);
|
||
}
|
||
|
||
// 更新状态
|
||
state.lastRun = new Date().toISOString();
|
||
state.lastKeyword = KEYWORD;
|
||
saveState(state);
|
||
|
||
// 输出统计
|
||
log(`========== 完成 ==========`);
|
||
log(`总计公告: ${stats.total} | ${TYPES.join('、')}: ${caigou.length}`);
|
||
log(`新增下载: ${stats.success} | 重复跳过: ${stats.skipDuplicate} | 无附件跳过: ${stats.skipNoAttachment} | 下载失败: ${stats.failed}`);
|
||
log(`状态已保存: ${STATE_FILE}`);
|
||
console.log('\n========== 汇总 ==========');
|
||
console.log(`✅ 成功: ${stats.success} | ❌ 失败: ${stats.failed} | ⏩ 重复: ${stats.skipDuplicate} | ⚠️ 无附件: ${stats.skipNoAttachment}`);
|
||
console.log(`📁 保存目录: ${OUTPUT_DIR}`);
|
||
console.log(`📄 公告清单: ${listFileName}`);
|
||
}
|
||
|
||
main().catch(err => { console.error('Fatal:', err); process.exit(1); });
|