#!/usr/bin/env node
/**
 * 中电建阳光采购网 - 招采公告抓取脚本 v1.2
 *
 * v1.2 新增：
 * - 支持多公告类型：招采公告、资格预审公告、采购预告、中标结果公示
 * - 类型在配置区可自定义增减
 *
 * 改进点（继承v1.1）：
 * 1. 分页逻辑 - 自动遍历所有页
 * 2. 去重机制 - 基于 systemId 持久化记录
 * 3. 空附件处理 - pictureUrl 为空时跳过并告警
 * 4. 稳定文件名 - 基于 systemId + 内容 hash
 * 5. 并发下载 - 控制并发数，避免被限流
 * 6. 失败重试 - 3次重试，间隔递增
 * 7. 状态持久化 - downloaded.json 记录已下载文件
 * 8. 差量更新 - 只抓自上次运行以来的新增公告
 * 9. 详细报告 - 成功/失败/跳过数量统计
 */

const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');
const crypto = require('crypto');

// ========== 配置区 ==========

// 支持命令行传入 --keyword（例如: node fetch-v1.2.js --keyword 电缆）
const args = process.argv.slice(2)
const keywordIdx = args.indexOf('--keyword')
const KEYWORD = (keywordIdx >= 0 && args[keywordIdx + 1]) ? args[keywordIdx + 1] : ''

// 公告类型：可按需增减。支持的类型值见 API 返回的 announcementType 字段
const TYPES = ['招采公告'];
// 可选：留空则从上次抓取日期继续；有值则从指定日期开始
const START_DATE = '';
const END_DATE = new Date().toISOString().split('T')[0];  // 今天
// 跨天首次运行(state 为空)时回溯 N 天, 避免错过前一天后半天发布的公告
const BACKFILL_DAYS = 3;
const OUTPUT_DIR = path.join(process.env.HOME, 'desktop/招标公告', END_DATE);
const PDF_DIR = path.join(OUTPUT_DIR, 'pdf');     // PDF 附件
const HTML_DIR = path.join(OUTPUT_DIR, 'html');    // HTML 转换（待安装转换技能）
const STATE_FILE = path.join(OUTPUT_DIR, 'downloaded.json');  // 已下载记录
const LOG_FILE = path.join(OUTPUT_DIR, 'fetch.log');  // 运行日志
const CONCURRENCY = 3;  // 并发下载数
const RETRY_TIMES = 3;
const RETRY_DELAY_BASE = 1000;  // ms
// ==========================

// ---------- 工具函数 ----------

function log(msg, type = 'INFO') {
  const ts = new Date().toISOString().replace('T', ' ').substring(0, 19);
  const line = `[${ts}] [${type}] ${msg}`;
  console.log(line);
  fs.appendFileSync(LOG_FILE, line + '\n');
}

function loadState() {
  try {
    return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
  } catch {
    return { lastRun: null, downloaded: {}, lastId: null };
  }
}

function saveState(state) {
  fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
}

function md5(str) {
  return crypto.createHash('md5').update(str).digest('hex').substring(0, 8);
}

function sanitizeFilename(name, maxLen = 50) {
  return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_').replace(/\s+/g, '_').substring(0, maxLen);
}

function fileExtFromUrl(url) {
  if (!url) return null;
  const m = url.match(/fileType=(\d+)/);
  const typeMap = { 2: '.pdf', 3: '.pdf', 4: '.pdf', 6: '.pdf', 7: '.zip' };
  return m ? (typeMap[m[1]] || '.pdf') : (path.extname(new URL(url).pathname) || '.pdf');
}

// ---------- HTTP 请求 ----------

function request(url, options = {}, retries = RETRY_TIMES) {
  return new Promise((resolve, reject) => {
    const parsedUrl = new URL(url);
    const protocol = parsedUrl.protocol === 'https:' ? https : http;
    const reqOptions = {
      hostname: parsedUrl.hostname,
      path: parsedUrl.pathname + parsedUrl.search,
      method: options.method || 'GET',
      headers: {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'application/json, text/plain, */*',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        ...options.headers,
      },
      timeout: 15000,
    };

    const req = protocol.request(reqOptions, (res) => {
      let data = '';
      res.on('data', chunk => data += chunk);
      res.on('end', () => {
        if (res.statusCode === 301 || res.statusCode === 302) {
          const location = res.headers.location;
          if (location.startsWith('http')) {
            request(location, options, retries).then(resolve).catch(reject);
          } else {
            const base = `${parsedUrl.protocol}//${parsedUrl.host}`;
            request(base + location, options, retries).then(resolve).catch(reject);
          }
          return;
        }
        if (res.statusCode !== 200) {
          reject(new Error(`HTTP ${res.statusCode}`));
          return;
        }
        try {
          resolve(JSON.parse(data));
        } catch {
          resolve(data);
        }
      });
    });

    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });

    if (options.body) req.write(options.body);
    req.end();
  }).catch(err => {
    if (retries > 0) {
      const delay = RETRY_DELAY_BASE * (RETRY_TIMES - retries + 1);
      return new Promise(r => setTimeout(r, delay))
        .then(() => request(url, options, retries - 1));
    }
    throw err;
  });
}

// ---------- API 调用 ----------

async function fetchAnnouncements(keyword, startDate, endDate, page = 1, pageSize = 50) {
  const postData = JSON.stringify({
    keyWords: keyword,
    pageNum: page,
    pageSize,
    publishStartTime: startDate ? `${startDate} 00:00:00` : '',
    publishEndTime: `${endDate} 23:59:59`,
  });

  return request('https://bid.powerchina.cn/newcbs/recpro-newmember/BidAnnouncementSummary/list', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Origin': 'https://bid.powerchina.cn',
      'Referer': 'https://bid.powerchina.cn/search',
    },
    body: postData,
  });
}

// ---------- 下载文件 ----------

async function downloadFile(fileUrl, destPath, retries = RETRY_TIMES) {
  return new Promise((resolve, reject) => {
    const parsedUrl = new URL(fileUrl);
    const protocol = parsedUrl.protocol === 'https:' ? https : http;
    const file = fs.createWriteStream(destPath);

    const req = protocol.request({
      hostname: parsedUrl.hostname,
      path: parsedUrl.pathname + parsedUrl.search,
      method: 'GET',
      headers: {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        'Referer': 'https://bid.powerchina.cn/',
      },
      timeout: 30000,
    }, (res) => {
      if (res.statusCode === 301 || res.statusCode === 302) {
        file.close();
        downloadFile(res.headers.location.startsWith('http')
          ? res.headers.location
          : `${parsedUrl.protocol}//${parsedUrl.host}${res.headers.location}`,
          destPath, retries
        ).then(resolve).catch(reject);
        return;
      }
      if (res.statusCode !== 200) {
        file.close();
        reject(new Error(`Download HTTP ${res.statusCode}`));
        return;
      }
      res.pipe(file);
      file.on('finish', () => { file.close(); resolve(); });
    });

    req.on('error', (err) => { file.close(); reject(err); });
    req.on('timeout', () => { req.destroy(); file.close(); reject(new Error('Download timeout')); });
    req.end();
  }).catch(err => {
    if (retries > 0) {
      const delay = RETRY_DELAY_BASE * (RETRY_TIMES - retries + 1);
      return new Promise(r => setTimeout(r, delay))
        .then(() => downloadFile(fileUrl, destPath, retries - 1));
    }
    throw err;
  });
}

// ---------- 并发控制 ----------

async function runWithConcurrency(tasks, limit) {
  const results = [];
  for (let i = 0; i < tasks.length; i += limit) {
    const batch = tasks.slice(i, i + limit);
    const batchResults = await Promise.allSettled(batch.map(t => t()));
    results.push(...batchResults);
  }
  return results;
}

// ---------- 地址提取 ----------

const PROVINCES = [
  '北京', '天津', '上海', '重庆',
  '河北', '山西', '辽宁', '吉林', '黑龙江',
  '江苏', '浙江', '安徽', '福建', '江西', '山东',
  '河南', '湖北', '湖南', '广东', '海南',
  '四川', '贵州', '云南', '陕西', '甘肃', '青海',
  '台湾', '内蒙古', '广西', '西藏', '宁夏', '新疆',
]
const TWO_CHAR_PROVS = PROVINCES.filter(p => p.length === 2)
const SPECIAL_PROVS = ['内蒙古', '广西', '新疆', '西藏', '宁夏']

function extractRegionFromTitle(title) {
  if (!title) return ''
  // 匹配 "XX省"、"XX市"（直辖市）、"XX自治区"
  const m = title.match(/([\u4e00-\u9fff]{2,4}(?:省|市|自治区))/)
  if (m) {
    const region = m[1]
    if (region.includes('新疆')) return '新疆'
    if (region.includes('广西')) return '广西'
    if (region.includes('内蒙古')) return '内蒙古'
    if (region.includes('宁夏')) return '宁夏'
    if (region.includes('西藏')) return '西藏'
    if (['北京', '天津', '上海', '重庆'].some(c => region.includes(c))) return region.slice(0, 2)
    if (region.endsWith('省') || region.endsWith('自治区')) return region.slice(0, 2)
    return ''
  }
  // 直接匹配省份关键词
  for (const prov of SPECIAL_PROVS) {
    if (title.includes(prov)) return prov
  }
  for (const prov of TWO_CHAR_PROVS) {
    if (title.includes(prov)) return prov
  }
  return ''
}

function extractCityFromTitle(title) {
  if (!title) return ''
  const m = title.match(/([\u4e00-\u9fff]{2,4}(?:市))/)
  if (m) {
    const city = m[1]
    if (!['北京市', '天津市', '上海市', '重庆市'].includes(city)) return city
  }
  return ''
}

// ---------- 生成公告清单 ----------

function buildAttachmentName(r, index) {
  const ext = fileExtFromUrl(r.pictureUrl);
  const dateStr = (r.publishTime || '').substring(0, 10) || END_DATE;
  const shortTitle = sanitizeFilename(r.title, 40);
  return `${dateStr}_${index}_${shortTitle}${ext}`;
}

function formatDate(dt) {
  if (!dt) return '-';
  // 兼容 "2026-04-13T18:55:00" 和 "2026-04-13 18:55" 两种格式
  const normalized = dt.includes('T') ? dt.replace('T', ' ').substring(0, 16) : dt.substring(0, 16);
  return normalized;
}

function generateAnnouncementList(announcements, sinceDate, endDate) {
  const lines = [];
  const count = announcements.length;
  const today = new Date().toISOString().substring(0, 10);

  // 标题
  // 标题：去掉日期中的无效前导零（如 04→4），让范围更易读
  const fmtDate = d => d.replace(/-0(\d)/g, '-$1');
  const rangeLabel = sinceDate === END_DATE ? '当天' : `${fmtDate(sinceDate)}至${fmtDate(END_DATE)}`;
  lines.push(`# ${KEYWORD} 近${rangeLabel}采购公告`);
  lines.push('');
  lines.push(`**筛选条件**：${KEYWORD} | ${sinceDate} ~ ${endDate} | ${TYPES.join('、')}`);
  lines.push('');
  lines.push(`**公告总数**：${count}条`);
  lines.push('');
  lines.push(`> 数据来源：中电建阳光采购网 https://bid.powerchina.cn`);
  lines.push(`> 下载时间：${today}`);
  lines.push('');
  lines.push('---');
  lines.push('');

  announcements.forEach((r, i) => {
    const n = i + 1;
    const attachmentName = buildAttachmentName(r, n);
    const deadlineReg = formatDate(r.registrationDeadline);
    const deadlineSub = formatDate(r.submissionDeadline);
    const bidTime = formatDate(r.bidOpenTime);
    const publishTime = formatDate(r.publishTime);

    lines.push(`## ${n}. ${r.title}`);
    lines.push(`- **发布日期**：\`${publishTime}\``);
    lines.push(`- **公告ID**：\`${r.id}\`（systemId: \`${r.systemId}\`）`);
    lines.push(`- **详情链接**：https://bid.powerchina.cn/notice/detail?id=${r.id}`);
    lines.push(`- **采购分类**：${r.titleTypeName || '-'}`);
    lines.push(`- **报名截止**：\`${deadlineReg}\``);
    lines.push(`- **投标截止**：\`${deadlineSub}\``);
    lines.push(`- **开标时间**：\`${bidTime}\``);
    lines.push(`- **附件**：\`pdf/${attachmentName}\``);
    lines.push('');
  });

  return lines.join('\n');
}

// ---------- 主流程 ----------

async function main() {
  // 初始化
  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
  fs.mkdirSync(PDF_DIR, { recursive: true });
  fs.mkdirSync(HTML_DIR, { recursive: true });
  if (!fs.existsSync(LOG_FILE)) fs.writeFileSync(LOG_FILE, '');
  const state = loadState();
  const stats = { total: 0, new: 0, skipDuplicate: 0, skipNoAttachment: 0, success: 0, failed: 0, retried: 0 };

  log(`========== 开始抓取: ${KEYWORD} ==========`);

  // 确定起始日期（差量：上次运行日期 或 配置的 START_DATE；跨天首次运行回溯 N 天）
  const sinceDate = START_DATE || (state.lastRun ? state.lastRun.split('T')[0] : null) || (() => {
    const d = new Date(Date.now() - BACKFILL_DAYS * 86400 * 1000);
    return d.toISOString().split('T')[0];
  })();
  if (START_DATE) {
    log(`指定日期范围: ${START_DATE} ~ ${END_DATE}`);
  } else if (state.lastRun) {
    log(`差量模式: 自 ${state.lastRun.substring(0, 10)} 以来新增`);
  } else if (sinceDate !== END_DATE) {
    log(`跨天补抓: ${sinceDate} ~ ${END_DATE}`);
  } else {
    log(`首次运行: ${sinceDate} ~ ${END_DATE}`);
  }

  // 分页获取所有公告
  let page = 1;
  let hasMore = true;
  let allAnnouncements = [];

  while (hasMore) {
    log(`获取第 ${page} 页...`);
    let data;
    try {
      data = await fetchAnnouncements(KEYWORD, sinceDate, END_DATE, page, 50);
    } catch (err) {
      log(`第 ${page} 页请求失败: ${err.message}`, 'ERROR');
      break;
    }

    const rows = Array.isArray(data) ? data : (data.rows || []);
    allAnnouncements.push(...rows);
    stats.total += rows.length;

    if (rows.length < 50 || !data.total || allAnnouncements.length >= data.total) {
      hasMore = false;
    } else {
      page++;
      await new Promise(r => setTimeout(r, 500));  // 防止请求过快
    }
  }

  log(`共获取 ${stats.total} 条公告（分 ${page} 页）`);

  // 过滤目标公告类型（由 TYPES 配置定义），且仅取货物类
  const caigou = allAnnouncements.filter(r =>
    TYPES.includes(r.announcementType) && r.titleTypeName === '货物类'
  );
  log(`其中 ${TYPES.join('、')}（货物类）: ${caigou.length} 条`);

  // 生成公告清单（所有本次抓到的招采公告，无论是否已下载）
  const listMd = generateAnnouncementList(caigou, sinceDate, END_DATE);
  const listFileName = `公告清单_${KEYWORD}_${sinceDate}-${END_DATE}.md`;
  const listPath = path.join(OUTPUT_DIR, listFileName);
  fs.writeFileSync(listPath, listMd, 'utf8');
  log(`📄 公告清单已生成: ${listFileName}`);

  // 构建下载任务（只下载未记录的）
  const tasks = [];
  for (const r of caigou) {
    const systemId = r.systemId;
    if (!systemId) {
      stats.skipNoAttachment++;
      log(`跳过（无systemId）: ${r.title}`, 'WARN');
      continue;
    }

    const key = `${systemId}_${r.announcementType}`;
    if (state.downloaded[key]) {
      stats.skipDuplicate++;
      continue;
    }

    const picUrl = r.pictureUrl;
    if (!picUrl) {
      stats.skipNoAttachment++;
      log(`跳过（无附件）: ${r.title}`, 'WARN');
      state.downloaded[key] = { id: r.id, title: r.title, skipped: true, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', time: new Date().toISOString() };
      continue;
    }

    const ext = fileExtFromUrl(picUrl);
    const dateStr = (r.publishTime || '').substring(0, 10) || END_DATE;
    const shortTitle = sanitizeFilename(r.title, 40);
    const fname = `${dateStr}_${systemId.substring(0, 8)}_${shortTitle}${ext}`;
    const fpath = path.join(PDF_DIR, fname);
    const shortName = sanitizeFilename(r.title, 20);

    tasks.push(async () => {
      log(`下载 [${shortName}]: ${fname}`);

      // 从标题中提取省份（详细地址由后端调用 extract_address_from_pdf.py 从 PDF 正文提取）
      const region = extractRegionFromTitle(r.title) || ''
      const regionDetail = extractCityFromTitle(r.title) || ''

      try {
        await downloadFile(picUrl, fpath);
        const size = fs.statSync(fpath).size;
        state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', path: `pdf/${fname}`, size, time: new Date().toISOString(), region, regionDetail };
        stats.success++;
        log(`  ✅ ${fname} (${size} bytes)  region=${region || '-'} addr=${regionDetail || '-'}`);
      } catch (err) {
        stats.failed++;
        state.downloaded[key] = { id: r.id, title: r.title, deadline: r.submissionDeadline || '', publishTime: r.publishTime || '', failed: true, error: err.message, time: new Date().toISOString(), region, regionDetail };
        log(`  ❌ ${fname}: ${err.message}`, 'ERROR');
      }
    });
  }

  // 并发执行
  if (tasks.length > 0) {
    log(`开始下载 ${tasks.length} 个附件（并发数: ${CONCURRENCY}）...`);
    await runWithConcurrency(tasks, CONCURRENCY);
  }

  // 更新状态
  state.lastRun = new Date().toISOString();
  state.lastKeyword = KEYWORD;
  saveState(state);

  // 输出统计
  log(`========== 完成 ==========`);
  log(`总计公告: ${stats.total} | ${TYPES.join('、')}: ${caigou.length}`);
  log(`新增下载: ${stats.success} | 重复跳过: ${stats.skipDuplicate} | 无附件跳过: ${stats.skipNoAttachment} | 下载失败: ${stats.failed}`);
  log(`状态已保存: ${STATE_FILE}`);
  console.log('\n========== 汇总 ==========');
  console.log(`✅ 成功: ${stats.success}  |  ❌ 失败: ${stats.failed}  |  ⏩ 重复: ${stats.skipDuplicate}  |  ⚠️ 无附件: ${stats.skipNoAttachment}`);
  console.log(`📁 保存目录: ${OUTPUT_DIR}`);
  console.log(`📄 公告清单: ${listFileName}`);
}

main().catch(err => { console.error('Fatal:', err); process.exit(1); });