5a49a37235
- HANDOVER.md: 完整交接文档(架构/数据模型/API/业务流/已知坑/工作区状态/待办) - PRD-智能比价助手.md: 比价功能产品需求文档(供外部平台参考) - 数据维护脚本: 清理AI垃圾品类SQL、修复损坏标题SQL、线索重分类/重抓脚本
108 lines
3.5 KiB
JavaScript
108 lines
3.5 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* 清理状态文件中的乱码标题记录 + 删除 DB 中乱码线索，然后重新抓取对应日期
|
||
* 用于修复 fetch-v1.2.js chunk 边界截断导致的标题乱码问题
|
||
*
|
||
* 用法: node scripts/refetch_corrupted.js # 清理并重新抓取
|
||
* node scripts/refetch_corrupted.js --dry-run # 仅预览不做操作
|
||
*/
|
||
|
||
const { execSync } = require('child_process')
|
||
const fs = require('fs')
|
||
const path = require('path')
|
||
|
||
const BASE_DIR = '/data/pingtai/招标公告'
|
||
const SCRIPT = '/opt/pingtai/scripts/tender-fetcher/fetch-v1.2.js'
|
||
const DRY_RUN = process.argv.includes('--dry-run')
|
||
|
||
/** 收集所有乱码条目 */
|
||
function findCorrupted() {
|
||
const results = []
|
||
const dirs = fs.readdirSync(BASE_DIR).filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort()
|
||
for (const dir of dirs) {
|
||
const statePath = path.join(BASE_DIR, dir, 'downloaded.json')
|
||
if (!fs.existsSync(statePath)) continue
|
||
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'))
|
||
for (const [key, v] of Object.entries(state.downloaded || {})) {
|
||
if (v.title && v.title.includes('\uFFFD')) {
|
||
results.push({ date: dir, key, statePath, info: v })
|
||
}
|
||
}
|
||
}
|
||
return results
|
||
}
|
||
|
||
/** 从状态文件中移除指定 key */
|
||
function removeFromState(statePath, key) {
|
||
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'))
|
||
if (state.downloaded?.[key]) {
|
||
delete state.downloaded[key]
|
||
fs.writeFileSync(statePath, JSON.stringify(state, null, 2))
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
async function main() {
|
||
const corrupted = findCorrupted()
|
||
const affectedDates = [...new Set(corrupted.map(c => c.date))].sort()
|
||
const affectedIds = corrupted.map(c => c.info.id).filter(Boolean)
|
||
|
||
console.log(`找到 ${corrupted.length} 条乱码记录，涉及 ${affectedDates.length} 天`)
|
||
console.log(`受影响日期: ${affectedDates.join(', ')}`)
|
||
console.log(`受影响 ID: ${affectedIds.join(', ')}`)
|
||
|
||
if (corrupted.length === 0) {
|
||
console.log('无需清理')
|
||
return
|
||
}
|
||
|
||
// 1. 清理状态文件
|
||
console.log('\n=== 清理状态文件 ===')
|
||
let cleaned = 0
|
||
for (const c of corrupted) {
|
||
if (DRY_RUN) {
|
||
console.log(` [DRY] 将从 ${c.date}/downloaded.json 移除 key: ${c.key}`)
|
||
cleaned++
|
||
continue
|
||
}
|
||
if (removeFromState(c.statePath, c.key)) {
|
||
console.log(` ✅ ${c.date}: 已移除 ${c.key}`)
|
||
cleaned++
|
||
}
|
||
}
|
||
console.log(`已清理 ${cleaned}/${corrupted.length} 条状态记录`)
|
||
|
||
// 2. 重新抓取每个受影响日期
|
||
console.log('\n=== 重新抓取 ===')
|
||
for (const date of affectedDates) {
|
||
if (DRY_RUN) {
|
||
console.log(` [DRY] 将执行: START_DATE=${date} node ${SCRIPT}`)
|
||
continue
|
||
}
|
||
try {
|
||
console.log(` ⏳ 抓取 ${date} 的数据...`)
|
||
const output = execSync(`START_DATE=${date} node ${SCRIPT}`, {
|
||
cwd: path.dirname(SCRIPT),
|
||
timeout: 120000,
|
||
encoding: 'utf8',
|
||
})
|
||
// 提取最后几行输出
|
||
const lines = output.trim().split('\n').slice(-5).join('\n')
|
||
console.log(` ✅ ${date}: 完成\n${lines}`)
|
||
} catch (e) {
|
||
console.error(` ❌ ${date}: ${e.message}`)
|
||
}
|
||
// 间隔等待，避免请求过频
|
||
await new Promise(r => setTimeout(r, 2000))
|
||
}
|
||
|
||
console.log('\n=== 完成 ===')
|
||
if (!DRY_RUN) {
|
||
console.log('提示：执行后端采集器以更新 DB（手动触发"中电建"采集器运行）')
|
||
console.log('或等待下次定时调度自动更新')
|
||
}
|
||
}
|
||
|
||
main().catch(console.error)
|