afac18e843
- W26: NestJS API + Next.js Web + Prisma + SQLite scaffold - W26: Lead 模型 + Crawler + 询价发布助手 (线索池 + 一键发布) - W29.x: MCP 集成 (Integration 模型 + connectMcp + publish_rfq 5 工具端到端跑通) - W30: 3 字段补充 (notes / quotationType / quotationDemand) - W31 P1: 自动填 enterpriseId / userId (Prisma companyUserId 字段 + get_app_identity tool + lead.service.ts publishArgs) - W31: detail 详情页 + admin UI 集成管理 (OpenAppListView + AuditLogListView) 关联: ~/Documents/02-平台开发/产品开发/F002-W31-交接文档.md GIT-AUDIT-REPORT-2026-06-24.md
353 lines
13 KiB
Python
Executable File
353 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
自动发布：从今日下载目录读取公告 → 提取数据 → 发布到 direv
|
||
用于 cron 定时任务，发布完成后飞书推送结果
|
||
|
||
用法：
|
||
python3 auto_publish.py # 发布今日所有电建+新矿公告
|
||
python3 auto_publish.py --dry-run # 仅列出待发布，不实际发布
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from datetime import date, datetime, timedelta
|
||
|
||
import fitz
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
from batch_publish_v4 import (
|
||
publish_one,
|
||
fetch_cover_image,
|
||
find_notes_file,
|
||
compute_delivery,
|
||
AUTH_FILE,
|
||
CATEGORY_FILE_KEYWORDS,
|
||
)
|
||
from extract_notes import get_notes, summarize_with_llm, extract_from_pdf
|
||
|
||
# ============ 配置 ============
|
||
PROMPT = """从以下招标公告中提取信息，返回一个 JSON 对象：
|
||
{"title":"精简标题≤35字","deadline":"YYYY-MM-DD","region":"省市区县+项目名","category":"合并品类名如'混凝土制品'","unit":"单位","quantity":"总量数字"}
|
||
|
||
规则：
|
||
- deadline：优先用报价截止日期/投标截止日期，没有则用公告发布日期+7天，都没有则用 null
|
||
- quantity：把采购一览表中所有行的数量加总求和（数字），不要只取第一行或某一行的值。如果表格中单位不同无法加总，用出现最多的单位，只加总该单位的行
|
||
- unit：用出现次数最多的单位
|
||
- category 用最上层的品类名（如钢筋混凝土管→混凝土制品，地板→装修材料）
|
||
- region 写到区县+项目名即可
|
||
- 如果是工程分包/土石方/服务类，返回 {"skip": true}
|
||
"""
|
||
|
||
SKIP_WORDS = ['工程分包', '专业分包', '土石方', '劳务分包']
|
||
|
||
AUTH_PROFILES = os.path.expanduser("~/.openclaw/agents/main/agent/auth-profiles.json")
|
||
|
||
|
||
def load_deepseek_key():
|
||
try:
|
||
with open(AUTH_PROFILES) as f:
|
||
data = json.load(f)
|
||
container = data.get("profiles", data)
|
||
for k, v in container.items():
|
||
if isinstance(v, dict) and v.get("provider") == "deepseek":
|
||
return v.get("key")
|
||
except:
|
||
pass
|
||
return None
|
||
|
||
|
||
def extract_lead_from_pdf(pdf_path, title):
|
||
"""用 DeepSeek 从 PDF 中提取 direv 发布所需字段"""
|
||
doc = fitz.open(pdf_path)
|
||
text = ''.join(page.get_text() for page in doc)[:4000]
|
||
doc.close()
|
||
|
||
api_key = load_deepseek_key()
|
||
if not api_key:
|
||
raise RuntimeError("DeepSeek API Key 未配置")
|
||
|
||
import requests
|
||
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||
payload = {
|
||
"model": "deepseek-chat",
|
||
"messages": [{"role": "user", "content": PROMPT + "\n原公告标题：" + title + "\n\n公告内容：\n" + text}],
|
||
"temperature": 0,
|
||
"max_tokens": 500,
|
||
"response_format": {"type": "json_object"},
|
||
}
|
||
|
||
r = requests.post("https://api.deepseek.com/v1/chat/completions",
|
||
headers=headers, json=payload, timeout=60)
|
||
r.raise_for_status()
|
||
content = r.json()["choices"][0]["message"]["content"]
|
||
content = re.sub(r'^```json?\s*', '', content.strip())
|
||
content = re.sub(r'\s*```$', '', content)
|
||
return json.loads(content)
|
||
|
||
|
||
def extract_lead_from_html(html_path, title):
|
||
"""用 DeepSeek 从 HTML 中提取 direv 发布所需字段"""
|
||
with open(html_path, 'r', encoding='utf-8') as f:
|
||
html = f.read()
|
||
text = re.sub(r'<[^>]+>', ' ', html)
|
||
text = re.sub(r'&[a-z]+;', ' ', text)
|
||
text = re.sub(r'\s+', ' ', text).strip()[:4000]
|
||
|
||
api_key = load_deepseek_key()
|
||
if not api_key:
|
||
raise RuntimeError("DeepSeek API Key 未配置")
|
||
|
||
import requests
|
||
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||
payload = {
|
||
"model": "deepseek-chat",
|
||
"messages": [{"role": "user", "content": PROMPT + "\n原公告标题：" + title + "\n\n公告内容：\n" + text}],
|
||
"temperature": 0,
|
||
"max_tokens": 500,
|
||
"response_format": {"type": "json_object"},
|
||
}
|
||
|
||
r = requests.post("https://api.deepseek.com/v1/chat/completions",
|
||
headers=headers, json=payload, timeout=60)
|
||
r.raise_for_status()
|
||
content = r.json()["choices"][0]["message"]["content"]
|
||
content = re.sub(r'^```json?\s*', '', content.strip())
|
||
content = re.sub(r'\s*```$', '', content)
|
||
return json.loads(content)
|
||
|
||
|
||
def gather_xkgroup_leads(date_dir=None):
|
||
"""从新矿集团下载目录收集待发布公告（默认只取最新一天）"""
|
||
if date_dir is not None:
|
||
base = os.path.expanduser(f"~/desktop/招标公告/{date_dir}")
|
||
if not os.path.exists(os.path.join(base, "xkgroup_downloaded.json")):
|
||
return []
|
||
bases = [base]
|
||
else:
|
||
today = datetime.now()
|
||
# 只取今天目录（新矿每天7天窗口全量抓取，无需扫历史）
|
||
date_str = today.strftime("%Y-%m-%d")
|
||
base = os.path.expanduser(f"~/desktop/招标公告/{date_str}")
|
||
if not os.path.exists(os.path.join(base, "xkgroup_downloaded.json")):
|
||
return []
|
||
bases = [base]
|
||
|
||
leads = []
|
||
seen_ids = set()
|
||
for base in bases:
|
||
with open(os.path.join(base, "xkgroup_downloaded.json")) as f:
|
||
data = json.load(f)
|
||
|
||
published_file = os.path.join(base, "direv_published.json")
|
||
published = {}
|
||
if os.path.exists(published_file):
|
||
with open(published_file) as f:
|
||
published = json.load(f)
|
||
|
||
for notice_id, v in data.items():
|
||
if notice_id in seen_ids:
|
||
continue
|
||
seen_ids.add(notice_id)
|
||
if v.get("type") != "寻源公告":
|
||
continue
|
||
title = v.get("title", "")
|
||
if any(w in title for w in SKIP_WORDS):
|
||
continue
|
||
if notice_id in published:
|
||
continue
|
||
html_path = os.path.join(base, v.get("html", ""))
|
||
if not v.get("html") or not os.path.isfile(html_path):
|
||
pdf_path = os.path.join(base, v.get("pdf", ""))
|
||
if pdf_path and os.path.isfile(pdf_path):
|
||
html_path = pdf_path
|
||
else:
|
||
continue
|
||
leads.append({
|
||
"key": notice_id,
|
||
"title": title,
|
||
"html_path": html_path,
|
||
"base_dir": base,
|
||
"source": "新矿",
|
||
"publish_time": v.get("publish_time", ""),
|
||
"org": v.get("org", "")
|
||
})
|
||
|
||
return leads
|
||
|
||
|
||
def gather_leads(date_dir=None):
|
||
"""从下载目录收集待发布公告（扫描最近7天所有目录）"""
|
||
if date_dir is not None:
|
||
base = os.path.expanduser(f"~/desktop/招标公告/{date_dir}")
|
||
if not os.path.exists(os.path.join(base, "downloaded.json")):
|
||
return []
|
||
bases = [base]
|
||
else:
|
||
today = datetime.now()
|
||
bases = []
|
||
for days_back in range(7):
|
||
date_str = (today - timedelta(days=days_back)).strftime("%Y-%m-%d")
|
||
base = os.path.expanduser(f"~/desktop/招标公告/{date_str}")
|
||
if os.path.exists(os.path.join(base, "downloaded.json")):
|
||
bases.append(base)
|
||
if not bases:
|
||
return []
|
||
|
||
leads = []
|
||
for base in bases:
|
||
with open(os.path.join(base, "downloaded.json")) as f:
|
||
data = json.load(f)
|
||
|
||
published_file = os.path.join(base, "direv_published.json")
|
||
published = {}
|
||
if os.path.exists(published_file):
|
||
with open(published_file) as f:
|
||
published = json.load(f)
|
||
|
||
for k, v in data.get("downloaded", {}).items():
|
||
if v.get("skipped") or v.get("failed"):
|
||
continue
|
||
title = v.get("title", "")
|
||
if any(w in title for w in SKIP_WORDS):
|
||
continue
|
||
if k in published:
|
||
continue
|
||
pdf_path = os.path.join(base, v["path"])
|
||
if not os.path.exists(pdf_path):
|
||
continue
|
||
leads.append({"key": k, "title": title, "pdf_path": pdf_path, "base_dir": base})
|
||
|
||
return leads
|
||
|
||
|
||
async def auto_publish(dry_run=False):
|
||
leads_dianjian = gather_leads()
|
||
leads_xinkuang = gather_xkgroup_leads()
|
||
leads = leads_dianjian + leads_xinkuang
|
||
if not leads:
|
||
print("无待发布公告")
|
||
return {"total": 0, "success": 0, "skip": 0, "failed": 0}
|
||
|
||
print(f"待发布: {len(leads)} 条（电建 {len(leads_dianjian)} + 新矿 {len(leads_xinkuang)}）\n")
|
||
|
||
if dry_run:
|
||
for i, l in enumerate(leads, 1):
|
||
src = l.get("source", "电建")
|
||
print(f" {i}. [{src}] {l['title'][:60]}")
|
||
return {"total": len(leads), "success": 0, "skip": 0, "failed": 0, "dry_run": True}
|
||
|
||
# 提取结构化数据
|
||
announcements = []
|
||
for i, l in enumerate(leads):
|
||
cat = l["title"][:40]
|
||
src = l.get("source", "电建")
|
||
print(f"[{i+1}/{len(leads)}] [{src}] 提取: {cat}...", end=" ", flush=True)
|
||
try:
|
||
if src == "新矿":
|
||
parsed = extract_lead_from_html(l["html_path"], l["title"])
|
||
else:
|
||
parsed = extract_lead_from_pdf(l.get("pdf_path", l.get("html_path")), l["title"])
|
||
if parsed.get("skip"):
|
||
print("⏩ 跳过（非货物）")
|
||
continue
|
||
# 截止日期已过 → 跳过
|
||
dl = parsed.get("deadline")
|
||
if dl and dl < str(date.today()):
|
||
print(f"⏩ 跳过（已截止: {dl}）")
|
||
continue
|
||
# 清理 category 多余空格
|
||
if parsed.get("category"):
|
||
parsed["category"] = parsed["category"].strip()
|
||
# 默认值：unit/quantity 缺失时自动补，避免 direv 表单验证错误
|
||
if not parsed.get("unit"):
|
||
parsed["unit"] = "批"
|
||
if not parsed.get("quantity"):
|
||
parsed["quantity"] = "1"
|
||
parsed["source"] = src
|
||
parsed["notes_path"] = l.get("html_path", l.get("pdf_path", ""))
|
||
parsed["_key"] = l["key"]
|
||
parsed["_base_dir"] = l["base_dir"]
|
||
announcements.append(parsed)
|
||
print("✅")
|
||
except Exception as e:
|
||
print(f"❌ {e}")
|
||
|
||
if not announcements:
|
||
print("无可发布公告")
|
||
return {"total": len(leads), "success": 0, "skip": len(leads), "failed": 0}
|
||
|
||
print(f"\n发布 {len(announcements)} 条...")
|
||
|
||
# 封面图（先检查缓存，缺失的才调用 MiniMax）
|
||
print("预下载封面图...")
|
||
cover_paths = {}
|
||
for i, lead in enumerate(announcements):
|
||
cat = lead.get("category", "建材").strip()
|
||
safe = re.sub(r"[^\w\u4e00-\u9fff-]", "_", cat)[:20]
|
||
cached = os.path.join("/tmp/direv_covers", f"{safe}_cover.jpg")
|
||
print(f" [{i+1}] 封面: {cat}", end=" ", flush=True)
|
||
try:
|
||
if os.path.exists(cached):
|
||
cover_paths[i] = cached
|
||
print("✅")
|
||
else:
|
||
path = await asyncio.wait_for(fetch_cover_image(cat), timeout=90)
|
||
cover_paths[i] = path
|
||
print("✅")
|
||
except asyncio.TimeoutError:
|
||
print(f"❌ 超时")
|
||
cover_paths[i] = None
|
||
except Exception as e:
|
||
print(f"❌ {e}")
|
||
cover_paths[i] = None
|
||
cover_paths[i] = None
|
||
|
||
# 发布
|
||
from playwright.async_api import async_playwright
|
||
success = 0
|
||
failed = 0
|
||
|
||
async with async_playwright() as p:
|
||
browser = await p.chromium.launch(headless=True)
|
||
for i, lead in enumerate(announcements):
|
||
print(f"\n[{i+1}/{len(announcements)}] {lead['title']}")
|
||
try:
|
||
ok = await publish_one(browser, lead, i, len(announcements), cover_paths.get(i))
|
||
if ok:
|
||
# 记录已发布
|
||
pub_file = os.path.join(lead["_base_dir"], "direv_published.json")
|
||
pub_data = {}
|
||
if os.path.exists(pub_file):
|
||
with open(pub_file) as f:
|
||
pub_data = json.load(f)
|
||
pub_data[lead["_key"]] = {
|
||
"title": lead["title"],
|
||
"published_at": datetime.now().isoformat(),
|
||
}
|
||
with open(pub_file, "w") as f:
|
||
json.dump(pub_data, f, ensure_ascii=False, indent=2)
|
||
success += 1
|
||
else:
|
||
failed += 1
|
||
except Exception as e:
|
||
print(f" ❌ 异常: {e}")
|
||
failed += 1
|
||
await browser.close()
|
||
|
||
result = {"total": len(announcements), "success": success, "skip": len(leads) - len(announcements), "failed": failed}
|
||
print(f"\n========== 完成 ==========")
|
||
print(f"✅ {success} | ❌ {failed} | ⏩ {result['skip']}")
|
||
return result
|
||
|
||
|
||
if __name__ == "__main__":
|
||
dry_run = "--dry-run" in sys.argv
|
||
result = asyncio.run(auto_publish(dry_run=dry_run))
|
||
if result.get("dry_run"):
|
||
sys.exit(0)
|
||
if result["failed"] > 0:
|
||
sys.exit(1)
|