POD 趋势感知 Agent:缓存热点模式 + 三图合成 + 热点去重/风格去重 + review 兜底
- 缓存热点批量流程(有采集缓存不触发 Google) - 简报不足直接从采集缓存生成(轻量补齐) - 三图合成(模特/印花/底图)+ 底图压缩 <2MB - 热点去重→风格去重自动切换 + 不适合类目 review 兜底 - 透明背景(background=transparent)+ 提示词清洗(敏感词/背景描述) - 任务前 basemap 校验 + 模板国家校验 + 模特任务级分配
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
"""节点 1/6:抓取(fetch)。
|
||||
|
||||
按 config.sources 启用各可插拔数据源,汇总统一格式行。
|
||||
单源失败不影响其它源(内部逐个 try),整体再套 with_fallback 兜底。
|
||||
"""
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from graph.sources import get_source
|
||||
from graph.validate import validate_rows, with_fallback
|
||||
|
||||
|
||||
@with_fallback("fetch")
|
||||
def fetch_node(state: Dict[str, Any]) -> Dict[str, Any]:
|
||||
country = state["country"]
|
||||
config = state["config"]
|
||||
cc = state["country_config"]
|
||||
enabled = config.get("sources") or ["google_trends"]
|
||||
rows: List[Dict[str, Any]] = []
|
||||
errors = list(state.get("errors") or [])
|
||||
|
||||
# 采集缓存优先:采集(fetch_keywords)成功后写入 output/<国>/collected_keywords.json,
|
||||
# 这里直接用(跳过 Google 重抓),避免重复撞限流;无缓存才走数据源抓取
|
||||
use_collected = (config.get("fetch") or {}).get("use_collected", True)
|
||||
if use_collected:
|
||||
try:
|
||||
import json as _json
|
||||
from pathlib import Path as _Path
|
||||
p = _Path(state.get("cache_dir") or state.get("output_dir", "")) / "collected_keywords.json"
|
||||
if p.exists():
|
||||
data = _json.loads(p.read_text(encoding="utf-8"))
|
||||
cached_rows = data.get("keywords") or []
|
||||
if cached_rows:
|
||||
rows = [dict(r) for r in cached_rows] # 已过滤去重的关键词
|
||||
print(f"[fetch] 使用采集缓存 {len(rows)} 条({country},跳过 Google 抓取)")
|
||||
stats = dict(state.get("stats") or {})
|
||||
stats["fetch"] = {"raw_rows": len(rows), "sources": ["collected_cache"], "errors": 0}
|
||||
return {"raw_rows": rows, "errors": errors, "stats": stats}
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[fetch] 读取采集缓存失败(回退数据源): {e}")
|
||||
|
||||
for name in enabled:
|
||||
try:
|
||||
src = get_source(name)
|
||||
rows.extend(src.fetch(country, cc, config))
|
||||
except Exception as e: # noqa: BLE001
|
||||
errors.append({
|
||||
"node": "fetch", "type": type(e).__name__,
|
||||
"message": f"source[{name}]: {e}", "trace": "",
|
||||
})
|
||||
print(f"[fetch] 数据源 {name} 失败(跳过): {e}")
|
||||
|
||||
rows = validate_rows(rows, "fetch")
|
||||
stats = dict(state.get("stats") or {})
|
||||
stats["fetch"] = {"raw_rows": len(rows), "sources": enabled, "errors": len(errors)}
|
||||
return {"raw_rows": rows, "errors": errors, "stats": stats}
|
||||
Reference in New Issue
Block a user