POD 趋势感知 Agent：缓存热点模式 + 三图合成 + 热点去重/风格去重 + review 兜底
- 缓存热点批量流程（有采集缓存不触发 Google） - 简报不足直接从采集缓存生成（轻量补齐） - 三图合成（模特/印花/底图）+ 底图压缩 <2MB - 热点去重→风格去重自动切换 + 不适合类目 review 兜底 - 透明背景（background=transparent）+ 提示词清洗（敏感词/背景描述） - 任务前 basemap 校验 + 模板国家校验 + 模特任务级分配
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
"""数据源注册表（可插拔入口）。
|
||||
|
||||
config.yaml 的 ``sources: [google_trends, pinterest]`` 决定启用哪些。
|
||||
新增数据源：实现 graph/sources/base.DataSource，在此登记即可。
|
||||
"""
|
||||
from typing import Dict
|
||||
|
||||
from .base import DataSource
|
||||
from .google_trends_source import GoogleTrendsSource
|
||||
from .pinterest_source import PinterestSource
|
||||
|
||||
SOURCES: Dict[str, type] = {
|
||||
"google_trends": GoogleTrendsSource,
|
||||
"pinterest": PinterestSource,
|
||||
}
|
||||
|
||||
|
||||
def get_source(name: str) -> DataSource:
|
||||
cls = SOURCES.get(name)
|
||||
if cls is None:
|
||||
raise ValueError(f"未知数据源: {name}，可用: {list(SOURCES)}")
|
||||
return cls()
|
||||
@@ -0,0 +1,30 @@
|
||||
"""数据源抽象接口（可插拔核心）。
|
||||
|
||||
新增一个数据源只需：① 继承 DataSource 实现 fetch()；② 在 graph/sources/__init__.py
|
||||
的 SOURCES 注册表里登记。config.yaml 通过 ``sources: [google_trends, pinterest]`` 决定启用哪些。
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
class DataSource(ABC):
|
||||
#: 注册名（与 config.sources 中的字符串对应）
|
||||
name: str = "base"
|
||||
|
||||
@abstractmethod
|
||||
def fetch(
|
||||
self,
|
||||
country: str,
|
||||
country_config: Dict[str, Any],
|
||||
global_config: Dict[str, Any],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""抓取该国热点，返回统一格式行。
|
||||
|
||||
每行字段：``country, topic, seed, source, kind, raw_score``
|
||||
- source 标签用于后续权重与归一化（如 gt_trending / gt_style / gt_related / pinterest）
|
||||
- kind：trending / rising / top（用于按组归一化）
|
||||
- raw_score：Google Trends 相对指数 0-100，或 RSS 排名分
|
||||
|
||||
实现内部必须自行处理限流/重试/异常，返回空列表也不应抛异常到上层。
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Google Trends 数据源（可插拔实现）。
|
||||
|
||||
封装 pytrends + 官方 RSS，带本地缓存、指数退避重试、urllib3 兼容补丁。
|
||||
- gt_trending：国家实时趋势榜（RSS，稳定）
|
||||
- gt_style：按国家风格种子词抓 related_queries（设计灵感）
|
||||
- gt_related：按 POD 行业种子词抓 related_queries（行业交叉验证）
|
||||
|
||||
注意：related_queries 是「单关键词」接口，一次传多个词会触发 Google /sorry(429)，
|
||||
因此逐词串行 + 节流 + 快速失败。缓存按 (key, 日期) 分文件：不删除历史文件，
|
||||
24h 内读最新；超过 24h 重新抓取写当日新文件；抓取失败回退最新历史缓存兜底。
|
||||
"""
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
from urllib3.util.retry import Retry as _Retry
|
||||
|
||||
# pytrends 4.x 仍用 method_whitelist；urllib3>=2 已改名 allowed_methods。做兼容补丁。
|
||||
if "method_whitelist" not in _Retry.__init__.__code__.co_varnames:
|
||||
_orig_retry_init = _Retry.__init__
|
||||
|
||||
def _patched_retry_init(self, *args, **kwargs):
|
||||
if "method_whitelist" in kwargs:
|
||||
kwargs["allowed_methods"] = kwargs.pop("method_whitelist")
|
||||
_orig_retry_init(self, *args, **kwargs)
|
||||
|
||||
_Retry.__init__ = _patched_retry_init
|
||||
|
||||
from pytrends.request import TrendReq
|
||||
|
||||
from graph.paths import runtime_root
|
||||
from .base import DataSource
|
||||
|
||||
CACHE_DIR = runtime_root() / ".cache" / "google_trends"
|
||||
CACHE_TTL = 24 * 3600
|
||||
CACHE_VERSION = "v3"
|
||||
|
||||
|
||||
def _cache_fname(key: str, date_suffix: str = "") -> str:
|
||||
digest = hashlib.md5((CACHE_VERSION + "|" + key).encode("utf-8")).hexdigest()
|
||||
return f"{digest}.{date_suffix}.json" if date_suffix else f"{digest}.json"
|
||||
|
||||
|
||||
def _cache_date(p: Path) -> datetime.date:
|
||||
"""解析文件名里的 YYYYMMDD；无日期后缀则用 mtime。"""
|
||||
for token in p.name.split("."):
|
||||
if len(token) == 8 and token.isdigit():
|
||||
try:
|
||||
return datetime.datetime.strptime(token, "%Y%m%d").date()
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return datetime.date.fromtimestamp(p.stat().st_mtime)
|
||||
except Exception:
|
||||
return datetime.date.min
|
||||
|
||||
|
||||
def _cache_paths(key: str) -> List[Path]:
|
||||
"""该 key 的所有缓存文件（含旧版无日期后缀），按日期新旧降序。"""
|
||||
digest = hashlib.md5((CACHE_VERSION + "|" + key).encode("utf-8")).hexdigest()
|
||||
files = list(CACHE_DIR.glob(f"{digest}.*.json"))
|
||||
legacy = CACHE_DIR / f"{digest}.json"
|
||||
if legacy.exists():
|
||||
files.append(legacy)
|
||||
files.sort(key=_cache_date, reverse=True)
|
||||
return files
|
||||
|
||||
|
||||
def _cache_get(key: str):
|
||||
"""返回 24h 内有效的最新缓存；无则 None。"""
|
||||
for p in _cache_paths(key):
|
||||
try:
|
||||
fresh = (time.time() - p.stat().st_mtime) < CACHE_TTL
|
||||
except Exception:
|
||||
fresh = False
|
||||
if fresh:
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _cache_latest(key: str):
|
||||
"""取最新缓存文件内容（不限时效），用于抓取失败时的兜底（不删除缓存，取最新）。"""
|
||||
for p in _cache_paths(key):
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _cache_set(key: str, data) -> None:
|
||||
"""写当日新文件（保留历史，不覆盖）。"""
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
today = datetime.datetime.now().strftime("%Y%m%d")
|
||||
path = CACHE_DIR / _cache_fname(key, today)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def _retry(func, max_attempts=3, base_delay=3):
|
||||
last = None
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
return func()
|
||||
except Exception as e: # noqa: BLE001
|
||||
last = e
|
||||
if attempt == max_attempts - 1:
|
||||
break
|
||||
time.sleep(base_delay * (2 ** attempt))
|
||||
raise last if last else RuntimeError("retry failed")
|
||||
|
||||
|
||||
def fetch_related(keywords, geo="US", timeframe="today 3-m"):
|
||||
"""逐关键词串行请求 related_queries（单关键词接口，避免 429）。"""
|
||||
merged = {}
|
||||
for kw in keywords:
|
||||
time.sleep(3) # 节流
|
||||
|
||||
def _call(kw=kw):
|
||||
# timeout=(connect, read)：pytrends 默认 connect=2s 太短，网络波动即全挂，放宽到 10/30s
|
||||
pytrends = TrendReq(hl="en-US", tz=360, retries=2, backoff_factor=0.5, timeout=(10, 30))
|
||||
pytrends.build_payload(kw_list=[kw], timeframe=timeframe, geo=geo)
|
||||
return pytrends.related_queries()
|
||||
|
||||
try:
|
||||
data = _retry(_call, max_attempts=2, base_delay=1)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[GoogleTrends] {geo} 种子「{kw}」抓取失败(跳过): {e}")
|
||||
continue
|
||||
if isinstance(data, dict):
|
||||
merged.update(data)
|
||||
return merged
|
||||
|
||||
|
||||
def parse_related(raw, geo, source="gt_related"):
|
||||
rows = []
|
||||
for kw, payload in raw.items():
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
for kind in ("rising", "top"):
|
||||
df = payload.get(kind)
|
||||
if df is None or getattr(df, "empty", True):
|
||||
continue
|
||||
for _, r in df.iterrows():
|
||||
val = r["value"]
|
||||
if isinstance(val, str) and val.strip().lower() == "breakout":
|
||||
num = 100.0
|
||||
else:
|
||||
try:
|
||||
num = float(val)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
rows.append({
|
||||
"country": geo,
|
||||
"topic": str(r["query"]).strip(),
|
||||
"seed": kw,
|
||||
"source": source,
|
||||
"kind": kind,
|
||||
"raw_score": num,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_traffic(desc):
|
||||
m = re.search(r"([\d,]+)\+?\s*searches", desc or "", re.I)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1).replace(",", ""))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def fetch_trending(geo="US", limit=40):
|
||||
key = f"trending|{geo}|{limit}"
|
||||
cached = _cache_get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
url = f"https://trends.google.com/trending/rss?geo={geo}"
|
||||
try:
|
||||
resp = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"})
|
||||
resp.raise_for_status()
|
||||
root = ET.fromstring(resp.content)
|
||||
rows = []
|
||||
for idx, it in enumerate(root.findall(".//item")[:limit]):
|
||||
title = (it.findtext("title") or "").strip()
|
||||
if not title:
|
||||
continue
|
||||
score = _parse_traffic(it.findtext("description"))
|
||||
if score is None:
|
||||
score = float(limit - idx)
|
||||
rows.append({
|
||||
"country": geo, "topic": title, "seed": "",
|
||||
"source": "gt_trending", "kind": "trending", "raw_score": score,
|
||||
})
|
||||
_cache_set(key, rows)
|
||||
return rows
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[GoogleTrends 趋势] {geo} 抓取失败: {e}")
|
||||
latest = _cache_latest(key)
|
||||
if latest is not None:
|
||||
print(f"[GoogleTrends 趋势] {geo} 回退最新缓存({len(latest)}条)")
|
||||
return latest
|
||||
return []
|
||||
|
||||
|
||||
def get_rows(keywords, geo="US", timeframe="today 3-m", source="gt_related"):
|
||||
key = f"{','.join(keywords)}|{geo}|{timeframe}|{source}|rows"
|
||||
cached = _cache_get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
raw = fetch_related(keywords, geo=geo, timeframe=timeframe)
|
||||
rows = parse_related(raw, geo, source=source)
|
||||
if raw: # 有结果才写当日新缓存
|
||||
_cache_set(key, rows)
|
||||
return rows
|
||||
# 抓取无果（429/超时）：回退最新历史缓存，保证流水线不中断
|
||||
latest = _cache_latest(key)
|
||||
if latest is not None:
|
||||
print(f"[GoogleTrends] {geo} 种子「{','.join(keywords)}」抓取无结果，回退最新缓存({len(latest)}行)")
|
||||
return latest
|
||||
return rows
|
||||
|
||||
|
||||
class GoogleTrendsSource(DataSource):
|
||||
name = "google_trends"
|
||||
|
||||
def fetch(self, country, country_config, global_config):
|
||||
cc = country_config or {}
|
||||
trending_cfg = cc.get("trending", {})
|
||||
style_cfg = cc.get("style", {})
|
||||
related_cfg = cc.get("related", {})
|
||||
tf = cc.get("timeframe", "today 3-m")
|
||||
|
||||
rows: List[Dict[str, Any]] = []
|
||||
|
||||
# 1) 国家实时趋势榜（主源）
|
||||
if trending_cfg.get("enabled", True):
|
||||
limit = int(trending_cfg.get("limit", 40))
|
||||
rows.extend(fetch_trending(geo=country, limit=limit))
|
||||
|
||||
# 2) 风格种子词
|
||||
if style_cfg.get("enabled", True):
|
||||
seeds = style_cfg.get("seeds", []) or []
|
||||
if seeds:
|
||||
rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_style"))
|
||||
|
||||
# 3) 行业种子词
|
||||
if related_cfg.get("enabled", True):
|
||||
seeds = related_cfg.get("seed_keywords", []) or []
|
||||
if seeds:
|
||||
rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_related"))
|
||||
|
||||
return rows
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Pinterest 数据源（可插拔实现，默认不启用）。
|
||||
|
||||
Pinterest 官方 API v5 需要商业账号 + access token（pins:read scope），且 App 需通过 API Review。
|
||||
未配置或 Review 未过时直接返回空列表（不抛异常），由 config 的 sources 列表控制是否启用。
|
||||
|
||||
要启用：在 config.yaml 的 sources 里加入 "pinterest"，并填 pinterest.token / board / query。
|
||||
"""
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from .base import DataSource
|
||||
|
||||
|
||||
class PinterestSource(DataSource):
|
||||
name = "pinterest"
|
||||
|
||||
def fetch(self, country, country_config, global_config):
|
||||
cfg = (global_config or {}).get("pinterest", {}) or {}
|
||||
if not cfg.get("enabled", False):
|
||||
return []
|
||||
token = cfg.get("token", "")
|
||||
if not token:
|
||||
print("[Pinterest] 未配置 token（pinterest.enabled=true 但未填 token），跳过。")
|
||||
return []
|
||||
|
||||
# 官方 API v5 抓取逻辑（需商业账号 + Review 通过）。
|
||||
# 这里保留可插拔接口骨架；实际调用示例：
|
||||
# headers = {"Authorization": f"Bearer {token}"}
|
||||
# r = requests.get("https://api.pinterest.com/v5/pins/?query=...", headers=headers)
|
||||
# 因多数项目难以通过 Review，默认返回空，避免阻塞主流程。
|
||||
print("[Pinterest] 已配置但抓取逻辑未启用（需商业 Review）。返回空。")
|
||||
return []
|
||||
Reference in New Issue
Block a user