305 lines
11 KiB
Python
305 lines
11 KiB
Python
"""Google Trends 数据源（可插拔实现）。
|
||
|
||
封装 pytrends + 官方 RSS，带本地缓存、指数退避重试、urllib3 兼容补丁。
|
||
- gt_trending：国家实时趋势榜（RSS，稳定）
|
||
- gt_style：按国家风格种子词抓 related_queries（设计灵感）
|
||
- gt_related：按 POD 行业种子词抓 related_queries（行业交叉验证）
|
||
|
||
注意：related_queries 是「单关键词」接口，一次传多个词会触发 Google /sorry(429)，
|
||
因此逐词串行 + 节流 + 快速失败。缓存按 (key, 日期) 分文件：不删除历史文件，
|
||
24h 内读最新；超过 24h 重新抓取写当日新文件；抓取失败回退最新历史缓存兜底。
|
||
"""
|
||
import datetime
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import time
|
||
import xml.etree.ElementTree as ET
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List
|
||
|
||
import requests
|
||
import urllib3
|
||
from urllib3.util.retry import Retry as _Retry
|
||
|
||
# pytrends 4.x 仍用 method_whitelist；urllib3>=2 已改名 allowed_methods。做兼容补丁。
|
||
if "method_whitelist" not in _Retry.__init__.__code__.co_varnames:
|
||
_orig_retry_init = _Retry.__init__
|
||
|
||
def _patched_retry_init(self, *args, **kwargs):
|
||
if "method_whitelist" in kwargs:
|
||
kwargs["allowed_methods"] = kwargs.pop("method_whitelist")
|
||
_orig_retry_init(self, *args, **kwargs)
|
||
|
||
_Retry.__init__ = _patched_retry_init
|
||
|
||
from pytrends.request import TrendReq
|
||
|
||
from graph.paths import runtime_root
|
||
from .base import DataSource
|
||
|
||
CACHE_DIR = runtime_root() / ".cache" / "google_trends"
|
||
CACHE_TTL = 24 * 3600
|
||
CACHE_VERSION = "v3"
|
||
|
||
|
||
def _cache_fname(key: str, date_suffix: str = "") -> str:
|
||
digest = hashlib.md5((CACHE_VERSION + "|" + key).encode("utf-8")).hexdigest()
|
||
return f"{digest}.{date_suffix}.json" if date_suffix else f"{digest}.json"
|
||
|
||
|
||
def _cache_date(p: Path) -> datetime.date:
|
||
"""解析文件名里的 YYYYMMDD；无日期后缀则用 mtime。"""
|
||
for token in p.name.split("."):
|
||
if len(token) == 8 and token.isdigit():
|
||
try:
|
||
return datetime.datetime.strptime(token, "%Y%m%d").date()
|
||
except ValueError:
|
||
pass
|
||
try:
|
||
return datetime.date.fromtimestamp(p.stat().st_mtime)
|
||
except Exception:
|
||
return datetime.date.min
|
||
|
||
|
||
def _cache_paths(key: str) -> List[Path]:
|
||
"""该 key 的所有缓存文件（含旧版无日期后缀），按日期新旧降序。"""
|
||
digest = hashlib.md5((CACHE_VERSION + "|" + key).encode("utf-8")).hexdigest()
|
||
files = list(CACHE_DIR.glob(f"{digest}.*.json"))
|
||
legacy = CACHE_DIR / f"{digest}.json"
|
||
if legacy.exists():
|
||
files.append(legacy)
|
||
files.sort(key=_cache_date, reverse=True)
|
||
return files
|
||
|
||
|
||
def _cache_get(key: str):
|
||
"""返回 24h 内有效的最新缓存；无则 None。"""
|
||
for p in _cache_paths(key):
|
||
try:
|
||
fresh = (time.time() - p.stat().st_mtime) < CACHE_TTL
|
||
except Exception:
|
||
fresh = False
|
||
if fresh:
|
||
try:
|
||
return json.loads(p.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _cache_latest(key: str):
|
||
"""取最新缓存文件内容（不限时效），用于抓取失败时的兜底（不删除缓存，取最新）。"""
|
||
for p in _cache_paths(key):
|
||
try:
|
||
return json.loads(p.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _cache_set(key: str, data) -> None:
|
||
"""写当日新文件（保留历史，不覆盖）。"""
|
||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||
today = datetime.datetime.now().strftime("%Y%m%d")
|
||
path = CACHE_DIR / _cache_fname(key, today)
|
||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
def _retry(func, max_attempts=3, base_delay=3):
|
||
last = None
|
||
for attempt in range(max_attempts):
|
||
try:
|
||
return func()
|
||
except Exception as e: # noqa: BLE001
|
||
last = e
|
||
if attempt == max_attempts - 1:
|
||
break
|
||
time.sleep(base_delay * (2 ** attempt))
|
||
raise last if last else RuntimeError("retry failed")
|
||
|
||
|
||
def _probe_google(timeout: float = 3.0) -> bool:
|
||
"""快速探测 trends.google.com 是否可达（连接+首字节）。
|
||
|
||
不可达时逐词 related_queries 会每个词超时 ~24s（10s 连接 ×2 重试 + 3s 节流），
|
||
24 个种子词要干等 10 分钟；探测失败直接跳过逐词抓取，回退缓存快速返回。
|
||
"""
|
||
try:
|
||
resp = requests.get("https://trends.google.com/trending/rss?geo=US",
|
||
timeout=timeout, headers={"User-Agent": "Mozilla/5.0"})
|
||
return resp.status_code < 500
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def fetch_related(keywords, geo="US", timeframe="today 3-m", time_budget=40):
|
||
"""逐关键词串行请求 related_queries（单关键词接口，避免 429）。
|
||
|
||
time_budget：整批逐词抓取的总时间预算（秒）。网络不稳/被限流时，每个词都可能
|
||
超时 ~10s+，24 个种子词会干等 10 分钟；超预算提前结束，回退缓存快速返回。
|
||
"""
|
||
merged = {}
|
||
t0 = time.time()
|
||
for kw in keywords:
|
||
if time.time() - t0 > time_budget:
|
||
print(f"[GoogleTrends] {geo} 逐词抓取超过 {time_budget}s 预算，提前结束，回退缓存")
|
||
break
|
||
time.sleep(3) # 节流
|
||
|
||
def _call(kw=kw):
|
||
# timeout=(connect, read)：pytrends 默认 connect=2s 太短，网络波动即全挂，放宽到 5/15s
|
||
pytrends = TrendReq(hl="en-US", tz=360, retries=1, backoff_factor=0.5, timeout=(5, 15))
|
||
pytrends.build_payload(kw_list=[kw], timeframe=timeframe, geo=geo)
|
||
return pytrends.related_queries()
|
||
|
||
try:
|
||
data = _retry(_call, max_attempts=1, base_delay=1)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f"[GoogleTrends] {geo} 种子「{kw}」抓取失败(跳过): {e}")
|
||
continue
|
||
if isinstance(data, dict):
|
||
merged.update(data)
|
||
return merged
|
||
|
||
|
||
def parse_related(raw, geo, source="gt_related"):
|
||
rows = []
|
||
for kw, payload in raw.items():
|
||
if not isinstance(payload, dict):
|
||
continue
|
||
for kind in ("rising", "top"):
|
||
df = payload.get(kind)
|
||
if df is None or getattr(df, "empty", True):
|
||
continue
|
||
for _, r in df.iterrows():
|
||
val = r["value"]
|
||
if isinstance(val, str) and val.strip().lower() == "breakout":
|
||
num = 100.0
|
||
else:
|
||
try:
|
||
num = float(val)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
rows.append({
|
||
"country": geo,
|
||
"topic": str(r["query"]).strip(),
|
||
"seed": kw,
|
||
"source": source,
|
||
"kind": kind,
|
||
"raw_score": num,
|
||
})
|
||
return rows
|
||
|
||
|
||
def _parse_traffic(desc):
|
||
m = re.search(r"([\d,]+)\+?\s*searches", desc or "", re.I)
|
||
if m:
|
||
try:
|
||
return float(m.group(1).replace(",", ""))
|
||
except ValueError:
|
||
return None
|
||
return None
|
||
|
||
|
||
def fetch_trending(geo="US", limit=40, skip_network=False):
|
||
key = f"trending|{geo}|{limit}"
|
||
cached = _cache_get(key)
|
||
if cached is not None:
|
||
return cached
|
||
if skip_network:
|
||
latest = _cache_latest(key)
|
||
if latest is not None:
|
||
print(f"[GoogleTrends 趋势] {geo} 网络不可达，回退最新缓存({len(latest)}条)")
|
||
return latest
|
||
return []
|
||
url = f"https://trends.google.com/trending/rss?geo={geo}"
|
||
try:
|
||
resp = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"})
|
||
resp.raise_for_status()
|
||
root = ET.fromstring(resp.content)
|
||
rows = []
|
||
for idx, it in enumerate(root.findall(".//item")[:limit]):
|
||
title = (it.findtext("title") or "").strip()
|
||
if not title:
|
||
continue
|
||
score = _parse_traffic(it.findtext("description"))
|
||
if score is None:
|
||
score = float(limit - idx)
|
||
rows.append({
|
||
"country": geo, "topic": title, "seed": "",
|
||
"source": "gt_trending", "kind": "trending", "raw_score": score,
|
||
})
|
||
_cache_set(key, rows)
|
||
return rows
|
||
except Exception as e: # noqa: BLE001
|
||
print(f"[GoogleTrends 趋势] {geo} 抓取失败: {e}")
|
||
latest = _cache_latest(key)
|
||
if latest is not None:
|
||
print(f"[GoogleTrends 趋势] {geo} 回退最新缓存({len(latest)}条)")
|
||
return latest
|
||
return []
|
||
|
||
|
||
def get_rows(keywords, geo="US", timeframe="today 3-m", source="gt_related", time_budget=40):
|
||
key = f"{','.join(keywords)}|{geo}|{timeframe}|{source}|rows"
|
||
cached = _cache_get(key)
|
||
if cached is not None:
|
||
return cached
|
||
raw = fetch_related(keywords, geo=geo, timeframe=timeframe, time_budget=time_budget)
|
||
rows = parse_related(raw, geo, source=source)
|
||
if raw: # 有结果才写当日新缓存
|
||
_cache_set(key, rows)
|
||
return rows
|
||
# 抓取无果（429/超时）：回退最新历史缓存，保证流水线不中断
|
||
latest = _cache_latest(key)
|
||
if latest is not None:
|
||
print(f"[GoogleTrends] {geo} 种子「{','.join(keywords)}」抓取无结果，回退最新缓存({len(latest)}行)")
|
||
return latest
|
||
return rows
|
||
|
||
|
||
class GoogleTrendsSource(DataSource):
|
||
name = "google_trends"
|
||
|
||
def fetch(self, country, country_config, global_config):
|
||
cc = country_config or {}
|
||
trending_cfg = cc.get("trending", {})
|
||
style_cfg = cc.get("style", {})
|
||
related_cfg = cc.get("related", {})
|
||
tf = cc.get("timeframe", "today 3-m")
|
||
|
||
rows: List[Dict[str, Any]] = []
|
||
|
||
# 0) 快速连通性探测：trends.google.com 不可达 → 跳过逐词抓取（每个词~24s 超时，
|
||
# 24 个种子词要干等 10 分钟），只回退 trending 缓存；无缓存则返回空，
|
||
# 由 fetch_node 回退 collected_keywords.json，让「采集热点」快速返回。
|
||
if not _probe_google():
|
||
print(f"[GoogleTrends] {country} trends.google.com 不可达，跳过逐词抓取，回退缓存")
|
||
if trending_cfg.get("enabled", True):
|
||
rows.extend(fetch_trending(geo=country,
|
||
limit=int(trending_cfg.get("limit", 40)),
|
||
skip_network=True))
|
||
return rows
|
||
|
||
# 1) 国家实时趋势榜（主源）
|
||
if trending_cfg.get("enabled", True):
|
||
limit = int(trending_cfg.get("limit", 40))
|
||
rows.extend(fetch_trending(geo=country, limit=limit))
|
||
|
||
# 2) 风格种子词
|
||
if style_cfg.get("enabled", True):
|
||
seeds = style_cfg.get("seeds", []) or []
|
||
if seeds:
|
||
rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_style",
|
||
time_budget=40))
|
||
|
||
# 3) 行业种子词
|
||
if related_cfg.get("enabled", True):
|
||
seeds = related_cfg.get("seed_keywords", []) or []
|
||
if seeds:
|
||
rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_related",
|
||
time_budget=40))
|
||
|
||
return rows
|