diff --git a/graph/sources/google_trends_source.py b/graph/sources/google_trends_source.py index bb84460..c039818 100644 --- a/graph/sources/google_trends_source.py +++ b/graph/sources/google_trends_source.py @@ -119,20 +119,42 @@ def _retry(func, max_attempts=3, base_delay=3): raise last if last else RuntimeError("retry failed") -def fetch_related(keywords, geo="US", timeframe="today 3-m"): - """逐关键词串行请求 related_queries(单关键词接口,避免 429)。""" +def _probe_google(timeout: float = 3.0) -> bool: + """快速探测 trends.google.com 是否可达(连接+首字节)。 + + 不可达时逐词 related_queries 会每个词超时 ~24s(10s 连接 ×2 重试 + 3s 节流), + 24 个种子词要干等 10 分钟;探测失败直接跳过逐词抓取,回退缓存快速返回。 + """ + try: + resp = requests.get("https://trends.google.com/trending/rss?geo=US", + timeout=timeout, headers={"User-Agent": "Mozilla/5.0"}) + return resp.status_code < 500 + except Exception: + return False + + +def fetch_related(keywords, geo="US", timeframe="today 3-m", time_budget=40): + """逐关键词串行请求 related_queries(单关键词接口,避免 429)。 + + time_budget:整批逐词抓取的总时间预算(秒)。网络不稳/被限流时,每个词都可能 + 超时 ~10s+,24 个种子词会干等 10 分钟;超预算提前结束,回退缓存快速返回。 + """ merged = {} + t0 = time.time() for kw in keywords: + if time.time() - t0 > time_budget: + print(f"[GoogleTrends] {geo} 逐词抓取超过 {time_budget}s 预算,提前结束,回退缓存") + break time.sleep(3) # 节流 def _call(kw=kw): - # timeout=(connect, read):pytrends 默认 connect=2s 太短,网络波动即全挂,放宽到 10/30s - pytrends = TrendReq(hl="en-US", tz=360, retries=2, backoff_factor=0.5, timeout=(10, 30)) + # timeout=(connect, read):pytrends 默认 connect=2s 太短,网络波动即全挂,放宽到 5/15s + pytrends = TrendReq(hl="en-US", tz=360, retries=1, backoff_factor=0.5, timeout=(5, 15)) pytrends.build_payload(kw_list=[kw], timeframe=timeframe, geo=geo) return pytrends.related_queries() try: - data = _retry(_call, max_attempts=2, base_delay=1) + data = _retry(_call, max_attempts=1, base_delay=1) except Exception as e: # noqa: BLE001 print(f"[GoogleTrends] {geo} 种子「{kw}」抓取失败(跳过): {e}") continue @@ -180,11 +202,17 @@ def _parse_traffic(desc): return None -def fetch_trending(geo="US", limit=40): +def fetch_trending(geo="US", limit=40, skip_network=False): key = f"trending|{geo}|{limit}" cached = _cache_get(key) if cached is not None: return cached + if skip_network: + latest = _cache_latest(key) + if latest is not None: + print(f"[GoogleTrends 趋势] {geo} 网络不可达,回退最新缓存({len(latest)}条)") + return latest + return [] url = f"https://trends.google.com/trending/rss?geo={geo}" try: resp = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"}) @@ -213,12 +241,12 @@ def fetch_trending(geo="US", limit=40): return [] -def get_rows(keywords, geo="US", timeframe="today 3-m", source="gt_related"): +def get_rows(keywords, geo="US", timeframe="today 3-m", source="gt_related", time_budget=40): key = f"{','.join(keywords)}|{geo}|{timeframe}|{source}|rows" cached = _cache_get(key) if cached is not None: return cached - raw = fetch_related(keywords, geo=geo, timeframe=timeframe) + raw = fetch_related(keywords, geo=geo, timeframe=timeframe, time_budget=time_budget) rows = parse_related(raw, geo, source=source) if raw: # 有结果才写当日新缓存 _cache_set(key, rows) @@ -243,6 +271,17 @@ class GoogleTrendsSource(DataSource): rows: List[Dict[str, Any]] = [] + # 0) 快速连通性探测:trends.google.com 不可达 → 跳过逐词抓取(每个词~24s 超时, + # 24 个种子词要干等 10 分钟),只回退 trending 缓存;无缓存则返回空, + # 由 fetch_node 回退 collected_keywords.json,让「采集热点」快速返回。 + if not _probe_google(): + print(f"[GoogleTrends] {country} trends.google.com 不可达,跳过逐词抓取,回退缓存") + if trending_cfg.get("enabled", True): + rows.extend(fetch_trending(geo=country, + limit=int(trending_cfg.get("limit", 40)), + skip_network=True)) + return rows + # 1) 国家实时趋势榜(主源) if trending_cfg.get("enabled", True): limit = int(trending_cfg.get("limit", 40)) @@ -252,12 +291,14 @@ class GoogleTrendsSource(DataSource): if style_cfg.get("enabled", True): seeds = style_cfg.get("seeds", []) or [] if seeds: - rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_style")) + rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_style", + time_budget=40)) # 3) 行业种子词 if related_cfg.get("enabled", True): seeds = related_cfg.get("seed_keywords", []) or [] if seeds: - rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_related")) + rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_related", + time_budget=40)) return rows