From 4f3cd52a4012f5ae8d03fdc012144f799d8caa60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=BB=E6=B1=89=E7=86=99?= <3218485270@qq.com> Date: Mon, 24 Aug 2026 15:32:23 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E3=80=8C=E9=87=87=E9=9B=86?= =?UTF-8?q?=E7=83=AD=E7=82=B9=E3=80=8D=E9=95=BF=E6=97=B6=E9=97=B4=E6=97=A0?= =?UTF-8?q?=E5=93=8D=E5=BA=94=EF=BC=9AGoogle=20Trends=20=E4=B8=8D=E5=8F=AF?= =?UTF-8?q?=E8=BE=BE/=E9=99=90=E6=B5=81=E6=97=B6=E5=BF=AB=E9=80=9F?= =?UTF-8?q?=E5=9B=9E=E9=80=80=E7=BC=93=E5=AD=98=EF=BC=88=E8=BF=9E=E9=80=9A?= =?UTF-8?q?=E6=80=A7=E6=8E=A2=E6=B5=8B+=E9=80=90=E8=AF=8D=E6=8A=93?= =?UTF-8?q?=E5=8F=96=E6=97=B6=E9=97=B4=E9=A2=84=E7=AE=97+=E6=94=B6?= =?UTF-8?q?=E7=B4=A7=E8=B6=85=E6=97=B6=EF=BC=89=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=2024=20=E4=B8=AA=E7=A7=8D=E5=AD=90=E8=AF=8D=E5=B9=B2=E7=AD=89?= =?UTF-8?q?=2010=20=E5=88=86=E9=92=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- graph/sources/google_trends_source.py | 61 ++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/graph/sources/google_trends_source.py b/graph/sources/google_trends_source.py index bb84460..c039818 100644 --- a/graph/sources/google_trends_source.py +++ b/graph/sources/google_trends_source.py @@ -119,20 +119,42 @@ def _retry(func, max_attempts=3, base_delay=3): raise last if last else RuntimeError("retry failed") -def fetch_related(keywords, geo="US", timeframe="today 3-m"): - """逐关键词串行请求 related_queries（单关键词接口，避免 429）。""" +def _probe_google(timeout: float = 3.0) -> bool: + """快速探测 trends.google.com 是否可达（连接+首字节）。 + + 不可达时逐词 related_queries 会每个词超时 ~24s（10s 连接 ×2 重试 + 3s 节流）， + 24 个种子词要干等 10 分钟；探测失败直接跳过逐词抓取，回退缓存快速返回。 + """ + try: + resp = requests.get("https://trends.google.com/trending/rss?geo=US", + timeout=timeout, headers={"User-Agent": "Mozilla/5.0"}) + return resp.status_code < 500 + except Exception: + return False + + +def fetch_related(keywords, geo="US", timeframe="today 3-m", time_budget=40): + """逐关键词串行请求 related_queries（单关键词接口，避免 429）。 + + time_budget：整批逐词抓取的总时间预算（秒）。网络不稳/被限流时，每个词都可能 + 超时 ~10s+，24 个种子词会干等 10 分钟；超预算提前结束，回退缓存快速返回。 + """ merged = {} + t0 = time.time() for kw in keywords: + if time.time() - t0 > time_budget: + print(f"[GoogleTrends] {geo} 逐词抓取超过 {time_budget}s 预算，提前结束，回退缓存") + break time.sleep(3) # 节流 def _call(kw=kw): - # timeout=(connect, read)：pytrends 默认 connect=2s 太短，网络波动即全挂，放宽到 10/30s - pytrends = TrendReq(hl="en-US", tz=360, retries=2, backoff_factor=0.5, timeout=(10, 30)) + # timeout=(connect, read)：pytrends 默认 connect=2s 太短，网络波动即全挂，放宽到 5/15s + pytrends = TrendReq(hl="en-US", tz=360, retries=1, backoff_factor=0.5, timeout=(5, 15)) pytrends.build_payload(kw_list=[kw], timeframe=timeframe, geo=geo) return pytrends.related_queries() try: - data = _retry(_call, max_attempts=2, base_delay=1) + data = _retry(_call, max_attempts=1, base_delay=1) except Exception as e: # noqa: BLE001 print(f"[GoogleTrends] {geo} 种子「{kw}」抓取失败(跳过): {e}") continue @@ -180,11 +202,17 @@ def _parse_traffic(desc): return None -def fetch_trending(geo="US", limit=40): +def fetch_trending(geo="US", limit=40, skip_network=False): key = f"trending|{geo}|{limit}" cached = _cache_get(key) if cached is not None: return cached + if skip_network: + latest = _cache_latest(key) + if latest is not None: + print(f"[GoogleTrends 趋势] {geo} 网络不可达，回退最新缓存({len(latest)}条)") + return latest + return [] url = f"https://trends.google.com/trending/rss?geo={geo}" try: resp = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"}) @@ -213,12 +241,12 @@ def fetch_trending(geo="US", limit=40): return [] -def get_rows(keywords, geo="US", timeframe="today 3-m", source="gt_related"): +def get_rows(keywords, geo="US", timeframe="today 3-m", source="gt_related", time_budget=40): key = f"{','.join(keywords)}|{geo}|{timeframe}|{source}|rows" cached = _cache_get(key) if cached is not None: return cached - raw = fetch_related(keywords, geo=geo, timeframe=timeframe) + raw = fetch_related(keywords, geo=geo, timeframe=timeframe, time_budget=time_budget) rows = parse_related(raw, geo, source=source) if raw: # 有结果才写当日新缓存 _cache_set(key, rows) @@ -243,6 +271,17 @@ class GoogleTrendsSource(DataSource): rows: List[Dict[str, Any]] = [] + # 0) 快速连通性探测：trends.google.com 不可达 → 跳过逐词抓取（每个词~24s 超时， + # 24 个种子词要干等 10 分钟），只回退 trending 缓存；无缓存则返回空， + # 由 fetch_node 回退 collected_keywords.json，让「采集热点」快速返回。 + if not _probe_google(): + print(f"[GoogleTrends] {country} trends.google.com 不可达，跳过逐词抓取，回退缓存") + if trending_cfg.get("enabled", True): + rows.extend(fetch_trending(geo=country, + limit=int(trending_cfg.get("limit", 40)), + skip_network=True)) + return rows + # 1) 国家实时趋势榜（主源） if trending_cfg.get("enabled", True): limit = int(trending_cfg.get("limit", 40)) @@ -252,12 +291,14 @@ class GoogleTrendsSource(DataSource): if style_cfg.get("enabled", True): seeds = style_cfg.get("seeds", []) or [] if seeds: - rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_style")) + rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_style", + time_budget=40)) # 3) 行业种子词 if related_cfg.get("enabled", True): seeds = related_cfg.get("seed_keywords", []) or [] if seeds: - rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_related")) + rows.extend(get_rows(seeds, geo=country, timeframe=tf, source="gt_related", + time_budget=40)) return rows