From ca567ae198e3ca44644430812f960f1189fcec7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=BB=E6=B1=89=E7=86=99?= <3218485270@qq.com> Date: Mon, 24 Aug 2026 15:44:50 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20Pinterest=20=E7=88=AC?= =?UTF-8?q?=E5=8F=96=E5=85=A8=E9=83=A8=E5=A4=B1=E8=B4=A5=EF=BC=9A1)=20?= =?UTF-8?q?=E5=85=B1=E4=BA=AB=20.chrome=5Fsession=20=E7=99=BB=E5=BD=95?= =?UTF-8?q?=E6=80=81=E4=B8=8B=20Chrome=20=E5=8D=95=E4=BE=8B=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=E5=90=AF=E5=8A=A8=E4=BA=92=E7=9B=B8=E6=8A=A2?= =?UTF-8?q?=E5=8D=A0=E5=AF=BC=E8=87=B4=20browser=20has=20been=20closed?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E5=8F=91=E5=BC=BA=E5=88=B6=E4=B8=BA=201?= =?UTF-8?q?=EF=BC=9B2)=20=E7=B3=BB=E7=BB=9F=E4=BB=A3=E7=90=86=206696=20?= =?UTF-8?q?=E6=98=AF=20SOCKS5=EF=BC=8C=E4=B9=8B=E5=89=8D=E6=8C=89=20HTTP?= =?UTF-8?q?=20=E4=BB=A3=E7=90=86=E4=BC=A0=E7=BB=99=20Chrome=20=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=20ERR=5FCONNECTION=5FCLOSED=EF=BC=8C=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E4=BB=A3=E7=90=86=E7=B1=BB=E5=9E=8B=E6=8E=A2=E6=B5=8B?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=20socks5://=20scheme=EF=BC=9B3)=20=E7=88=AC?= =?UTF-8?q?=E5=8F=96=E5=89=8D=E4=B8=80=E6=AC=A1=E6=80=A7=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E4=BB=A3=E7=90=86=EF=BC=8C=E5=A4=B1=E6=95=88=E6=97=B6=E7=BB=99?= =?UTF-8?q?=E5=87=BA=E6=98=8E=E7=A1=AE=E8=AD=A6=E5=91=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.yaml | 2 +- graph/nodes/pinterest_scrape_node.py | 17 +++++ pinterest_scraper/pinterest_image_capture.py | 69 ++++++++++++++++++-- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/config.yaml b/config.yaml index 509dd2e..1ec6c18 100644 --- a/config.yaml +++ b/config.yaml @@ -46,7 +46,7 @@ pinterest: analyze_per_term: 6 # 每个搜索词最多分析几张图（生成设计简报） max_designs: 10 # 本次最多生成多少个设计 ref_images_per_design: 1 # 生图时每个设计附带几张爬取图作为参考（发给生图模型） - scrape_concurrency: 2 # 同时爬取几个搜索词（每个会开一个 Chrome 窗口） + scrape_concurrency: 1 # 同时爬取几个搜索词（共享 .chrome_session 登录态，Chrome 单例，必须=1） headless: false # 爬取时是否无头（false=显示 Chrome 窗口，首次需手动登录） # 跨源融合权重（按 source 标签，无需和为 1） diff --git a/graph/nodes/pinterest_scrape_node.py b/graph/nodes/pinterest_scrape_node.py index 6b4fb81..cfb7fcf 100644 --- a/graph/nodes/pinterest_scrape_node.py +++ b/graph/nodes/pinterest_scrape_node.py @@ -46,6 +46,23 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]: headless = bool(pcfg.get("headless", False)) proxy = pcfg.get("proxy") or None + # 所有搜索词共享同一个 .chrome_session 登录态目录，Chrome 对同一 user-data-dir 是单例， + # 并发启动会互相抢占导致 "browser has been closed"，必须串行爬取。 + if concurrency > 1: + print(f"[pinterest_scrape] 共享登录态目录不支持并发，scrape_concurrency 强制为 1（原 {concurrency}）") + concurrency = 1 + + # 一次性探测并校验代理（Pinterest 需代理才能访问；代理失效时给出明确警告，避免逐词静默失败） + if proxy is None: + try: + from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy + proxy = detect_proxy() or get_system_proxy() + except Exception: # noqa: BLE001 + proxy = None + if proxy and not _validate_proxy(proxy): + print(f"[pinterest_scrape] 警告：代理 {proxy} 无法连通外网，请检查代理/VPN 是否正常，" + f"否则 Pinterest 将无法访问（爬取会失败）") + results: Dict[str, List[str]] = {} skipped: List[str] = [] diff --git a/pinterest_scraper/pinterest_image_capture.py b/pinterest_scraper/pinterest_image_capture.py index bc07474..6c3704b 100644 --- a/pinterest_scraper/pinterest_image_capture.py +++ b/pinterest_scraper/pinterest_image_capture.py @@ -39,6 +39,59 @@ def _probe_proxy(url: str) -> bool: return False +def _validate_proxy(proxy_url: str) -> bool: + """实测代理能否连通外网（短超时）。SOCKS5 用原生握手+CONNECT，HTTP 用 urllib。""" + if proxy_url.startswith("socks5://"): + host_port = proxy_url.split("://", 1)[1] + host, _, port = host_port.rpartition(":") + try: + s = socket.create_connection((host, int(port)), timeout=3) + try: + s.sendall(bytes([0x05, 0x01, 0x00])) # greeting + if s.recv(2) != b"\x05\x00": + return False + # CONNECT www.gstatic.com:443 + addr = b"\x03" + bytes([len("www.gstatic.com")]) + b"www.gstatic.com" + req = bytes([0x05, 0x01, 0x00, 0x03]) + addr + (443).to_bytes(2, "big") + s.sendall(req) + resp = s.recv(10) + return len(resp) >= 2 and resp[1] == 0x00 + finally: + s.close() + except OSError: + return False + return _probe_proxy(proxy_url) + + +def _normalize_proxy(proxy: str) -> str: + """探测代理类型（SOCKS5 / HTTP）并补全正确 scheme。 + + 系统设置里的代理端口常见是 SOCKS5（Clash/v2ray），但 Playwright/Chrome 需要显式 + socks5:// 前缀；按 HTTP 代理去连会全部 ERR_CONNECTION_CLOSED。 + """ + proxy = proxy.strip().rstrip("/") + if proxy.startswith(("http://", "https://", "socks5://", "socks4://")): + return proxy + host_port = proxy + if "://" in proxy: + host_port = proxy.split("://", 1)[1] + host, _, port = host_port.rpartition(":") + if not host or not port.isdigit(): + return f"http://{host_port}" + try: + s = socket.create_connection((host, int(port)), timeout=2) + try: + s.sendall(bytes([0x05, 0x01, 0x00])) # SOCKS5 greeting: ver5, 1 method, no-auth + resp = s.recv(2) + if resp == b"\x05\x00": + return f"socks5://{host_port}" + finally: + s.close() + except OSError: + pass + return f"http://{host_port}" + + def _read_windows_registry_proxy() -> str | None: """直接读取 Windows 系统代理配置（设置 → 网络 → 代理），免去端口扫描猜测。 @@ -65,7 +118,7 @@ def _read_windows_registry_proxy() -> str | None: first = proxy_server.split(";")[0] if "=" in first: first = first.split("=", 1)[1] - return ("http://" + first) if not first.startswith("http") else first + return first except Exception: return None @@ -84,19 +137,21 @@ def detect_proxy() -> str | None: for env in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"): val = os.environ.get(env) if val: - return val.rstrip("/") + return _normalize_proxy(val.rstrip("/")) # 2. 直接读 Windows 注册表里的系统代理（你手动在系统设置里填的端口，这里精确拿到） reg_proxy = _read_windows_registry_proxy() if reg_proxy: - print(f"使用代理: {reg_proxy}（来自系统设置/注册表）") - return reg_proxy + proxy = _normalize_proxy(reg_proxy) + print(f"使用代理: {proxy}（来自系统设置/注册表）") + return proxy # 3. urllib 系统代理兜底（已涵盖注册表/环境变量，跨平台） sys_proxy = get_system_proxy() if sys_proxy: - print(f"使用代理: {sys_proxy}（系统代理）") - return sys_proxy + proxy = _normalize_proxy(sys_proxy) + print(f"使用代理: {proxy}（系统代理）") + return proxy # 4. 兜底：实测本地常见代理端口（先快速判断端口是否监听，避免无谓阻塞） candidates = [ @@ -129,7 +184,7 @@ def detect_proxy() -> str | None: except OSError: continue # 端口没开，直接跳过（快速） if _probe_proxy(f"http://{hp}"): - return f"http://{hp}" + return _normalize_proxy(hp) return None