diff --git a/config.yaml b/config.yaml index 509dd2e..1ec6c18 100644 --- a/config.yaml +++ b/config.yaml @@ -46,7 +46,7 @@ pinterest: analyze_per_term: 6 # 每个搜索词最多分析几张图(生成设计简报) max_designs: 10 # 本次最多生成多少个设计 ref_images_per_design: 1 # 生图时每个设计附带几张爬取图作为参考(发给生图模型) - scrape_concurrency: 2 # 同时爬取几个搜索词(每个会开一个 Chrome 窗口) + scrape_concurrency: 1 # 同时爬取几个搜索词(共享 .chrome_session 登录态,Chrome 单例,必须=1) headless: false # 爬取时是否无头(false=显示 Chrome 窗口,首次需手动登录) # 跨源融合权重(按 source 标签,无需和为 1) diff --git a/graph/nodes/pinterest_scrape_node.py b/graph/nodes/pinterest_scrape_node.py index 6b4fb81..cfb7fcf 100644 --- a/graph/nodes/pinterest_scrape_node.py +++ b/graph/nodes/pinterest_scrape_node.py @@ -46,6 +46,23 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]: headless = bool(pcfg.get("headless", False)) proxy = pcfg.get("proxy") or None + # 所有搜索词共享同一个 .chrome_session 登录态目录,Chrome 对同一 user-data-dir 是单例, + # 并发启动会互相抢占导致 "browser has been closed",必须串行爬取。 + if concurrency > 1: + print(f"[pinterest_scrape] 共享登录态目录不支持并发,scrape_concurrency 强制为 1(原 {concurrency})") + concurrency = 1 + + # 一次性探测并校验代理(Pinterest 需代理才能访问;代理失效时给出明确警告,避免逐词静默失败) + if proxy is None: + try: + from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy + proxy = detect_proxy() or get_system_proxy() + except Exception: # noqa: BLE001 + proxy = None + if proxy and not _validate_proxy(proxy): + print(f"[pinterest_scrape] 警告:代理 {proxy} 无法连通外网,请检查代理/VPN 是否正常," + f"否则 Pinterest 将无法访问(爬取会失败)") + results: Dict[str, List[str]] = {} skipped: List[str] = [] diff --git a/pinterest_scraper/pinterest_image_capture.py b/pinterest_scraper/pinterest_image_capture.py index bc07474..6c3704b 100644 --- a/pinterest_scraper/pinterest_image_capture.py +++ b/pinterest_scraper/pinterest_image_capture.py @@ -39,6 +39,59 @@ def _probe_proxy(url: str) -> bool: return False +def _validate_proxy(proxy_url: str) -> bool: + """实测代理能否连通外网(短超时)。SOCKS5 用原生握手+CONNECT,HTTP 用 urllib。""" + if proxy_url.startswith("socks5://"): + host_port = proxy_url.split("://", 1)[1] + host, _, port = host_port.rpartition(":") + try: + s = socket.create_connection((host, int(port)), timeout=3) + try: + s.sendall(bytes([0x05, 0x01, 0x00])) # greeting + if s.recv(2) != b"\x05\x00": + return False + # CONNECT www.gstatic.com:443 + addr = b"\x03" + bytes([len("www.gstatic.com")]) + b"www.gstatic.com" + req = bytes([0x05, 0x01, 0x00, 0x03]) + addr + (443).to_bytes(2, "big") + s.sendall(req) + resp = s.recv(10) + return len(resp) >= 2 and resp[1] == 0x00 + finally: + s.close() + except OSError: + return False + return _probe_proxy(proxy_url) + + +def _normalize_proxy(proxy: str) -> str: + """探测代理类型(SOCKS5 / HTTP)并补全正确 scheme。 + + 系统设置里的代理端口常见是 SOCKS5(Clash/v2ray),但 Playwright/Chrome 需要显式 + socks5:// 前缀;按 HTTP 代理去连会全部 ERR_CONNECTION_CLOSED。 + """ + proxy = proxy.strip().rstrip("/") + if proxy.startswith(("http://", "https://", "socks5://", "socks4://")): + return proxy + host_port = proxy + if "://" in proxy: + host_port = proxy.split("://", 1)[1] + host, _, port = host_port.rpartition(":") + if not host or not port.isdigit(): + return f"http://{host_port}" + try: + s = socket.create_connection((host, int(port)), timeout=2) + try: + s.sendall(bytes([0x05, 0x01, 0x00])) # SOCKS5 greeting: ver5, 1 method, no-auth + resp = s.recv(2) + if resp == b"\x05\x00": + return f"socks5://{host_port}" + finally: + s.close() + except OSError: + pass + return f"http://{host_port}" + + def _read_windows_registry_proxy() -> str | None: """直接读取 Windows 系统代理配置(设置 → 网络 → 代理),免去端口扫描猜测。 @@ -65,7 +118,7 @@ def _read_windows_registry_proxy() -> str | None: first = proxy_server.split(";")[0] if "=" in first: first = first.split("=", 1)[1] - return ("http://" + first) if not first.startswith("http") else first + return first except Exception: return None @@ -84,19 +137,21 @@ def detect_proxy() -> str | None: for env in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"): val = os.environ.get(env) if val: - return val.rstrip("/") + return _normalize_proxy(val.rstrip("/")) # 2. 直接读 Windows 注册表里的系统代理(你手动在系统设置里填的端口,这里精确拿到) reg_proxy = _read_windows_registry_proxy() if reg_proxy: - print(f"使用代理: {reg_proxy}(来自系统设置/注册表)") - return reg_proxy + proxy = _normalize_proxy(reg_proxy) + print(f"使用代理: {proxy}(来自系统设置/注册表)") + return proxy # 3. urllib 系统代理兜底(已涵盖注册表/环境变量,跨平台) sys_proxy = get_system_proxy() if sys_proxy: - print(f"使用代理: {sys_proxy}(系统代理)") - return sys_proxy + proxy = _normalize_proxy(sys_proxy) + print(f"使用代理: {proxy}(系统代理)") + return proxy # 4. 兜底:实测本地常见代理端口(先快速判断端口是否监听,避免无谓阻塞) candidates = [ @@ -129,7 +184,7 @@ def detect_proxy() -> str | None: except OSError: continue # 端口没开,直接跳过(快速) if _probe_proxy(f"http://{hp}"): - return f"http://{hp}" + return _normalize_proxy(hp) return None