修复 Pinterest 爬取全部失败:1) 共享 .chrome_session 登录态下 Chrome 单例,并发启动互相抢占导致 browser has been closed,并发强制为 1;2) 系统代理 6696 是 SOCKS5,之前按 HTTP 代理传给 Chrome 导致 ERR_CONNECTION_CLOSED,新增代理类型探测返回 socks5:// scheme;3) 爬取前一次性校验代理,失效时给出明确警告

This commit is contained in:
2026-08-24 15:44:50 +08:00
parent 4f3cd52a40
commit ca567ae198
3 changed files with 80 additions and 8 deletions
+1 -1
View File
@@ -46,7 +46,7 @@ pinterest:
analyze_per_term: 6 # 每个搜索词最多分析几张图(生成设计简报)
max_designs: 10 # 本次最多生成多少个设计
ref_images_per_design: 1 # 生图时每个设计附带几张爬取图作为参考(发给生图模型)
scrape_concurrency: 2 # 同时爬取几个搜索词(每个会开一个 Chrome 窗口
scrape_concurrency: 1 # 同时爬取几个搜索词(共享 .chrome_session 登录态,Chrome 单例,必须=1
headless: false # 爬取时是否无头(false=显示 Chrome 窗口,首次需手动登录)
# 跨源融合权重(按 source 标签,无需和为 1)
+17
View File
@@ -46,6 +46,23 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
headless = bool(pcfg.get("headless", False))
proxy = pcfg.get("proxy") or None
# 所有搜索词共享同一个 .chrome_session 登录态目录,Chrome 对同一 user-data-dir 是单例,
# 并发启动会互相抢占导致 "browser has been closed",必须串行爬取。
if concurrency > 1:
print(f"[pinterest_scrape] 共享登录态目录不支持并发,scrape_concurrency 强制为 1(原 {concurrency}")
concurrency = 1
# 一次性探测并校验代理(Pinterest 需代理才能访问;代理失效时给出明确警告,避免逐词静默失败)
if proxy is None:
try:
from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy
proxy = detect_proxy() or get_system_proxy()
except Exception: # noqa: BLE001
proxy = None
if proxy and not _validate_proxy(proxy):
print(f"[pinterest_scrape] 警告:代理 {proxy} 无法连通外网,请检查代理/VPN 是否正常,"
f"否则 Pinterest 将无法访问(爬取会失败)")
results: Dict[str, List[str]] = {}
skipped: List[str] = []
+62 -7
View File
@@ -39,6 +39,59 @@ def _probe_proxy(url: str) -> bool:
return False
def _validate_proxy(proxy_url: str) -> bool:
"""实测代理能否连通外网(短超时)。SOCKS5 用原生握手+CONNECTHTTP 用 urllib。"""
if proxy_url.startswith("socks5://"):
host_port = proxy_url.split("://", 1)[1]
host, _, port = host_port.rpartition(":")
try:
s = socket.create_connection((host, int(port)), timeout=3)
try:
s.sendall(bytes([0x05, 0x01, 0x00])) # greeting
if s.recv(2) != b"\x05\x00":
return False
# CONNECT www.gstatic.com:443
addr = b"\x03" + bytes([len("www.gstatic.com")]) + b"www.gstatic.com"
req = bytes([0x05, 0x01, 0x00, 0x03]) + addr + (443).to_bytes(2, "big")
s.sendall(req)
resp = s.recv(10)
return len(resp) >= 2 and resp[1] == 0x00
finally:
s.close()
except OSError:
return False
return _probe_proxy(proxy_url)
def _normalize_proxy(proxy: str) -> str:
"""探测代理类型(SOCKS5 / HTTP)并补全正确 scheme。
系统设置里的代理端口常见是 SOCKS5Clash/v2ray),但 Playwright/Chrome 需要显式
socks5:// 前缀;按 HTTP 代理去连会全部 ERR_CONNECTION_CLOSED。
"""
proxy = proxy.strip().rstrip("/")
if proxy.startswith(("http://", "https://", "socks5://", "socks4://")):
return proxy
host_port = proxy
if "://" in proxy:
host_port = proxy.split("://", 1)[1]
host, _, port = host_port.rpartition(":")
if not host or not port.isdigit():
return f"http://{host_port}"
try:
s = socket.create_connection((host, int(port)), timeout=2)
try:
s.sendall(bytes([0x05, 0x01, 0x00])) # SOCKS5 greeting: ver5, 1 method, no-auth
resp = s.recv(2)
if resp == b"\x05\x00":
return f"socks5://{host_port}"
finally:
s.close()
except OSError:
pass
return f"http://{host_port}"
def _read_windows_registry_proxy() -> str | None:
"""直接读取 Windows 系统代理配置(设置 → 网络 → 代理),免去端口扫描猜测。
@@ -65,7 +118,7 @@ def _read_windows_registry_proxy() -> str | None:
first = proxy_server.split(";")[0]
if "=" in first:
first = first.split("=", 1)[1]
return ("http://" + first) if not first.startswith("http") else first
return first
except Exception:
return None
@@ -84,19 +137,21 @@ def detect_proxy() -> str | None:
for env in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"):
val = os.environ.get(env)
if val:
return val.rstrip("/")
return _normalize_proxy(val.rstrip("/"))
# 2. 直接读 Windows 注册表里的系统代理(你手动在系统设置里填的端口,这里精确拿到)
reg_proxy = _read_windows_registry_proxy()
if reg_proxy:
print(f"使用代理: {reg_proxy}(来自系统设置/注册表)")
return reg_proxy
proxy = _normalize_proxy(reg_proxy)
print(f"使用代理: {proxy}(来自系统设置/注册表)")
return proxy
# 3. urllib 系统代理兜底(已涵盖注册表/环境变量,跨平台)
sys_proxy = get_system_proxy()
if sys_proxy:
print(f"使用代理: {sys_proxy}(系统代理)")
return sys_proxy
proxy = _normalize_proxy(sys_proxy)
print(f"使用代理: {proxy}(系统代理)")
return proxy
# 4. 兜底:实测本地常见代理端口(先快速判断端口是否监听,避免无谓阻塞)
candidates = [
@@ -129,7 +184,7 @@ def detect_proxy() -> str | None:
except OSError:
continue # 端口没开,直接跳过(快速)
if _probe_proxy(f"http://{hp}"):
return f"http://{hp}"
return _normalize_proxy(hp)
return None