修复 Pinterest 爬取全部失败:1) 共享 .chrome_session 登录态下 Chrome 单例,并发启动互相抢占导致 browser has been closed,并发强制为 1;2) 系统代理 6696 是 SOCKS5,之前按 HTTP 代理传给 Chrome 导致 ERR_CONNECTION_CLOSED,新增代理类型探测返回 socks5:// scheme;3) 爬取前一次性校验代理,失效时给出明确警告
This commit is contained in:
+1
-1
@@ -46,7 +46,7 @@ pinterest:
|
||||
analyze_per_term: 6 # 每个搜索词最多分析几张图(生成设计简报)
|
||||
max_designs: 10 # 本次最多生成多少个设计
|
||||
ref_images_per_design: 1 # 生图时每个设计附带几张爬取图作为参考(发给生图模型)
|
||||
scrape_concurrency: 2 # 同时爬取几个搜索词(每个会开一个 Chrome 窗口)
|
||||
scrape_concurrency: 1 # 同时爬取几个搜索词(共享 .chrome_session 登录态,Chrome 单例,必须=1)
|
||||
headless: false # 爬取时是否无头(false=显示 Chrome 窗口,首次需手动登录)
|
||||
|
||||
# 跨源融合权重(按 source 标签,无需和为 1)
|
||||
|
||||
@@ -46,6 +46,23 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
|
||||
headless = bool(pcfg.get("headless", False))
|
||||
proxy = pcfg.get("proxy") or None
|
||||
|
||||
# 所有搜索词共享同一个 .chrome_session 登录态目录,Chrome 对同一 user-data-dir 是单例,
|
||||
# 并发启动会互相抢占导致 "browser has been closed",必须串行爬取。
|
||||
if concurrency > 1:
|
||||
print(f"[pinterest_scrape] 共享登录态目录不支持并发,scrape_concurrency 强制为 1(原 {concurrency})")
|
||||
concurrency = 1
|
||||
|
||||
# 一次性探测并校验代理(Pinterest 需代理才能访问;代理失效时给出明确警告,避免逐词静默失败)
|
||||
if proxy is None:
|
||||
try:
|
||||
from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy
|
||||
proxy = detect_proxy() or get_system_proxy()
|
||||
except Exception: # noqa: BLE001
|
||||
proxy = None
|
||||
if proxy and not _validate_proxy(proxy):
|
||||
print(f"[pinterest_scrape] 警告:代理 {proxy} 无法连通外网,请检查代理/VPN 是否正常,"
|
||||
f"否则 Pinterest 将无法访问(爬取会失败)")
|
||||
|
||||
results: Dict[str, List[str]] = {}
|
||||
skipped: List[str] = []
|
||||
|
||||
|
||||
@@ -39,6 +39,59 @@ def _probe_proxy(url: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _validate_proxy(proxy_url: str) -> bool:
|
||||
"""实测代理能否连通外网(短超时)。SOCKS5 用原生握手+CONNECT,HTTP 用 urllib。"""
|
||||
if proxy_url.startswith("socks5://"):
|
||||
host_port = proxy_url.split("://", 1)[1]
|
||||
host, _, port = host_port.rpartition(":")
|
||||
try:
|
||||
s = socket.create_connection((host, int(port)), timeout=3)
|
||||
try:
|
||||
s.sendall(bytes([0x05, 0x01, 0x00])) # greeting
|
||||
if s.recv(2) != b"\x05\x00":
|
||||
return False
|
||||
# CONNECT www.gstatic.com:443
|
||||
addr = b"\x03" + bytes([len("www.gstatic.com")]) + b"www.gstatic.com"
|
||||
req = bytes([0x05, 0x01, 0x00, 0x03]) + addr + (443).to_bytes(2, "big")
|
||||
s.sendall(req)
|
||||
resp = s.recv(10)
|
||||
return len(resp) >= 2 and resp[1] == 0x00
|
||||
finally:
|
||||
s.close()
|
||||
except OSError:
|
||||
return False
|
||||
return _probe_proxy(proxy_url)
|
||||
|
||||
|
||||
def _normalize_proxy(proxy: str) -> str:
|
||||
"""探测代理类型(SOCKS5 / HTTP)并补全正确 scheme。
|
||||
|
||||
系统设置里的代理端口常见是 SOCKS5(Clash/v2ray),但 Playwright/Chrome 需要显式
|
||||
socks5:// 前缀;按 HTTP 代理去连会全部 ERR_CONNECTION_CLOSED。
|
||||
"""
|
||||
proxy = proxy.strip().rstrip("/")
|
||||
if proxy.startswith(("http://", "https://", "socks5://", "socks4://")):
|
||||
return proxy
|
||||
host_port = proxy
|
||||
if "://" in proxy:
|
||||
host_port = proxy.split("://", 1)[1]
|
||||
host, _, port = host_port.rpartition(":")
|
||||
if not host or not port.isdigit():
|
||||
return f"http://{host_port}"
|
||||
try:
|
||||
s = socket.create_connection((host, int(port)), timeout=2)
|
||||
try:
|
||||
s.sendall(bytes([0x05, 0x01, 0x00])) # SOCKS5 greeting: ver5, 1 method, no-auth
|
||||
resp = s.recv(2)
|
||||
if resp == b"\x05\x00":
|
||||
return f"socks5://{host_port}"
|
||||
finally:
|
||||
s.close()
|
||||
except OSError:
|
||||
pass
|
||||
return f"http://{host_port}"
|
||||
|
||||
|
||||
def _read_windows_registry_proxy() -> str | None:
|
||||
"""直接读取 Windows 系统代理配置(设置 → 网络 → 代理),免去端口扫描猜测。
|
||||
|
||||
@@ -65,7 +118,7 @@ def _read_windows_registry_proxy() -> str | None:
|
||||
first = proxy_server.split(";")[0]
|
||||
if "=" in first:
|
||||
first = first.split("=", 1)[1]
|
||||
return ("http://" + first) if not first.startswith("http") else first
|
||||
return first
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -84,19 +137,21 @@ def detect_proxy() -> str | None:
|
||||
for env in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"):
|
||||
val = os.environ.get(env)
|
||||
if val:
|
||||
return val.rstrip("/")
|
||||
return _normalize_proxy(val.rstrip("/"))
|
||||
|
||||
# 2. 直接读 Windows 注册表里的系统代理(你手动在系统设置里填的端口,这里精确拿到)
|
||||
reg_proxy = _read_windows_registry_proxy()
|
||||
if reg_proxy:
|
||||
print(f"使用代理: {reg_proxy}(来自系统设置/注册表)")
|
||||
return reg_proxy
|
||||
proxy = _normalize_proxy(reg_proxy)
|
||||
print(f"使用代理: {proxy}(来自系统设置/注册表)")
|
||||
return proxy
|
||||
|
||||
# 3. urllib 系统代理兜底(已涵盖注册表/环境变量,跨平台)
|
||||
sys_proxy = get_system_proxy()
|
||||
if sys_proxy:
|
||||
print(f"使用代理: {sys_proxy}(系统代理)")
|
||||
return sys_proxy
|
||||
proxy = _normalize_proxy(sys_proxy)
|
||||
print(f"使用代理: {proxy}(系统代理)")
|
||||
return proxy
|
||||
|
||||
# 4. 兜底:实测本地常见代理端口(先快速判断端口是否监听,避免无谓阻塞)
|
||||
candidates = [
|
||||
@@ -129,7 +184,7 @@ def detect_proxy() -> str | None:
|
||||
except OSError:
|
||||
continue # 端口没开,直接跳过(快速)
|
||||
if _probe_proxy(f"http://{hp}"):
|
||||
return f"http://{hp}"
|
||||
return _normalize_proxy(hp)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user