修复 Pinterest 爬取全部失败:1) 共享 .chrome_session 登录态下 Chrome 单例,并发启动互相抢占导致 browser has been closed,并发强制为 1;2) 系统代理 6696 是 SOCKS5,之前按 HTTP 代理传给 Chrome 导致 ERR_CONNECTION_CLOSED,新增代理类型探测返回 socks5:// scheme;3) 爬取前一次性校验代理,失效时给出明确警告
This commit is contained in:
+1
-1
@@ -46,7 +46,7 @@ pinterest:
|
|||||||
analyze_per_term: 6 # 每个搜索词最多分析几张图(生成设计简报)
|
analyze_per_term: 6 # 每个搜索词最多分析几张图(生成设计简报)
|
||||||
max_designs: 10 # 本次最多生成多少个设计
|
max_designs: 10 # 本次最多生成多少个设计
|
||||||
ref_images_per_design: 1 # 生图时每个设计附带几张爬取图作为参考(发给生图模型)
|
ref_images_per_design: 1 # 生图时每个设计附带几张爬取图作为参考(发给生图模型)
|
||||||
scrape_concurrency: 2 # 同时爬取几个搜索词(每个会开一个 Chrome 窗口)
|
scrape_concurrency: 1 # 同时爬取几个搜索词(共享 .chrome_session 登录态,Chrome 单例,必须=1)
|
||||||
headless: false # 爬取时是否无头(false=显示 Chrome 窗口,首次需手动登录)
|
headless: false # 爬取时是否无头(false=显示 Chrome 窗口,首次需手动登录)
|
||||||
|
|
||||||
# 跨源融合权重(按 source 标签,无需和为 1)
|
# 跨源融合权重(按 source 标签,无需和为 1)
|
||||||
|
|||||||
@@ -46,6 +46,23 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
|
|||||||
headless = bool(pcfg.get("headless", False))
|
headless = bool(pcfg.get("headless", False))
|
||||||
proxy = pcfg.get("proxy") or None
|
proxy = pcfg.get("proxy") or None
|
||||||
|
|
||||||
|
# 所有搜索词共享同一个 .chrome_session 登录态目录,Chrome 对同一 user-data-dir 是单例,
|
||||||
|
# 并发启动会互相抢占导致 "browser has been closed",必须串行爬取。
|
||||||
|
if concurrency > 1:
|
||||||
|
print(f"[pinterest_scrape] 共享登录态目录不支持并发,scrape_concurrency 强制为 1(原 {concurrency})")
|
||||||
|
concurrency = 1
|
||||||
|
|
||||||
|
# 一次性探测并校验代理(Pinterest 需代理才能访问;代理失效时给出明确警告,避免逐词静默失败)
|
||||||
|
if proxy is None:
|
||||||
|
try:
|
||||||
|
from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy
|
||||||
|
proxy = detect_proxy() or get_system_proxy()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
proxy = None
|
||||||
|
if proxy and not _validate_proxy(proxy):
|
||||||
|
print(f"[pinterest_scrape] 警告:代理 {proxy} 无法连通外网,请检查代理/VPN 是否正常,"
|
||||||
|
f"否则 Pinterest 将无法访问(爬取会失败)")
|
||||||
|
|
||||||
results: Dict[str, List[str]] = {}
|
results: Dict[str, List[str]] = {}
|
||||||
skipped: List[str] = []
|
skipped: List[str] = []
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,59 @@ def _probe_proxy(url: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_proxy(proxy_url: str) -> bool:
|
||||||
|
"""实测代理能否连通外网(短超时)。SOCKS5 用原生握手+CONNECT,HTTP 用 urllib。"""
|
||||||
|
if proxy_url.startswith("socks5://"):
|
||||||
|
host_port = proxy_url.split("://", 1)[1]
|
||||||
|
host, _, port = host_port.rpartition(":")
|
||||||
|
try:
|
||||||
|
s = socket.create_connection((host, int(port)), timeout=3)
|
||||||
|
try:
|
||||||
|
s.sendall(bytes([0x05, 0x01, 0x00])) # greeting
|
||||||
|
if s.recv(2) != b"\x05\x00":
|
||||||
|
return False
|
||||||
|
# CONNECT www.gstatic.com:443
|
||||||
|
addr = b"\x03" + bytes([len("www.gstatic.com")]) + b"www.gstatic.com"
|
||||||
|
req = bytes([0x05, 0x01, 0x00, 0x03]) + addr + (443).to_bytes(2, "big")
|
||||||
|
s.sendall(req)
|
||||||
|
resp = s.recv(10)
|
||||||
|
return len(resp) >= 2 and resp[1] == 0x00
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return _probe_proxy(proxy_url)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_proxy(proxy: str) -> str:
|
||||||
|
"""探测代理类型(SOCKS5 / HTTP)并补全正确 scheme。
|
||||||
|
|
||||||
|
系统设置里的代理端口常见是 SOCKS5(Clash/v2ray),但 Playwright/Chrome 需要显式
|
||||||
|
socks5:// 前缀;按 HTTP 代理去连会全部 ERR_CONNECTION_CLOSED。
|
||||||
|
"""
|
||||||
|
proxy = proxy.strip().rstrip("/")
|
||||||
|
if proxy.startswith(("http://", "https://", "socks5://", "socks4://")):
|
||||||
|
return proxy
|
||||||
|
host_port = proxy
|
||||||
|
if "://" in proxy:
|
||||||
|
host_port = proxy.split("://", 1)[1]
|
||||||
|
host, _, port = host_port.rpartition(":")
|
||||||
|
if not host or not port.isdigit():
|
||||||
|
return f"http://{host_port}"
|
||||||
|
try:
|
||||||
|
s = socket.create_connection((host, int(port)), timeout=2)
|
||||||
|
try:
|
||||||
|
s.sendall(bytes([0x05, 0x01, 0x00])) # SOCKS5 greeting: ver5, 1 method, no-auth
|
||||||
|
resp = s.recv(2)
|
||||||
|
if resp == b"\x05\x00":
|
||||||
|
return f"socks5://{host_port}"
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return f"http://{host_port}"
|
||||||
|
|
||||||
|
|
||||||
def _read_windows_registry_proxy() -> str | None:
|
def _read_windows_registry_proxy() -> str | None:
|
||||||
"""直接读取 Windows 系统代理配置(设置 → 网络 → 代理),免去端口扫描猜测。
|
"""直接读取 Windows 系统代理配置(设置 → 网络 → 代理),免去端口扫描猜测。
|
||||||
|
|
||||||
@@ -65,7 +118,7 @@ def _read_windows_registry_proxy() -> str | None:
|
|||||||
first = proxy_server.split(";")[0]
|
first = proxy_server.split(";")[0]
|
||||||
if "=" in first:
|
if "=" in first:
|
||||||
first = first.split("=", 1)[1]
|
first = first.split("=", 1)[1]
|
||||||
return ("http://" + first) if not first.startswith("http") else first
|
return first
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -84,19 +137,21 @@ def detect_proxy() -> str | None:
|
|||||||
for env in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"):
|
for env in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"):
|
||||||
val = os.environ.get(env)
|
val = os.environ.get(env)
|
||||||
if val:
|
if val:
|
||||||
return val.rstrip("/")
|
return _normalize_proxy(val.rstrip("/"))
|
||||||
|
|
||||||
# 2. 直接读 Windows 注册表里的系统代理(你手动在系统设置里填的端口,这里精确拿到)
|
# 2. 直接读 Windows 注册表里的系统代理(你手动在系统设置里填的端口,这里精确拿到)
|
||||||
reg_proxy = _read_windows_registry_proxy()
|
reg_proxy = _read_windows_registry_proxy()
|
||||||
if reg_proxy:
|
if reg_proxy:
|
||||||
print(f"使用代理: {reg_proxy}(来自系统设置/注册表)")
|
proxy = _normalize_proxy(reg_proxy)
|
||||||
return reg_proxy
|
print(f"使用代理: {proxy}(来自系统设置/注册表)")
|
||||||
|
return proxy
|
||||||
|
|
||||||
# 3. urllib 系统代理兜底(已涵盖注册表/环境变量,跨平台)
|
# 3. urllib 系统代理兜底(已涵盖注册表/环境变量,跨平台)
|
||||||
sys_proxy = get_system_proxy()
|
sys_proxy = get_system_proxy()
|
||||||
if sys_proxy:
|
if sys_proxy:
|
||||||
print(f"使用代理: {sys_proxy}(系统代理)")
|
proxy = _normalize_proxy(sys_proxy)
|
||||||
return sys_proxy
|
print(f"使用代理: {proxy}(系统代理)")
|
||||||
|
return proxy
|
||||||
|
|
||||||
# 4. 兜底:实测本地常见代理端口(先快速判断端口是否监听,避免无谓阻塞)
|
# 4. 兜底:实测本地常见代理端口(先快速判断端口是否监听,避免无谓阻塞)
|
||||||
candidates = [
|
candidates = [
|
||||||
@@ -129,7 +184,7 @@ def detect_proxy() -> str | None:
|
|||||||
except OSError:
|
except OSError:
|
||||||
continue # 端口没开,直接跳过(快速)
|
continue # 端口没开,直接跳过(快速)
|
||||||
if _probe_proxy(f"http://{hp}"):
|
if _probe_proxy(f"http://{hp}"):
|
||||||
return f"http://{hp}"
|
return _normalize_proxy(hp)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user