修复 Pinterest 爬取全部失败:1) 共享 .chrome_session 登录态下 Chrome 单例,并发启动互相抢占导致 browser has been closed,并发强制为 1;2) 系统代理 6696 是 SOCKS5,之前按 HTTP 代理传给 Chrome 导致 ERR_CONNECTION_CLOSED,新增代理类型探测返回 socks5:// scheme;3) 爬取前一次性校验代理,失效时给出明确警告

This commit is contained in:
2026-08-24 15:44:50 +08:00
parent 4f3cd52a40
commit ca567ae198
3 changed files with 80 additions and 8 deletions
+17
View File
@@ -46,6 +46,23 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
headless = bool(pcfg.get("headless", False))
proxy = pcfg.get("proxy") or None
# 所有搜索词共享同一个 .chrome_session 登录态目录,Chrome 对同一 user-data-dir 是单例,
# 并发启动会互相抢占导致 "browser has been closed",必须串行爬取。
if concurrency > 1:
print(f"[pinterest_scrape] 共享登录态目录不支持并发,scrape_concurrency 强制为 1(原 {concurrency}")
concurrency = 1
# 一次性探测并校验代理(Pinterest 需代理才能访问;代理失效时给出明确警告,避免逐词静默失败)
if proxy is None:
try:
from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy
proxy = detect_proxy() or get_system_proxy()
except Exception: # noqa: BLE001
proxy = None
if proxy and not _validate_proxy(proxy):
print(f"[pinterest_scrape] 警告:代理 {proxy} 无法连通外网,请检查代理/VPN 是否正常,"
f"否则 Pinterest 将无法访问(爬取会失败)")
results: Dict[str, List[str]] = {}
skipped: List[str] = []