修复 Pinterest 爬取全部失败:1) 共享 .chrome_session 登录态下 Chrome 单例,并发启动互相抢占导致 browser has been closed,并发强制为 1;2) 系统代理 6696 是 SOCKS5,之前按 HTTP 代理传给 Chrome 导致 ERR_CONNECTION_CLOSED,新增代理类型探测返回 socks5:// scheme;3) 爬取前一次性校验代理,失效时给出明确警告
This commit is contained in:
@@ -46,6 +46,23 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
|
||||
headless = bool(pcfg.get("headless", False))
|
||||
proxy = pcfg.get("proxy") or None
|
||||
|
||||
# 所有搜索词共享同一个 .chrome_session 登录态目录,Chrome 对同一 user-data-dir 是单例,
|
||||
# 并发启动会互相抢占导致 "browser has been closed",必须串行爬取。
|
||||
if concurrency > 1:
|
||||
print(f"[pinterest_scrape] 共享登录态目录不支持并发,scrape_concurrency 强制为 1(原 {concurrency})")
|
||||
concurrency = 1
|
||||
|
||||
# 一次性探测并校验代理(Pinterest 需代理才能访问;代理失效时给出明确警告,避免逐词静默失败)
|
||||
if proxy is None:
|
||||
try:
|
||||
from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy
|
||||
proxy = detect_proxy() or get_system_proxy()
|
||||
except Exception: # noqa: BLE001
|
||||
proxy = None
|
||||
if proxy and not _validate_proxy(proxy):
|
||||
print(f"[pinterest_scrape] 警告:代理 {proxy} 无法连通外网,请检查代理/VPN 是否正常,"
|
||||
f"否则 Pinterest 将无法访问(爬取会失败)")
|
||||
|
||||
results: Dict[str, List[str]] = {}
|
||||
skipped: List[str] = []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user