"""Pinterest 参考模式节点 2/3：爬取图片（pinterest_scrape）。 对 pinterest_search 生成的搜索词（按需：每次 1 个），调 pinterest_scraper.scraper.scrape_pinterest （Playwright 启动本地 Chrome）搜索 Pinterest 并下载图片到 output/pinterest_ref/<国家>/<搜索词>/。 - 单个搜索词失败（未登录/网络/无结果）跳过，不中断整批。 - 只有用了才标记已用：爬取成功（真正用掉该搜索词）→ 持久化已用词； 爬取失败 → 记入本轮 attempted（不持久化），避免同轮重复生成。 - 已爬取过且图片数达标的搜索词跳过（断点续爬，避免重复开 Chrome）。 """ import concurrent.futures from pathlib import Path from typing import Any, Dict, List from graph.pinterest import ( image_md5, load_image_pool, load_used_images, load_used_terms, merge_used, save_image_pool, save_used_terms, ) from graph.validate import with_fallback def _term_dir(output_dir: str, country: str, term: str) -> Path: safe = "".join(ch for ch in term if ch.isalnum() or ch in "-_ ").strip() or "term" return Path(output_dir) / "pinterest_ref" / country / safe def _already_scraped(term_dir: Path) -> bool: """该搜索词已爬取过（目录里已有 ≥1 张图）→ 跳过，避免重复开 Chrome。""" if not term_dir.exists(): return False return any(p.is_file() and p.suffix.lower() in (".jpg", ".jpeg", ".png", ".webp") for p in term_dir.iterdir()) @with_fallback("pinterest_scrape") def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]: terms: List[str] = state.get("pinterest_search_terms") or [] if not terms: print("[pinterest_scrape] 无搜索词，跳过爬取") return {"pinterest_images": {}, "errors": state.get("errors") or []} country = state["country"] config = state["config"] output_dir = state["output_dir"] errors = list(state.get("errors") or []) pcfg = config.get("pinterest") or {} images_per_term = int(pcfg.get("images_per_term", 40)) concurrency = int(pcfg.get("scrape_concurrency", 2)) headless = bool(pcfg.get("headless", False)) proxy = pcfg.get("proxy") or None search_mode = str(pcfg.get("search_mode") or "direct").strip().lower() login_check = bool(pcfg.get("login_check", True)) login_wait = bool(pcfg.get("login_wait", False)) # 爬取前静态检测 Pinterest 登录态（不启动 Chrome，只读 .chrome_session cookies）： # 未登录/无会话 → 跳过本轮爬取并告警，避免每个搜索词都启动 Chrome 后才发现未登录。 login_state: Dict[str, Any] = {"status": "unknown"} if login_check: try: from pinterest_scraper.pinterest_image_capture import check_login_state login_state = check_login_state() except Exception as e: # noqa: BLE001 login_state = {"status": "unknown", "detail": f"登录态检测失败: {e}"} status = login_state.get("status") if status in ("logged_out", "no_session"): print(f"[pinterest_scrape] ⚠️ 未检测到 Pinterest 登录态（{login_state.get('detail')}）。" f"跳过本轮 {len(terms)} 个搜索词爬取。") stats = dict(state.get("stats") or {}) stats["pinterest_scrape"] = { "terms": len(terms), "scraped": 0, "skipped": 0, "failed": 0, "images": 0, "pool": len((load_image_pool(output_dir, country).get("images")) or []), "login_status": status, } return {"pinterest_images": {}, "pinterest_attempted": state.get("pinterest_attempted") or [], "pinterest_login": login_state, "stats": stats, "errors": errors} print(f"[pinterest_scrape] 登录态检测：{status}（{login_state.get('detail')}）") # 所有搜索词共享同一个 .chrome_session 登录态目录，Chrome 对同一 user-data-dir 是单例， # 并发启动会互相抢占导致 "browser has been closed"，必须串行爬取。 if concurrency > 1: print(f"[pinterest_scrape] 共享登录态目录不支持并发，scrape_concurrency 强制为 1（原 {concurrency}）") concurrency = 1 # 默认走代理：config.pinterest.proxy 未配置时自动探测（环境变量/系统代理/本地常见端口）， # 本地 VPN 已开启时探测到的代理即可访问 Pinterest；代理失效时给出明确警告，避免逐词静默失败。 if proxy is None: try: from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy proxy = detect_proxy() or get_system_proxy() except Exception: # noqa: BLE001 proxy = None if not proxy: print("[pinterest_scrape] 警告：未检测到代理，将直连下载。国内网络通常无法访问 " "i.pinimg.com，请先开启代理/VPN（Clash/v2ray 等）再运行，否则图片下载会全部失败") elif not _validate_proxy(proxy): print(f"[pinterest_scrape] 警告：代理 {proxy} 无法连通外网，请检查代理/VPN 是否正常，" f"否则 Pinterest 将无法访问（爬取会失败）") results: Dict[str, List[str]] = {} skipped: List[str] = [] failed: List[str] = [] def _one(term: str) -> None: term_dir = _term_dir(output_dir, country, term) # direct 模式：固定关键词允许重复爬取（图池不足时自动再搜，Pinterest 每次可能返回不同图）； # llm 模式：已爬取过且达标 → 跳过（断点续爬，避免重复开 Chrome） if search_mode != "direct" and _already_scraped(term_dir): skipped.append(term) print(f"[pinterest_scrape] 已爬取过（跳过）: {term}") return try: from pinterest_scraper.scraper import scrape_pinterest files = scrape_pinterest(term, count=images_per_term, save_dir=str(term_dir), proxy=proxy, headless=headless, login_wait=login_wait) results[term] = files except Exception as e: # noqa: BLE001 failed.append(term) errors.append({"node": "pinterest_scrape", "type": type(e).__name__, "message": f"term[{term}]: {e}", "trace": ""}) print(f"[pinterest_scrape] 爬取失败（跳过）: {term}: {e}") print(f"[pinterest_scrape] 开始爬取 {len(terms)} 个搜索词（并发 {concurrency}）…") with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex: list(ex.map(_one, terms)) # 只有用了才标记已用：爬取成功（含已爬取跳过）的词 → 持久化已用；失败词 → 本轮 attempted（不持久化） # direct 模式：固定关键词不拉黑（可跨轮复用），仅 llm 模式持久化已用词 used = load_used_terms(output_dir, country) consumed = (list(results.keys()) + skipped) if search_mode != "direct" else [] new_used = merge_used(used, consumed) if new_used != used: save_used_terms(output_dir, country, new_used) print(f"[pinterest_scrape] 已用搜索词更新：新增 {len(consumed)} 个，累计 {len(new_used)}") attempted = merge_used(state.get("pinterest_attempted") or [], failed) # 新爬取的图片注册进图池（含 md5），供分析节点按需取用； # 进图池前做 md5 校验去重：md5 已存在于图池 / 已拉黑（used_images）/ 本批重复 → 跳过 pool = load_image_pool(output_dir, country) existing = pool.get("images") or [] known_paths = {str(img.get("path")) for img in existing} known_md5s = {str(img.get("md5") or "").strip().lower() for img in existing} used_md5s = load_used_images(output_dir, country) new_imgs: List[Dict[str, Any]] = [] seen_md5: set = set() for term, files in results.items(): for f in files: if f in known_paths: continue m = str(image_md5(f) or "").strip().lower() if not m: continue if m in known_md5s or m in used_md5s or m in seen_md5: print(f"[pinterest_scrape] 图池 md5 去重跳过: {f}") continue seen_md5.add(m) new_imgs.append({"path": f, "md5": m, "term": term}) if new_imgs: pool["images"] = existing + new_imgs save_image_pool(output_dir, country, pool) print(f"[pinterest_scrape] 图池新增 {len(new_imgs)} 张图片，累计 {len(pool['images'])} 张") # 新一批图爬取完成 → 重置 400 计数（per 种子词），记录当前种子词 pipe = state.get("pinterest_pipeline") if pipe is not None and hasattr(pipe, "reset_400"): for term in results.keys(): pipe.reset_400(term) total = sum(len(v) for v in results.values()) stats = dict(state.get("stats") or {}) stats["pinterest_scrape"] = { "terms": len(terms), "scraped": len(results), "skipped": len(skipped), "failed": len(failed), "images": total, "pool": len(pool.get("images") or []), "login_status": login_state.get("status", "unknown"), } print(f"[pinterest_scrape] 完成：{len(results)} 个搜索词，共 {total} 张图（跳过 {len(skipped)}，失败 {len(failed)}）") return {"pinterest_images": results, "pinterest_attempted": attempted, "pinterest_login": login_state, "stats": stats, "errors": errors}