pod_trend_agent/graph/nodes/pinterest_scrape_node.py

"""Pinterest 参考模式节点 2/3：爬取图片（pinterest_scrape）。

对 pinterest_search 生成的搜索词（按需：每次 1 个），调 pinterest_scraper.scraper.scrape_pinterest
（Playwright 启动本地 Chrome）搜索 Pinterest 并下载图片到 output/pinterest_ref/<国家>/<搜索词>/。

- 单个搜索词失败（未登录/网络/无结果）跳过，不中断整批。
- 只有用了才标记已用：爬取成功（真正用掉该搜索词）→ 持久化已用词；
  爬取失败 → 记入本轮 attempted（不持久化），避免同轮重复生成。
- 已爬取过且图片数达标的搜索词跳过（断点续爬，避免重复开 Chrome）。
"""
import concurrent.futures
from pathlib import Path
from typing import Any, Dict, List

from graph.pinterest import (
    image_md5,
    load_image_pool,
    load_used_images,
    load_used_terms,
    merge_used,
    save_image_pool,
    save_used_terms,
)
from graph.validate import with_fallback


def _term_dir(output_dir: str, country: str, term: str) -> Path:
    safe = "".join(ch for ch in term if ch.isalnum() or ch in "-_ ").strip() or "term"
    return Path(output_dir) / "pinterest_ref" / country / safe


def _already_scraped(term_dir: Path) -> bool:
    """该搜索词已爬取过（目录里已有 ≥1 张图）→ 跳过，避免重复开 Chrome。"""
    if not term_dir.exists():
        return False
    return any(p.is_file() and p.suffix.lower() in (".jpg", ".jpeg", ".png", ".webp")
               for p in term_dir.iterdir())


@with_fallback("pinterest_scrape")
def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
    terms: List[str] = state.get("pinterest_search_terms") or []
    if not terms:
        print("[pinterest_scrape] 无搜索词，跳过爬取")
        return {"pinterest_images": {}, "errors": state.get("errors") or []}

    country = state["country"]
    config = state["config"]
    output_dir = state["output_dir"]
    errors = list(state.get("errors") or [])

    pcfg = config.get("pinterest") or {}
    images_per_term = int(pcfg.get("images_per_term", 40))
    concurrency = int(pcfg.get("scrape_concurrency", 2))
    headless = bool(pcfg.get("headless", False))
    proxy = pcfg.get("proxy") or None
    search_mode = str(pcfg.get("search_mode") or "direct").strip().lower()
    login_check = bool(pcfg.get("login_check", True))
    login_wait = bool(pcfg.get("login_wait", False))

    # 爬取前静态检测 Pinterest 登录态（不启动 Chrome，只读 .chrome_session cookies）：
    # 未登录/无会话 → 跳过本轮爬取并告警，避免每个搜索词都启动 Chrome 后才发现未登录。
    login_state: Dict[str, Any] = {"status": "unknown"}
    if login_check:
        try:
            from pinterest_scraper.pinterest_image_capture import check_login_state
            login_state = check_login_state()
        except Exception as e:  # noqa: BLE001
            login_state = {"status": "unknown", "detail": f"登录态检测失败: {e}"}
        status = login_state.get("status")
        if status in ("logged_out", "no_session"):
            print(f"[pinterest_scrape] ⚠️ 未检测到 Pinterest 登录态（{login_state.get('detail')}）。"
                  f"跳过本轮 {len(terms)} 个搜索词爬取。")
            stats = dict(state.get("stats") or {})
            stats["pinterest_scrape"] = {
                "terms": len(terms), "scraped": 0, "skipped": 0, "failed": 0,
                "images": 0, "pool": len((load_image_pool(output_dir, country).get("images")) or []),
                "login_status": status,
            }
            return {"pinterest_images": {}, "pinterest_attempted": state.get("pinterest_attempted") or [],
                    "pinterest_login": login_state, "stats": stats, "errors": errors}
        print(f"[pinterest_scrape] 登录态检测：{status}（{login_state.get('detail')}）")

    # 所有搜索词共享同一个 .chrome_session 登录态目录，Chrome 对同一 user-data-dir 是单例，
    # 并发启动会互相抢占导致 "browser has been closed"，必须串行爬取。
    if concurrency > 1:
        print(f"[pinterest_scrape] 共享登录态目录不支持并发，scrape_concurrency 强制为 1（原 {concurrency}）")
        concurrency = 1

    # 默认走代理：config.pinterest.proxy 未配置时自动探测（环境变量/系统代理/本地常见端口），
    # 本地 VPN 已开启时探测到的代理即可访问 Pinterest；代理失效时给出明确警告，避免逐词静默失败。
    if proxy is None:
        try:
            from pinterest_scraper.pinterest_image_capture import detect_proxy, get_system_proxy, _validate_proxy
            proxy = detect_proxy() or get_system_proxy()
        except Exception:  # noqa: BLE001
            proxy = None
    if not proxy:
        print("[pinterest_scrape] 警告：未检测到代理，将直连下载。国内网络通常无法访问 "
              "i.pinimg.com，请先开启代理/VPN（Clash/v2ray 等）再运行，否则图片下载会全部失败")
    elif not _validate_proxy(proxy):
        print(f"[pinterest_scrape] 警告：代理 {proxy} 无法连通外网，请检查代理/VPN 是否正常，"
              f"否则 Pinterest 将无法访问（爬取会失败）")

    results: Dict[str, List[str]] = {}
    skipped: List[str] = []
    failed: List[str] = []

    def _one(term: str) -> None:
        term_dir = _term_dir(output_dir, country, term)
        # direct 模式：固定关键词允许重复爬取（图池不足时自动再搜，Pinterest 每次可能返回不同图）；
        # llm 模式：已爬取过且达标 → 跳过（断点续爬，避免重复开 Chrome）
        if search_mode != "direct" and _already_scraped(term_dir):
            skipped.append(term)
            print(f"[pinterest_scrape] 已爬取过（跳过）: {term}")
            return
        try:
            from pinterest_scraper.scraper import scrape_pinterest
            files = scrape_pinterest(term, count=images_per_term,
                                     save_dir=str(term_dir), proxy=proxy, headless=headless,
                                     login_wait=login_wait)
            results[term] = files
        except Exception as e:  # noqa: BLE001
            failed.append(term)
            errors.append({"node": "pinterest_scrape", "type": type(e).__name__,
                           "message": f"term[{term}]: {e}", "trace": ""})
            print(f"[pinterest_scrape] 爬取失败（跳过）: {term}: {e}")

    print(f"[pinterest_scrape] 开始爬取 {len(terms)} 个搜索词（并发 {concurrency}）…")
    with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex:
        list(ex.map(_one, terms))

    # 只有用了才标记已用：爬取成功（含已爬取跳过）的词 → 持久化已用；失败词 → 本轮 attempted（不持久化）
    # direct 模式：固定关键词不拉黑（可跨轮复用），仅 llm 模式持久化已用词
    used = load_used_terms(output_dir, country)
    consumed = (list(results.keys()) + skipped) if search_mode != "direct" else []
    new_used = merge_used(used, consumed)
    if new_used != used:
        save_used_terms(output_dir, country, new_used)
        print(f"[pinterest_scrape] 已用搜索词更新：新增 {len(consumed)} 个，累计 {len(new_used)}")
    attempted = merge_used(state.get("pinterest_attempted") or [], failed)

    # 新爬取的图片注册进图池（含 md5），供分析节点按需取用；
    # 进图池前做 md5 校验去重：md5 已存在于图池 / 已拉黑（used_images）/ 本批重复 → 跳过
    pool = load_image_pool(output_dir, country)
    existing = pool.get("images") or []
    known_paths = {str(img.get("path")) for img in existing}
    known_md5s = {str(img.get("md5") or "").strip().lower() for img in existing}
    used_md5s = load_used_images(output_dir, country)
    new_imgs: List[Dict[str, Any]] = []
    seen_md5: set = set()
    for term, files in results.items():
        for f in files:
            if f in known_paths:
                continue
            m = str(image_md5(f) or "").strip().lower()
            if not m:
                continue
            if m in known_md5s or m in used_md5s or m in seen_md5:
                print(f"[pinterest_scrape] 图池 md5 去重跳过: {f}")
                continue
            seen_md5.add(m)
            new_imgs.append({"path": f, "md5": m, "term": term})
    if new_imgs:
        pool["images"] = existing + new_imgs
        save_image_pool(output_dir, country, pool)
        print(f"[pinterest_scrape] 图池新增 {len(new_imgs)} 张图片，累计 {len(pool['images'])} 张")

    # 新一批图爬取完成 → 重置 400 计数（per 种子词），记录当前种子词
    pipe = state.get("pinterest_pipeline")
    if pipe is not None and hasattr(pipe, "reset_400"):
        for term in results.keys():
            pipe.reset_400(term)

    total = sum(len(v) for v in results.values())
    stats = dict(state.get("stats") or {})
    stats["pinterest_scrape"] = {
        "terms": len(terms), "scraped": len(results), "skipped": len(skipped),
        "failed": len(failed), "images": total, "pool": len(pool.get("images") or []),
        "login_status": login_state.get("status", "unknown"),
    }
    print(f"[pinterest_scrape] 完成：{len(results)} 个搜索词，共 {total} 张图（跳过 {len(skipped)}，失败 {len(failed)}）")

    return {"pinterest_images": results, "pinterest_attempted": attempted,
            "pinterest_login": login_state, "stats": stats, "errors": errors}