新增 Pinterest 参考模式:独立于 Google Trends 的完整链路(12国种子词池 / LLM搜索词json_schema+防重复+已用词限100 / 并发爬图 / 多模态分析→原创简报 / 生图带爬取图参考图生图 / UI流程选择)

This commit is contained in:
2026-08-24 15:02:01 +08:00
parent 2144c36e60
commit 309d4a520c
30 changed files with 2076 additions and 23 deletions
+80
View File
@@ -0,0 +1,80 @@
"""Pinterest 参考模式节点 2/3:爬取图片(pinterest_scrape)。
对 pinterest_search 生成的每个搜索词,调 pinterest_scraper.scraper.scrape_pinterest
Playwright 启动本地 Chrome)搜索 Pinterest 并下载图片到
output/pinterest_ref/<国家>/<搜索词>/。
- 单个搜索词失败(未登录/网络/无结果)跳过,不中断整批。
- 并发数由 config.pinterest.scrape_concurrency 控制(每个并发开一个 Chrome 窗口)。
- 已爬取过且图片数达标的搜索词跳过(断点续爬,避免重复开 Chrome)。
"""
import concurrent.futures
from pathlib import Path
from typing import Any, Dict, List
from graph.validate import with_fallback
def _term_dir(output_dir: str, country: str, term: str) -> Path:
safe = "".join(ch for ch in term if ch.isalnum() or ch in "-_ ").strip() or "term"
return Path(output_dir) / "pinterest_ref" / country / safe
def _already_scraped(term_dir: Path) -> bool:
"""该搜索词已爬取过(目录里已有 ≥1 张图)→ 跳过,避免重复开 Chrome。"""
if not term_dir.exists():
return False
return any(p.is_file() and p.suffix.lower() in (".jpg", ".jpeg", ".png", ".webp")
for p in term_dir.iterdir())
@with_fallback("pinterest_scrape")
def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
terms: List[str] = state.get("pinterest_search_terms") or []
if not terms:
print("[pinterest_scrape] 无搜索词,跳过爬取")
return {"pinterest_images": {}, "errors": state.get("errors") or []}
country = state["country"]
config = state["config"]
output_dir = state["output_dir"]
errors = list(state.get("errors") or [])
pcfg = config.get("pinterest") or {}
images_per_term = int(pcfg.get("images_per_term", 40))
concurrency = int(pcfg.get("scrape_concurrency", 2))
headless = bool(pcfg.get("headless", False))
proxy = pcfg.get("proxy") or None
results: Dict[str, List[str]] = {}
skipped: List[str] = []
def _one(term: str) -> None:
term_dir = _term_dir(output_dir, country, term)
if _already_scraped(term_dir):
skipped.append(term)
print(f"[pinterest_scrape] 已爬取过(跳过): {term}")
return
try:
from pinterest_scraper.scraper import scrape_pinterest
files = scrape_pinterest(term, count=images_per_term,
save_dir=str(term_dir), proxy=proxy, headless=headless)
results[term] = files
except Exception as e: # noqa: BLE001
errors.append({"node": "pinterest_scrape", "type": type(e).__name__,
"message": f"term[{term}]: {e}", "trace": ""})
print(f"[pinterest_scrape] 爬取失败(跳过): {term}: {e}")
print(f"[pinterest_scrape] 开始爬取 {len(terms)} 个搜索词(并发 {concurrency})…")
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex:
list(ex.map(_one, terms))
total = sum(len(v) for v in results.values())
stats = dict(state.get("stats") or {})
stats["pinterest_scrape"] = {
"terms": len(terms), "scraped": len(results), "skipped": len(skipped),
"images": total,
}
print(f"[pinterest_scrape] 完成:{len(results)} 个搜索词,共 {total} 张图(跳过 {len(skipped)}")
return {"pinterest_images": results, "stats": stats, "errors": errors}