模板导出增强 + 模特性别分组 + 三合一提示词精简

1) 模板导出:识别「基码表-胸围」填 sku.bust(多个胸围列都填);申报价格模糊匹配多列统一按加价后价格填写;详情图文不再拼接 img_url_2;SPU 款式来源统一填「现货款」;商品产地国家简称映射(沙特→沙特阿拉伯)
2) 模特性别分组:model_features 按男女分组,按模板类目含男/女固定取对应性别模特(含 Pinterest 模式 pipeline)
3) 三合一提示词:去掉 DESIGN CONTENT 四要素描述(设计已由设计稿提供)
4) 生图尺寸:全部改为读 config 不再硬编码(设计图 compose.design_size / 合成图 compose.size / 种草图 seed_shot.size)
This commit is contained in:
2026-08-26 18:05:11 +08:00
parent e317547b8b
commit b7f429db89
93 changed files with 2708 additions and 583 deletions
+69 -9
View File
@@ -1,17 +1,26 @@
"""Pinterest 参考模式节点 2/3:爬取图片(pinterest_scrape)。
对 pinterest_search 生成的每个搜索词,调 pinterest_scraper.scraper.scrape_pinterest
Playwright 启动本地 Chrome)搜索 Pinterest 并下载图片到
output/pinterest_ref/<国家>/<搜索词>/。
对 pinterest_search 生成的搜索词(按需:每次 1 个),调 pinterest_scraper.scraper.scrape_pinterest
Playwright 启动本地 Chrome)搜索 Pinterest 并下载图片到 output/pinterest_ref/<国家>/<搜索词>/。
- 单个搜索词失败(未登录/网络/无结果)跳过,不中断整批。
- 并发数由 config.pinterest.scrape_concurrency 控制(每个并发开一个 Chrome 窗口)。
- 只有用了才标记已用:爬取成功(真正用掉该搜索词)→ 持久化已用词;
爬取失败 → 记入本轮 attempted(不持久化),避免同轮重复生成。
- 已爬取过且图片数达标的搜索词跳过(断点续爬,避免重复开 Chrome)。
"""
import concurrent.futures
from pathlib import Path
from typing import Any, Dict, List
from graph.pinterest import (
image_md5,
load_image_pool,
load_used_images,
load_used_terms,
merge_used,
save_image_pool,
save_used_terms,
)
from graph.validate import with_fallback
@@ -45,6 +54,7 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
concurrency = int(pcfg.get("scrape_concurrency", 2))
headless = bool(pcfg.get("headless", False))
proxy = pcfg.get("proxy") or None
search_mode = str(pcfg.get("search_mode") or "direct").strip().lower()
# 所有搜索词共享同一个 .chrome_session 登录态目录,Chrome 对同一 user-data-dir 是单例,
# 并发启动会互相抢占导致 "browser has been closed",必须串行爬取。
@@ -59,16 +69,22 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
proxy = detect_proxy() or get_system_proxy()
except Exception: # noqa: BLE001
proxy = None
if proxy and not _validate_proxy(proxy):
if not proxy:
print("[pinterest_scrape] 警告:未检测到代理,将直连下载。国内网络通常无法访问 "
"i.pinimg.com,请先开启代理/VPNClash/v2ray 等)再运行,否则图片下载会全部失败")
elif not _validate_proxy(proxy):
print(f"[pinterest_scrape] 警告:代理 {proxy} 无法连通外网,请检查代理/VPN 是否正常,"
f"否则 Pinterest 将无法访问(爬取会失败)")
results: Dict[str, List[str]] = {}
skipped: List[str] = []
failed: List[str] = []
def _one(term: str) -> None:
term_dir = _term_dir(output_dir, country, term)
if _already_scraped(term_dir):
# direct 模式:固定关键词允许重复爬取(图池不足时自动再搜,Pinterest 每次可能返回不同图);
# llm 模式:已爬取过且达标 → 跳过(断点续爬,避免重复开 Chrome)
if search_mode != "direct" and _already_scraped(term_dir):
skipped.append(term)
print(f"[pinterest_scrape] 已爬取过(跳过): {term}")
return
@@ -78,6 +94,7 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
save_dir=str(term_dir), proxy=proxy, headless=headless)
results[term] = files
except Exception as e: # noqa: BLE001
failed.append(term)
errors.append({"node": "pinterest_scrape", "type": type(e).__name__,
"message": f"term[{term}]: {e}", "trace": ""})
print(f"[pinterest_scrape] 爬取失败(跳过): {term}: {e}")
@@ -86,12 +103,55 @@ def pinterest_scrape_node(state: Dict[str, Any]) -> Dict[str, Any]:
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex:
list(ex.map(_one, terms))
# 只有用了才标记已用:爬取成功(含已爬取跳过)的词 → 持久化已用;失败词 → 本轮 attempted(不持久化)
# direct 模式:固定关键词不拉黑(可跨轮复用),仅 llm 模式持久化已用词
used = load_used_terms(output_dir, country)
consumed = (list(results.keys()) + skipped) if search_mode != "direct" else []
new_used = merge_used(used, consumed)
if new_used != used:
save_used_terms(output_dir, country, new_used)
print(f"[pinterest_scrape] 已用搜索词更新:新增 {len(consumed)} 个,累计 {len(new_used)}")
attempted = merge_used(state.get("pinterest_attempted") or [], failed)
# 新爬取的图片注册进图池(含 md5),供分析节点按需取用;
# 进图池前做 md5 校验去重:md5 已存在于图池 / 已拉黑(used_images)/ 本批重复 → 跳过
pool = load_image_pool(output_dir, country)
existing = pool.get("images") or []
known_paths = {str(img.get("path")) for img in existing}
known_md5s = {str(img.get("md5") or "").strip().lower() for img in existing}
used_md5s = load_used_images(output_dir, country)
new_imgs: List[Dict[str, Any]] = []
seen_md5: set = set()
for term, files in results.items():
for f in files:
if f in known_paths:
continue
m = str(image_md5(f) or "").strip().lower()
if not m:
continue
if m in known_md5s or m in used_md5s or m in seen_md5:
print(f"[pinterest_scrape] 图池 md5 去重跳过: {f}")
continue
seen_md5.add(m)
new_imgs.append({"path": f, "md5": m, "term": term})
if new_imgs:
pool["images"] = existing + new_imgs
save_image_pool(output_dir, country, pool)
print(f"[pinterest_scrape] 图池新增 {len(new_imgs)} 张图片,累计 {len(pool['images'])}")
# 新一批图爬取完成 → 重置 400 计数(per 种子词),记录当前种子词
pipe = state.get("pinterest_pipeline")
if pipe is not None and hasattr(pipe, "reset_400"):
for term in results.keys():
pipe.reset_400(term)
total = sum(len(v) for v in results.values())
stats = dict(state.get("stats") or {})
stats["pinterest_scrape"] = {
"terms": len(terms), "scraped": len(results), "skipped": len(skipped),
"images": total,
"failed": len(failed), "images": total, "pool": len(pool.get("images") or []),
}
print(f"[pinterest_scrape] 完成:{len(results)} 个搜索词,共 {total} 张图(跳过 {len(skipped)}")
print(f"[pinterest_scrape] 完成:{len(results)} 个搜索词,共 {total} 张图(跳过 {len(skipped)},失败 {len(failed)}")
return {"pinterest_images": results, "stats": stats, "errors": errors}
return {"pinterest_images": results, "pinterest_attempted": attempted,
"stats": stats, "errors": errors}