"""Pinterest 简报池 + 并发生成流水线。 分析节点产出简报后立即推入简报池，后台 worker 逐条并发生成： 生成设计(compose) → 三合一(product) → OSS上传 → 生成种草图(seed_shot) 不等全部分析完，边分析边生成，显著缩短总耗时。 集成： pinterest_init 创建 PinterestPipeline（存 state["pinterest_pipeline"]） pinterest_analyze 每批产出简报 → pipe.add_briefs(new_briefs) pinterest_finalize → pipe.finish()（排空 + 合并产品 + 补写报告）→ template_export 并发上限与 product_node 一致（默认 5），避免压垮图像网关。 """ import concurrent.futures import re import threading import time from pathlib import Path from typing import Any, Dict, List, Optional from graph.paths import project_root, runtime_root class PinterestPipeline: def __init__(self, state: Dict[str, Any]): self.config = state["config"] self.country = state.get("country", "") self.output_dir = Path(state["output_dir"]) self.cache_dir = Path(state.get("cache_dir") or self.output_dir) self.task_timestamp = str(state.get("task_timestamp") or time.strftime("%Y%m%d%H%M%S")) # 简报池 + 信号 self._lock = threading.Lock() self._cond = threading.Condition(self._lock) self._briefs: List[Dict[str, Any]] = [] self._done = False self._cursor = 0 # 结果 self._products: List[Dict[str, Any]] = [] self._products_lock = threading.Lock() self._errors: List[Dict[str, Any]] = [] self._errors_lock = threading.Lock() # 路径解析（复用 product_node 的 _abs 逻辑：运行根优先，其次数据根） pcfg = self.config.get("product") or {} def _abs(key: str, default: str) -> Path: p = Path(pcfg.get(key, default)) if p.is_absolute(): return p for root in (runtime_root(), project_root()): cand = root / p if cand.exists(): return cand return project_root() / p self._db_path = _abs("db_path", "db/spu_sku.db") self._basemap_root = _abs("basemap_dir", "basemap") self._material_root = _abs("material_library_dir", "material_library") self._category = pcfg.get("model_category", "T-shirt") self._prefix = str(pcfg.get("code_prefix") or "DG").strip() # 任务清单（spu_tasks → [(spu, skus)]，简报按序号绑定） self._worklist = self._build_worklist() # 图像后端（compose + product 共用）/ 标题后端 self._ib = self._init_image_backend() self._title_backend = self._init_title_backend() # 分析后端（失败/侵权时从图池补充图片重新分析用） self._analyze_backend = self._init_analyze_backend() self._country_config = state.get("country_config") or {} # 补充重试次数：设计生成失败/侵权时，从图池取新图重新分析的最多尝试次数 self._supply_attempts = int((self.config.get("pinterest") or {}).get("supply_attempts", 3)) # 400 计数（per 种子词）：多模态 + 生图模型合计，超限放弃当前种子词 self._err400_lock = threading.Lock() self._err400_count = 0 self._err400_limit = int((self.config.get("pinterest") or {}).get("err400_limit", 15)) self._err400_aborted = False self._err400_term = "" # 模特分配（一个 SPU 一个模特，SPU>模特数循环兜底） self._model_assign = self._assign_models() # 材质映射（seed_shot 用） self._material_map = self._load_materials() # 类目 → 性别（seed_shot 用）：模版「类目」表头值含「男」→ 男模；含「女」→ 女模；都不含 → 全部随机 self._gender = None tp = str((pcfg.get("template_path") or "") or "").strip() if tp: from graph.seed_shot import read_template_category, gender_from_category category = read_template_category(tp) self._gender = gender_from_category(category) if self._gender: print(f"[pinterest_pipeline] 类目「{category[:30]}…」含{'男' if self._gender == 'male' else '女'} → 固定 {self._gender} 模特") elif category: print(f"[pinterest_pipeline] 类目「{category[:30]}…」无男/女 → 男女模特随机") # OSS self._oss_cfg = self.config.get("oss") or {} self._oss_enabled = bool(self._oss_cfg.get("enabled", True)) and bool( self._oss_cfg.get("oss_bucket")) self.oss_seq = int(state.get("oss_seq") or 0) self._oss_lock = threading.Lock() # 并发线程池（默认 5，与 product_node 上限一致） concurrency = int(pcfg.get("concurrency") or 0) or 5 self._pool = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) # 分发线程：拉简报 → 逐条提交线程池 self._dispatcher = threading.Thread(target=self._dispatch, daemon=True) self._dispatcher.start() print(f"[pinterest_pipeline] 简报池启动：{len(self._worklist)} 个产品任务，" f"并发 {concurrency}（{self.country}）") # ------------------------------------------------------------------ # # 初始化辅助 # ------------------------------------------------------------------ # def _build_worklist(self) -> List[tuple]: pcfg = self.config.get("product") or {} spu_tasks = pcfg.get("spu_tasks") or [] worklist: List[tuple] = [] if not spu_tasks: return worklist try: from graph.product import list_spus spus = list_spus(str(self._db_path)) except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] SPU 读取失败: {e}") spus = [] for t in spu_tasks: code = (t.get("spu") or t.get("spu_code") or "").strip() spu = next((s for s in spus if s["code"] == code), None) if spu is None: print(f"[pinterest_pipeline] 任务款号 {code} 不在 db，跳过") continue worklist.append((spu, (t.get("skus") or "").strip())) return worklist def _init_image_backend(self): compose_cfg = self.config.get("compose") or {} backend_name = (compose_cfg.get("backend") or "").strip() if not backend_name: return None try: from graph.backends import get_image_backend ib = get_image_backend(backend_name) if ib is not None: ib.bind_config(compose_cfg) return ib except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 图像后端不可用: {e}") return None def _init_title_backend(self): ls_cfg = self.config.get("llm_screen") or {} if (ls_cfg.get("provider") or "") in ("", "mock"): return None try: from graph.llms import get_backend as _glb tb = _glb(ls_cfg.get("provider")) tb.bind_config(ls_cfg) if getattr(tb, "has_key", False): return tb except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 标题后端初始化失败: {e}") return None def _init_analyze_backend(self): """图片分析后端（失败/侵权时从图池补充图片重新分析用）。""" pcfg = self.config.get("pinterest") or {} provider = str(pcfg.get("provider") or "openai").strip().lower() if provider == "static": return None try: from graph.llms import get_backend llm = get_backend(provider) if hasattr(llm, "bind_config"): llm.bind_config(self.config.get("llm_screen") or {}) if provider not in ("mock",) and not getattr(llm, "has_key", False): print(f"[pinterest_pipeline] {provider} 未配置 API key，降级 mock") llm = get_backend("mock") return llm except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 分析后端初始化失败: {e}") return None def _assign_models(self) -> Dict[str, Any]: model_assign: Dict[str, Any] = {} try: from graph.product import find_first_model_folder _folder, _all_models = find_first_model_folder(self._material_root, self._category) except Exception: # noqa: BLE001 _all_models = [] if _all_models: seen: Dict[str, str] = {} for _i, (spu, _skus) in enumerate(self._worklist): code = spu.get("code", "") if code not in seen: seen[code] = _all_models[_i % len(_all_models)] model_assign[code] = seen[code] return model_assign def _load_materials(self) -> Dict[str, str]: material_map: Dict[str, str] = {} try: from graph.product import list_spus for s in list_spus(str(self._db_path)): m = " ".join(str(s.get("material", "")).replace("\r", " ").replace("\n", " ").split()) material_map[s["code"]] = m except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 材质读取失败（用空）: {e}") return material_map # ------------------------------------------------------------------ # # 对外接口 # ------------------------------------------------------------------ # def reset_400(self, term: str = "") -> None: """新一批图爬取完成后调用：重置 400 计数并记录当前种子词。""" with self._err400_lock: self._err400_count = 0 self._err400_aborted = False self._err400_term = term or "" def record_400(self) -> bool: """记录一次 400（含内容/图片）。返回 True 表示本次触发放弃当前种子词。""" with self._err400_lock: self._err400_count += 1 if self._err400_count > self._err400_limit and not self._err400_aborted: self._err400_aborted = True return True return False def is_400_aborted(self) -> bool: with self._err400_lock: return self._err400_aborted def _abort_current_term(self) -> None: """放弃当前种子词：清空其未完成简报 + 图池未消费图片（已完成的保留）。""" term = self._err400_term with self._cond: kept = [b for b in self._briefs if not self._brief_of_term(b, term)] dropped = len(self._briefs) - len(kept) self._briefs = kept if dropped: print(f"[pinterest_pipeline] 放弃「{term}」未完成简报 {dropped} 条") self._drop_term_pool(term) @staticmethod def _brief_of_term(b: Dict[str, Any], term: str) -> bool: """简报是否属于某种子词（topic 去掉 #N 后缀后 == term）。""" if not term: return False topic = str(b.get("topic") or "").strip() base = re.sub(r"\s+#\d+$", "", topic).strip().lower() return bool(base) and base == term.strip().lower() def _drop_term_pool(self, term: str) -> None: """清除图池中属于当前种子词的图片，并把它们的 md5 全部拉黑（used_images.json）。 400 超限说明这批图反复触发内容/图片 400，整批拉黑防止下次重新爬取到相同图再次触发。 """ if not term: return try: from graph.pinterest import ( load_image_pool, save_image_pool, load_used_images, save_used_images, ) pool = load_image_pool(str(self.output_dir), self.country) imgs = pool.get("images") or [] term_imgs = [img for img in imgs if str(img.get("term") or "").strip().lower() == term.strip().lower()] kept = [img for img in imgs if img not in term_imgs] if len(kept) < len(imgs): pool["images"] = kept save_image_pool(str(self.output_dir), self.country, pool) print(f"[pinterest_pipeline] 清除图池「{term}」图片 {len(imgs) - len(kept)} 张") md5s = [str(img.get("md5") or "").strip().lower() for img in term_imgs] md5s = [m for m in md5s if m] if md5s: used = load_used_images(str(self.output_dir), self.country) before = len(used) used.update(md5s) if len(used) > before: save_used_images(str(self.output_dir), self.country, used) print(f"[pinterest_pipeline] 400 超限：拉黑「{term}」图片 md5 {len(md5s)} 个") except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 清除图池失败: {e}") def add_briefs(self, briefs: List[Dict[str, Any]]) -> None: if not briefs: return with self._cond: self._briefs.extend(briefs) self._cond.notify_all() print(f"[pinterest_pipeline] 简报池 +{len(briefs)} 条（待处理 {len(self._briefs)}）") def finish(self) -> tuple: """排空简报池、等待全部产品完成，返回 (products, errors)。""" with self._cond: self._done = True self._cond.notify_all() self._dispatcher.join() self._pool.shutdown(wait=True) with self._products_lock: products = list(self._products) with self._errors_lock: errors = list(self._errors) print(f"[pinterest_pipeline] 收尾：完成 {len(products)} 个产品，错误 {len(errors)}") return products, errors # ------------------------------------------------------------------ # # 后台线程 # ------------------------------------------------------------------ # def _dispatch(self) -> None: while True: with self._cond: while not self._briefs and not self._done: self._cond.wait() if self._done and not self._briefs: break batch = self._briefs self._briefs = [] for b in batch: with self._cond: idx = self._cursor self._cursor += 1 self._pool.submit(self._process_one, b, idx) # ------------------------------------------------------------------ # # 单条简报完整链路：设计 → 三合一 → OSS → 种草图 # ------------------------------------------------------------------ # def _process_one(self, brief: Dict[str, Any], idx: int) -> None: try: # 0) 先定货号：整条链路（设计/三合一/种草图）都用它命名与匹配，避免序号错位 if idx >= len(self._worklist): print(f"[pinterest_pipeline] 简报 {idx} 无对应产品任务，跳过") return spu, skus = self._worklist[idx] img_code = f"{self._prefix}{idx:03d}" # 1) 生成设计（compose）——直接按货号命名 designs/{img_code}_design.png design_path = self._gen_design(brief, img_code) if self.is_400_aborted(): # 当前种子词 400 超限已放弃：正在生成的当个也放弃，不进入后续链路 print(f"[pinterest_pipeline] 当前种子词 400 超限已放弃，跳过简报 {idx}") return if not design_path: # 生成失败/侵权（MD5 全局重复/API 错误）→ 从图池补充图片重新分析，最多尝试 N 次； # 图池不足 → 返回 None，由路由在下一轮触发搜索 for _ in range(self._supply_attempts): new_brief = self._supply_from_pool( reason=f"简报「{brief.get('topic','')}」设计生成失败") if new_brief is None: return brief = new_brief design_path = self._gen_design(brief, img_code) if design_path: break if not design_path: return brief["design_path"] = design_path # 2) 三合一（product）——同一货号 prod = self._process_spu(brief, spu, skus, img_code, design_path) if not prod: return # 3) OSS 上传 self._upload_product(prod) # 4) 种草图——同一货号 self._seed_shot(prod) # 去重记录 try: from graph.nodes.product_node import _record_used _record_used(self.cache_dir, prod) except Exception: # noqa: BLE001 pass with self._products_lock: self._products.append(prod) print(f"[pinterest_pipeline] 产品完成: {prod.get('img_code', '')}" f"（累计 {len(self._products)}）") except Exception as e: # noqa: BLE001 with self._errors_lock: self._errors.append({"node": "pinterest_pipeline", "type": type(e).__name__, "message": f"简报 {idx} 处理失败: {e}", "trace": ""}) print(f"[pinterest_pipeline] 简报 {idx} 处理失败: {e}") def _gen_design(self, brief: Dict[str, Any], img_code: str) -> Optional[str]: if self._ib is None: return None try: from graph.nodes.compose_node import generate_design design_dir = self.output_dir / "designs" design_dir.mkdir(parents=True, exist_ok=True) def _on_400(): if self.record_400(): self._abort_current_term() return generate_design(self._ib, brief, design_dir, img_code, self._errors, on_400=_on_400, size=str((self.config.get("compose") or {}).get("design_size") or "1024x1024")) except Exception as e: # noqa: BLE001 with self._errors_lock: self._errors.append({"node": "compose", "type": type(e).__name__, "message": f"设计稿生成失败 {brief.get('topic', '')}: {e}", "trace": ""}) return None def _supply_from_pool(self, reason: str) -> Optional[Dict[str, Any]]: """生成失败/侵权时，从图池取一张未消费图片重新分析，产出新简报。 图池不足 → 返回 None（由路由在下一轮触发搜索）。该图片分析后 md5 一律拉黑 （合适/不合适都拉黑），避免重复分析。 """ from graph.pinterest import ( compress_image, load_image_pool, load_used_images, pool_unused_images, save_used_images, ) pool = load_image_pool(str(self.output_dir), self.country) used = load_used_images(str(self.output_dir), self.country) unused = pool_unused_images(pool, used) if not unused: print(f"[pinterest_pipeline] 图池无未消费图片，无法补充（{reason}），等待路由搜索") return None img = unused[0] compressed = compress_image(img["path"]) llm = self._analyze_backend if llm is None or not hasattr(llm, "analyze_pinterest_images"): return None try: def _on_400(): if self.record_400(): self._abort_current_term() res = llm.analyze_pinterest_images([compressed], img.get("term", ""), self.country, on_400=_on_400) or [] except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 补充分析失败: {e}") res = [] # 该图片已消费 → 拉黑（合适/不合适都拉黑） if img.get("md5"): used.add(str(img["md5"]).lower()) save_used_images(str(self.output_dir), self.country, used) if not res or not isinstance(res[0], dict): return None b = res[0] from graph.nodes.pinterest_analyze_node import _brief_suitable if not _brief_suitable(b): print(f"[pinterest_pipeline] 补充简报侵权/不适合印花，丢弃: {b.get('topic','')}") return None b["ref_images"] = [img["path"]] b["source_md5"] = str(img.get("md5") or "").strip().lower() try: from graph.nodes.pinterest_analyze_node import _enrich_briefs from graph.nodes.prompt_node import prompt_node screened = _enrich_briefs([b], self.country) if not screened: return None r = prompt_node({ "config": self.config, "country": self.country, "country_config": self._country_config, "screened": screened, }) new_briefs = r.get("briefs") or [] if new_briefs: print(f"[pinterest_pipeline] 图池补充成功（{reason}）: {new_briefs[0].get('topic','')}") return new_briefs[0] except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 补充简报装配失败: {e}") return None def _process_spu(self, brief: Dict[str, Any], spu, skus: str, img_code: str, design_path: str) -> Optional[Dict[str, Any]]: from graph.nodes.product_node import _process_spu as _ps prod_dir = self.output_dir / "product" prod_dir.mkdir(parents=True, exist_ok=True) # 设计稿已按货号命名（designs/{img_code}_design.png），直接复用，无需再拷贝 brief = dict(brief) brief["design_path"] = design_path r = _ps(self._db_path, self._basemap_root, self._material_root, self._category, prod_dir, brief, self._ib, spu, skus, self.config.get("product") or {}, self._errors, design_path, self._title_backend, self.country, img_code=img_code, model_img=self._model_assign.get(spu.get("code", ""))) if r: r["img_code"] = img_code return r def _upload_product(self, r: Dict[str, Any]) -> None: if not self._oss_enabled: return from graph.oss_upload import build_oss_key, compress_for_oss, upload_to_oss from graph.nodes.oss_upload_node import KIND_ORDER, _gen_rand4 # 货号直接取产品 img_code：同一货号的所有图片（合成/平铺/底图）共用同一货号， # 避免独立计数器在有产品被跳过时与 img_code 错位 base_code = r.get("img_code") or r.get("oss_code") or "" if not base_code: return for kind in KIND_ORDER: src = r.get(f"{kind}_path") if not src or not Path(src).exists(): continue try: compressed = compress_for_oss(src, str(Path(src).with_suffix(".oss.jpg"))) key = build_oss_key(self.country, self.task_timestamp, base_code, _gen_rand4()) url = upload_to_oss(self._oss_cfg, compressed, key) if url: r[f"{kind}_url"] = url r["oss_code"] = base_code except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] OSS 上传失败 {src}: {e}") # 多色：首色用主图 url/code，额外色单独上传（同一货号） comps = r.get("color_composites") or [] if comps and r.get("composite_url"): comps[0]["url"] = r["composite_url"] comps[0]["code"] = r.get("oss_code", "") for cc in comps[1:]: src = cc.get("composite_path") if not src or not Path(src).exists(): continue try: compressed = compress_for_oss(src, str(Path(src).with_suffix(".oss.jpg"))) key = build_oss_key(self.country, self.task_timestamp, base_code, _gen_rand4()) url = upload_to_oss(self._oss_cfg, compressed, key) if url: cc["url"] = url cc["code"] = base_code except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] OSS 颜色图上传失败 {src}: {e}") def _seed_shot(self, r: Dict[str, Any]) -> None: ss_cfg = self.config.get("seed_shot") or {} count = int(ss_cfg.get("count", 1)) if count <= 0 or not bool(ss_cfg.get("enabled", True)) or self._ib is None: return from graph.nodes.seed_shot_node import _plan_seed_shots from graph.seed_shot import generate_seed_shots from graph.oss_upload import build_oss_key, compress_for_oss, upload_to_oss from graph.nodes.oss_upload_node import MAX_CODE, _gen_rand4 comps = r.get("color_composites") or [] if not comps and r.get("composite_path") and Path(r["composite_path"]).exists(): comps = [{"sku_code": r.get("sku_code"), "color": r.get("color", ""), "composite_path": r["composite_path"]}] if not comps: return plan = _plan_seed_shots(comps, count) cn = (r.get("cn_title") or "").strip() or r.get("topic", "") material = self._material_map.get(r.get("spu_code", ""), "") base_prefix = r.get("img_code") or r.get("oss_code") or "" pfx = base_prefix or "seed" shot_dir = self.output_dir / "seed_shots" shot_dir.mkdir(parents=True, exist_ok=True) size = str(ss_cfg.get("size") or "1536x2048") paths: List[str] = [] for cc, n in plan: base = cc.get("composite_path") if not base or not Path(base).exists(): print(f"[pinterest_pipeline] {r.get('spu_code', '')} 参考图缺失，跳过该色种草图") continue generated = generate_seed_shots(self._ib, base, cn, material, n, str(shot_dir), r.get("composite_negative", ""), size=size, prefix=pfx, gender=self._gender) paths.extend(generated) if not paths: return r["seed_shot_paths"] = paths urls: List[str] = [] for pth in paths: with self._oss_lock: if not base_prefix: if self.oss_seq >= MAX_CODE: break code = f"{self._prefix}{self.oss_seq:03d}" self.oss_seq += 1 else: code = base_prefix if self._oss_enabled: try: compressed = compress_for_oss(pth, str(Path(pth).with_suffix(".oss.jpg"))) url = upload_to_oss(self._oss_cfg, compressed, build_oss_key(self.country, self.task_timestamp, code, _gen_rand4())) if url: urls.append(url) r["seed_shot_urls"] = urls except Exception as e: # noqa: BLE001 print(f"[pinterest_pipeline] 种草图上传失败 {pth}: {e}")