inkreachhub/pod_trend_agent
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

v88 功能增强：产品落盘持久化 + 生图网关适配 + 模板导出优化

Browse Source 
- 产品持久化：每完成一个产品立即追加写入 products_pending.jsonl，崩溃不丢已完成产品，finish 读盘合并后统一写模板
- 503 致命错误提前终止：compose/product/seed_shot 端到端识别，提前终止搜索分析，丢弃未完成简报，保留已完成落盘产品直接合成模板
- 模特分配：material_library 合格模特图按任务序号独立随机，同 SPU 多款不再共用同一模特
- 图像网关适配：execution_mode/background 默认不再传入 yunfei 等标准网关，base_url 需带 /v1；429/5xx/空响应退避重试
- Pinterest 分析：删除 term 注入与纯文本降级，失败直接放弃；图片上传前 PIL 完整性校验；suitable_for_print=False 过滤丢弃
- 模板导出：不再产生空白 xlsx，文件名=模板原文件名_已填写；写入前按货号末 3 位升序排序
- 删除对接文档.md，更新 README，gitignore 排除测试产物
This commit is contained in:
3218485270
2026-08-28 10:28:35 +08:00
parent 685b7b0862
commit 2a96ec0870
28 changed files with 1187 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files
graph/pinterest_pipeline.py
+190 -22
View File
@@ -12,6 +12,7 @@
并发上限与 product_node 一致（默认 5），避免压垮图像网关。
"""
import concurrent.futures
import random
import re
import threading
import time
@@ -35,13 +36,23 @@ class PinterestPipeline:
self._briefs: List[Dict[str, Any]] = []
self._done = False
self._cursor = 0
# 正在处理（已提交线程池、尚未完成）的简报数，供路由判断简报池是否空闲
self._in_flight = 0
# 结果
# 结果：每完成一个产品立即落盘追加写入 products_pending.jsonl，
# 内存列表仅作缓存（finish 时再读盘合并），中途崩溃也不丢已完成产品。
self._products: List[Dict[str, Any]] = []
self._products_lock = threading.Lock()
self._errors: List[Dict[str, Any]] = []
self._errors_lock = threading.Lock()
# 致命图像服务错误（53/账户不可用）：置位后终止分发、丢弃未完成简报，仅保留已完成产品
self._fatal_lock = threading.Lock()
self._fatal_503 = False
# 已完成产品落盘文件（JSONL 追加写）：output/<country>/<ts>/products_pending.jsonl
self._pending_file = self.output_dir / "products_pending.jsonl"
# 路径解析（复用 product_node 的 _abs 逻辑：运行根优先，其次数据根）
pcfg = self.config.get("product") or {}
@@ -188,19 +199,28 @@ class PinterestPipeline:
return None
def _assign_models(self) -> Dict[str, Any]:
"""任务级模特分配：按 spu.mark 映射模特目录 → 过滤非 3:4 图片 → 每个任务随机抽取。
每个产品任务（含同一 SPU 的多个款）都独立随机抽一个模特，保证同款多产品模特不重复。
"""
model_assign: Dict[str, Any] = {}
try:
from graph.product import find_first_model_folder
_folder, _all_models = find_first_model_folder(self._material_root, self._category)
from graph.product import build_mark_model_map, find_model_images_for_mark
mark_map = build_mark_model_map(self._db_path, self._material_root)
except Exception: # noqa: BLE001
_all_models = []
if _all_models:
seen: Dict[str, str] = {}
for _i, (spu, _skus) in enumerate(self._worklist):
code = spu.get("code", "")
if code not in seen:
seen[code] = _all_models[_i % len(_all_models)]
model_assign[code] = seen[code]
mark_map = {}
# 每个任务按序号绑定独立随机模特（同 SPU 多款也各自随机，不共用）
for _i, (spu, _skus) in enumerate(self._worklist):
key = f"task_{_i}"
mark = str(spu.get("mark") or "").strip() or "1"
folder = mark_map.get(mark, self._category)
try:
pool_imgs = find_model_images_for_mark(self._db_path, self._material_root,
mark, folder)
except Exception: # noqa: BLE001
pool_imgs = []
if pool_imgs:
model_assign[key] = random.choice(pool_imgs) # 过滤后随机抽
return model_assign
def _load_materials(self) -> Dict[str, str]:
@@ -237,6 +257,48 @@ class PinterestPipeline:
with self._err400_lock:
return self._err400_aborted
# ------------------------------------------------------------------ #
# 致命图像服务错误（503 / 账户不可用）：重试无效，提前终止整个任务
# ------------------------------------------------------------------ #
@staticmethod
def is_fatal_503(exc) -> bool:
"""判断异常是否为「图像服务不可用」类致命错误（503 / No available compatible accounts）。
这类错误说明账户配额耗尽或网关故障，重试必然失败，应提前终止任务而非无意义重试。
"""
msg = str(exc)
if "503" in msg:
return True
low = msg.lower()
return "no available compatible accounts" in low or "account" in low and "not available" in low
def record_503(self) -> bool:
"""记录一次致命 503：首次触发即置位终止标志（后续请求直接短路不再提交）。
返回 True 表示本次触发终止（调用方应立即停止当前链路）。
"""
with self._fatal_lock:
first = not self._fatal_503
self._fatal_503 = True
if first:
print("[pinterest_pipeline] ⛔ 检测到图像服务 503（No available compatible accounts），"
"重试无效 → 提前终止任务，未完成产品将废弃，仅保留已完成产品")
return first
def is_fatal_503_aborted(self) -> bool:
with self._fatal_lock:
return self._fatal_503
def abort_unfinished(self) -> None:
"""终止分发：丢弃简报池中所有未完成简报（已完成的落盘产品保留）。"""
with self._cond:
dropped = len(self._briefs)
self._briefs = []
self._done = True
self._cond.notify_all()
if dropped:
print(f"[pinterest_pipeline] 503 终止：丢弃未完成简报 {dropped} 条（未完成产品废弃）")
def _abort_current_term(self) -> None:
"""放弃当前种子词：清空其未完成简报 + 图池未消费图片（已完成的保留）。"""
term = self._err400_term
@@ -298,18 +360,52 @@ class PinterestPipeline:
self._cond.notify_all()
print(f"[pinterest_pipeline] 简报池 +{len(briefs)} 条（待处理 {len(self._briefs)}）")
def pending_count(self) -> int:
"""简报池中待处理 + 正在处理的简报数（供路由判断是否需要补图/补分析）。"""
with self._cond:
queued = len(self._briefs)
return queued + self._in_flight
def wait_idle(self, timeout: Optional[float] = None) -> bool:
"""阻塞等待简报池消化完（无待处理且无在途），返回是否已空闲。
用 Condition 等待（_process_one 完成时 notify_all 唤醒），而非轮询 sleep，
避免 wait 循环疯狂刷屏。timeout 为 None 时无限等待（受 _done 保护）。
"""
with self._cond:
while (self._briefs or self._in_flight > 0) and not self._done:
if timeout is not None:
deadline = time.time() + timeout
remaining = deadline - time.time()
if remaining <= 0:
return False
self._cond.wait(min(remaining, 1.0))
else:
self._cond.wait()
return not self._briefs and self._in_flight <= 0
def finish(self) -> tuple:
"""排空简报池、等待全部产品完成，返回 (products, errors)。"""
"""排空简报池、等待全部产品完成，返回 (products, errors)。
产品来源：手动已完成（内存缓存）+ 落盘文件（products_pending.jsonl）
按货号去重合并——即使中途 503 终止/崩溃，已完成产品也不丢。
"""
with self._cond:
self._done = True
self._cond.notify_all()
self._dispatcher.join()
self._pool.shutdown(wait=True)
# 读盘 + 内存合并去重（内存为准，但以落盘为最终权威——崩溃恢复后走落盘）
pending = self.load_pending()
merged = {str(p.get("img_code", "")): p for p in pending}
with self._products_lock:
products = list(self._products)
for p in self._products:
merged[str(p.get("img_code", ""))] = p
products = [merged[k] for k in merged if k]
with self._errors_lock:
errors = list(self._errors)
print(f"[pinterest_pipeline] 收尾：完成 {len(products)} 个产品，错误 {len(errors)}")
print(f"[pinterest_pipeline] 收尾：完成 {len(products)} 个产品，错误 {len(errors)}"
f"{'（含落盘恢复 ' + str(len(pending)) + '）' if pending else ''}")
return products, errors
# ------------------------------------------------------------------ #
@@ -322,8 +418,16 @@ class PinterestPipeline:
self._cond.wait()
if self._done and not self._briefs:
break
if self._fatal_503:
# 致命 503：不再分发新简报（未完成的废弃，仅保留已完成落盘产品）
self._briefs = []
self._done = True
self._cond.notify_all()
break
batch = self._briefs
self._briefs = []
# 与 _briefs 清空同一临界区递增在途数，避免 wait_idle 误判空闲
self._in_flight += len(batch)
for b in batch:
with self._cond:
idx = self._cursor
@@ -335,6 +439,8 @@ class PinterestPipeline:
# ------------------------------------------------------------------ #
def _process_one(self, brief: Dict[str, Any], idx: int) -> None:
try:
if self.is_fatal_503_aborted():
return
# 0) 先定货号：整条链路（设计/三合一/种草图）都用它命名与匹配，避免序号错位
if idx >= len(self._worklist):
print(f"[pinterest_pipeline] 简报 {idx} 无对应产品任务，跳过")
@@ -343,6 +449,8 @@ class PinterestPipeline:
img_code = f"{self._prefix}{idx:03d}"
# 1) 生成设计（compose）——直接按货号命名 designs/{img_code}_design.png
design_path = self._gen_design(brief, img_code)
if self.is_fatal_503_aborted():
return
if self.is_400_aborted():
# 当前种子词 400 超限已放弃：正在生成的当个也放弃，不进入后续链路
print(f"[pinterest_pipeline] 当前种子词 400 超限已放弃，跳过简报 {idx}")
@@ -357,13 +465,17 @@ class PinterestPipeline:
return
brief = new_brief
design_path = self._gen_design(brief, img_code)
if self.is_fatal_503_aborted():
return
if design_path:
break
if not design_path:
return
brief["design_path"] = design_path
# 2) 三合一（product）——同一货号
prod = self._process_spu(brief, spu, skus, img_code, design_path)
# 2) 三合一（product）——同一货号；task_idx=本次简报序号，模特按任务独立随机
prod = self._process_spu(brief, spu, skus, img_code, design_path, task_idx=idx)
if self.is_fatal_503_aborted():
return
if not prod:
return
# 3) OSS 上传
@@ -376,6 +488,8 @@ class PinterestPipeline:
_record_used(self.cache_dir, prod)
except Exception: # noqa: BLE001
pass
# 5) 落盘：每完成一个产品立即追加写入 products_pending.jsonl（不依赖内存，崩溃不丢）
self._persist_product(prod)
with self._products_lock:
self._products.append(prod)
print(f"[pinterest_pipeline] 产品完成: {prod.get('img_code', '')}"
@@ -385,6 +499,42 @@ class PinterestPipeline:
self._errors.append({"node": "pinterest_pipeline", "type": type(e).__name__,
"message": f"简报 {idx} 处理失败: {e}", "trace": ""})
print(f"[pinterest_pipeline] 简报 {idx} 处理失败: {e}")
finally:
with self._cond:
self._in_flight = max(0, self._in_flight - 1)
self._cond.notify_all()
def _persist_product(self, prod: Dict[str, Any]) -> None:
"""把已完成产品追加写入 products_pending.jsonl（JSONL 每行一个产品）。
落盘失败不阻塞主流程（仅告警）；finish() 时读盘合并，保证已完成产品不丢。
"""
try:
import json as _json
self._pending_file.parent.mkdir(parents=True, exist_ok=True)
with open(self._pending_file, "a", encoding="utf-8") as f:
f.write(_json.dumps(prod, ensure_ascii=False) + "\n")
except Exception as e: # noqa: BLE001
print(f"[pinterest_pipeline] 产品落盘失败（不影响流程）: {e}")
def load_pending(self) -> List[Dict[str, Any]]:
"""读回 products_pending.jsonl 中已落盘的产品（进程重启/崩溃恢复用）。"""
import json as _json
out: List[Dict[str, Any]] = []
if not self._pending_file.exists():
return out
try:
for line in self._pending_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
out.append(_json.loads(line))
except Exception: # noqa: BLE001
continue
except Exception as e: # noqa: BLE001
print(f"[pinterest_pipeline] 读回落盘产品失败: {e}")
return out
def _gen_design(self, brief: Dict[str, Any], img_code: str) -> Optional[str]:
if self._ib is None:
@@ -398,10 +548,18 @@ class PinterestPipeline:
if self.record_400():
self._abort_current_term()
def _on_503():
self.record_503()
self.abort_unfinished()
return generate_design(self._ib, brief, design_dir, img_code, self._errors,
on_400=_on_400,
on_400=_on_400, on_503=_on_503,
size=str((self.config.get("compose") or {}).get("design_size") or "1024x1024"))
except Exception as e: # noqa: BLE001
if self.is_fatal_503(e):
self.record_503()
self.abort_unfinished()
return None
with self._errors_lock:
self._errors.append({"node": "compose", "type": type(e).__name__,
"message": f"设计稿生成失败 {brief.get('topic', '')}: {e}",
@@ -473,7 +631,7 @@ class PinterestPipeline:
return None
def _process_spu(self, brief: Dict[str, Any], spu, skus: str, img_code: str,
design_path: str) -> Optional[Dict[str, Any]]:
design_path: str, task_idx: int = 0) -> Optional[Dict[str, Any]]:
from graph.nodes.product_node import _process_spu as _ps
prod_dir = self.output_dir / "product"
prod_dir.mkdir(parents=True, exist_ok=True)
@@ -483,7 +641,9 @@ class PinterestPipeline:
r = _ps(self._db_path, self._basemap_root, self._material_root, self._category,
prod_dir, brief, self._ib, spu, skus, self.config.get("product") or {},
self._errors, design_path, self._title_backend, self.country,
img_code=img_code, model_img=self._model_assign.get(spu.get("code", "")))
img_code=img_code,
model_img=self._model_assign.get(f"task_{task_idx}"),
on_503=lambda: (self.record_503(), self.abort_unfinished()))
if r:
r["img_code"] = img_code
return r
@@ -559,9 +719,17 @@ class PinterestPipeline:
if not base or not Path(base).exists():
print(f"[pinterest_pipeline] {r.get('spu_code', '')} 参考图缺失，跳过该色种草图")
continue
generated = generate_seed_shots(self._ib, base, cn, material, n, str(shot_dir),
r.get("composite_negative", ""),
size=size, prefix=pfx, gender=self._gender)
try:
generated = generate_seed_shots(self._ib, base, cn, material, n, str(shot_dir),
r.get("composite_negative", ""),
size=size, prefix=pfx, gender=self._gender)
except Exception as e: # noqa: BLE001
if self.is_fatal_503(e):
self.record_503()
self.abort_unfinished()
return
print(f"[pinterest_pipeline] 种草图生成失败（跳过该色）: {e}")
continue
paths.extend(generated)
if not paths:
return