3218485270
f493bde8a9
- 缓存热点批量流程（有采集缓存不触发 Google） - 简报不足直接从采集缓存生成（轻量补齐） - 三图合成（模特/印花/底图）+ 底图压缩 <2MB - 热点去重→风格去重自动切换 + 不适合类目 review 兜底 - 透明背景（background=transparent）+ 提示词清洗（敏感词/背景描述） - 任务前 basemap 校验 + 模板国家校验 + 模特任务级分配
310 lines
16 KiB
Python
310 lines
16 KiB
Python
"""缓存热点批量产品流程（用户新流程入口）。
|
||
|
||
流程：
|
||
1. 选定国家 → 直接加载最新缓存热点（output/<国家>/design_briefs.json，不重跑种子/抓取）；
|
||
无缓存时回退跑一次完整流水线（run_country）生成缓存。
|
||
2. 按 SPU 数量（count）取 N 个「未用过」热点（safe 按分降序，跳过 used_designs.json 里已用的），
|
||
一个款号分配一个热点（spu_tasks 每项绑定 topic）。
|
||
3. 调 product_node：每款号用自己热点的 image_prompt 生成设计稿 → 三图合成 → 多模态标题 → 模板导出；
|
||
product_node 内部成功后把 (热点-风格-配色) 写入 used_designs.json 去重。
|
||
"""
|
||
import json
|
||
import re
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from graph.paths import project_root, runtime_root
|
||
|
||
# 全项目 review 兜底：不适合 T 恤印花的类目关键词（美甲/食谱/彩票/赛果/天气/比分/日程等）
|
||
_UNSUITABLE = re.compile(
|
||
r"\b(nails?|manicure|pedicure|recipes?|cooking|lottery|jackpot|results?|score|scores?|"
|
||
r"fixtures?|forecast|weather|temperature|map|directions?|parking|opening hours?|"
|
||
r"prices?|price|reviews?|jobs?|salary|mortgage|council tax|election|referendum|"
|
||
r"stock market|exchange rate|gas prices?)\b",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
def load_cached_briefs(output_dir: Path) -> List[Dict[str, Any]]:
|
||
"""读缓存简报（design_briefs.json，含 image_prompt/composite_prompt 四要素）。"""
|
||
p = output_dir / "design_briefs.json"
|
||
if not p.exists():
|
||
return []
|
||
try:
|
||
data = json.loads(p.read_text(encoding="utf-8"))
|
||
return [b for b in (data.get("design_briefs") or [])
|
||
if b.get("motif") and b.get("image_prompt")]
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
def load_used(output_dir: Path) -> List[Dict[str, Any]]:
|
||
p = output_dir / "used_designs.json"
|
||
if not p.exists():
|
||
return []
|
||
try:
|
||
return json.loads(p.read_text(encoding="utf-8")).get("used", []) or []
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
def fingerprint(b: Dict[str, Any]) -> str:
|
||
"""(热点-风格) 指纹，用于去重（不按配色，配色不影响主题唯一性）。"""
|
||
return "|".join(str(b.get(k, "")).strip().lower() for k in ("topic", "art_style"))
|
||
|
||
|
||
def assign_hotspots(briefs: List[Dict[str, Any]], used: List[Dict[str, Any]],
|
||
count: int, allow_review: bool = False,
|
||
exclude_topics: Optional[List[str]] = None) -> List[Dict[str, Any]]:
|
||
"""三级热点分配（严格 → 放宽 → 兜底）：
|
||
|
||
① 热点去重：只用「没用过」的热点（topic 不在 used_designs）
|
||
② 风格去重：热点池不够时放宽——同一热点允许换风格（(topic, art_style) 组合未用过）
|
||
③ 规则匹配：还不够时兜底——全部简报按分数/规则取（允许完全重复，mock 风格）
|
||
|
||
allow_review=True（openai 模式，LLM 已安全改写）：review 且带 concept 的也进候选池。
|
||
exclude_topics：不分配黑名单（如地名/美甲/版权剧名/真实人物等，来自国家配置 exclude_topics）。
|
||
每级内部按 score 降序 + 高分池随机（不总取第一个）。"""
|
||
import random
|
||
used_topics = {str(u.get("topic", "")).strip().lower() for u in used}
|
||
used_fp = {fingerprint(u) for u in used}
|
||
safe = sorted([b for b in briefs if b.get("risk_level") == "safe"],
|
||
key=lambda b: -(b.get("score") or 0))
|
||
pool = list(safe)
|
||
if allow_review:
|
||
reviewed = [b for b in briefs if b.get("risk_level") == "review" and (b.get("concept") or "").strip()]
|
||
pool += sorted(reviewed, key=lambda b: -(b.get("score") or 0))
|
||
# 黑名单过滤：不分配的热点（国家配置 exclude_topics，大小写不敏感）
|
||
if exclude_topics:
|
||
ban = {str(x).strip().lower() for x in exclude_topics if str(x).strip()}
|
||
_before = len(pool)
|
||
pool = [b for b in pool if str(b.get("topic", "")).strip().lower() not in ban]
|
||
if len(pool) < _before:
|
||
print(f"[batch] 黑名单过滤 {_before - len(pool)} 个不分配热点（exclude_topics）")
|
||
# 全项目 review 兜底：剔除不适合 T 恤印花的类目（美甲/食谱/彩票/赛果/天气等通用识别）
|
||
_before2 = len(pool)
|
||
pool = [b for b in pool if not _UNSUITABLE.search(str(b.get("topic", "")))]
|
||
if len(pool) < _before2:
|
||
print(f"[batch] review 兜底剔除 {_before2 - len(pool)} 个不适合类目热点（美甲/食谱/彩票/赛果/天气等）")
|
||
|
||
def _shuffle_top(stage: List[Dict[str, Any]], need: int) -> List[Dict[str, Any]]:
|
||
k = max(need * 2, 4)
|
||
top, rest = stage[:k], stage[k:]
|
||
random.shuffle(top)
|
||
return top + rest
|
||
|
||
# ① 热点去重（topic 未用过）
|
||
stage1 = [b for b in pool if str(b.get("topic", "")).strip().lower() not in used_topics]
|
||
# ② 风格去重（topic 用过，但 热点-风格 指纹未用过）
|
||
stage2 = [b for b in pool if str(b.get("topic", "")).strip().lower() in used_topics
|
||
and fingerprint(b) not in used_fp]
|
||
# ③ 规则匹配兜底（剩余未用指纹，允许低分热点；已用指纹一律不重复出）
|
||
stage3 = [b for b in pool if fingerprint(b) not in used_fp]
|
||
|
||
out: List[Dict[str, Any]] = []
|
||
out_fp: set = set() # 本批内 (topic, style) 指纹去重
|
||
for si, stage in enumerate((stage1, stage2, stage3)):
|
||
for b in _shuffle_top(stage, count - len(out)):
|
||
if len(out) >= count:
|
||
break
|
||
# 去重策略（用户指定）：
|
||
# ① 先热点去重：本批内优先不同 topic（stage1 全未用热点）；
|
||
# ② 热点用完后自动切风格去重：同一热点换新风格（topic 可重复，style 不同，
|
||
# 即 热点1-风格1 → 热点1-风格2 → 热点2-风格2）；
|
||
# fingerprint（topic+art_style）本批内绝不重复，保证同热点同风格只出一次。
|
||
fp = fingerprint(b)
|
||
if fp in out_fp:
|
||
continue
|
||
# 热点去重优先：本批已用过的 topic 只在「没有未用热点可挑」时放行（stage2/3）
|
||
topic_used = any(str(x.get("topic", "")).strip().lower()
|
||
== str(b.get("topic", "")).strip().lower() for x in out)
|
||
if topic_used and si == 0:
|
||
continue
|
||
out_fp.add(fp)
|
||
out.append(b)
|
||
if len(out) >= count:
|
||
break
|
||
return out[:count]
|
||
|
||
|
||
def _rebuild_briefs_from_cache(country: str, config: Dict[str, Any], project_root: Path,
|
||
cache_dir: Path, need: int) -> List[Dict[str, Any]]:
|
||
"""简报不足时：直接用采集缓存热点（collected_keywords）生成简报。
|
||
跳过 seed/fetch/score（无需重新采集、无需种子词），screen（排除已用）+ prompt 组装即可。"""
|
||
import json as _json
|
||
import time as _tm
|
||
cp = cache_dir / "collected_keywords.json"
|
||
if not cp.exists():
|
||
return []
|
||
ck = _json.loads(cp.read_text(encoding="utf-8")).get("keywords") or []
|
||
if not ck:
|
||
return []
|
||
from graph.loader import build_country_config
|
||
from graph.nodes.screen_node import screen_node
|
||
from graph.nodes.prompt_node import prompt_node
|
||
cc = build_country_config(config, country, project_root)
|
||
config.setdefault("product", {})["spu_count"] = need
|
||
state: Dict[str, Any] = {
|
||
"country": country, "config": config, "country_config": cc,
|
||
"prompts_dir": str(project_root / "prompts" / country),
|
||
"cache_dir": str(cache_dir), "output_dir": str(cache_dir),
|
||
"scored_rows": [dict(r) for r in ck],
|
||
"screened": [], "briefs": [], "errors": [], "stats": {},
|
||
}
|
||
try:
|
||
r1 = screen_node(state)
|
||
r2 = prompt_node({**state, "screened": r1.get("screened", [])})
|
||
briefs = r2.get("briefs", []) or []
|
||
if briefs:
|
||
(cache_dir / "design_briefs.json").write_text(
|
||
_json.dumps({"generated_at": _tm.strftime("%Y-%m-%dT%H:%M:%S"),
|
||
"total": len(briefs), "design_briefs": briefs},
|
||
ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return briefs
|
||
except Exception as e: # noqa: BLE001
|
||
print(f"[batch] 采集缓存生成简报失败: {e}")
|
||
return []
|
||
|
||
|
||
def run_product_batch(country: str, config: Dict[str, Any], project_root: Path,
|
||
output_root: Optional[Path], tasks: List[Dict[str, Any]],
|
||
count: int, log_q=None, task_timestamp: Optional[str] = None) -> Dict[str, Any]:
|
||
"""缓存模式入口：加载缓存热点 → 按量分配 → product_node 批量处理。
|
||
|
||
task_timestamp: 任务开始时间戳（YYYYMMDDHHMMSS），作为 OSS 路径段；缺失取当前时间。
|
||
"""
|
||
if log_q:
|
||
log_q.put(("log", f"\n===== 缓存热点产品流程 {country}（SPU 数量 {count}）=====\n"))
|
||
import time as _tm
|
||
cache_dir = (output_root or project_root) / "output" / country # 缓存/去重（根目录）
|
||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||
ts = task_timestamp or _tm.strftime("%Y%m%d_%H%M%S")
|
||
_base = ts
|
||
_i = 1
|
||
while (cache_dir / ts).exists(): # 时间戳文件夹唯一（同秒多任务防冲突/覆盖）
|
||
ts = f"{_base}_{_i}"
|
||
_i += 1
|
||
output_dir = cache_dir / ts # 本次任务产物（时间戳文件夹）
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
briefs = load_cached_briefs(cache_dir)
|
||
# 任务扩展：每个「集合」（款-颜色集 + 独立数量 count）按自己数量复制 count 份；
|
||
# 每份 skus 保留该款全部颜色集合（多色）→ 每个设计配全部颜色各出一张主图
|
||
picked_tasks: List[Dict[str, Any]] = []
|
||
if tasks:
|
||
for t in tasks:
|
||
n = int(t.get("count") or 0) or count or 1
|
||
for _ in range(n):
|
||
tt = dict(t)
|
||
tt.pop("count", None)
|
||
picked_tasks.append(tt)
|
||
else:
|
||
picked_tasks = [dict(t) for t in (tasks or [])]
|
||
if not briefs or len(briefs) < len(picked_tasks):
|
||
# 简报不足：直接用采集缓存热点生成简报（无需重新采集/种子词）
|
||
print(f"[batch] 简报 {len(briefs)} 条 < 需要 {len(picked_tasks)} 条 → 直接用采集缓存热点生成简报（无需重新采集）…")
|
||
briefs = _rebuild_briefs_from_cache(country, config, project_root, cache_dir, len(picked_tasks))
|
||
if not briefs:
|
||
print("[batch] 采集缓存无有效热点，无法生成简报")
|
||
return {"product": [], "briefs": [], "errors": [{"node": "batch", "message": "无缓存热点"}]}
|
||
|
||
used = load_used(cache_dir)
|
||
used_topics = {str(u.get("topic", "")).strip().lower() for u in used}
|
||
# 简报池未用数（design_briefs.json 旧简报里还没用过的）
|
||
fresh_count = sum(1 for b in briefs
|
||
if str(b.get("topic", "")).strip().lower() not in used_topics)
|
||
total_needed = len(picked_tasks) or count
|
||
# 采集池未用数（collected_keywords 全量里的未用热点——佐证热点池是否真的充足）
|
||
pool_fresh = fresh_count
|
||
try:
|
||
import json as _json
|
||
cp = cache_dir / "collected_keywords.json"
|
||
if cp.exists():
|
||
ck = _json.loads(cp.read_text(encoding="utf-8")).get("keywords") or []
|
||
pool_fresh = sum(1 for k in ck
|
||
if str(k.get("topic", "")).strip().lower() not in used_topics)
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
# 旧简报用完了（简报池未用 < 需要）→ 直接用采集缓存热点生成新简报（无需重新采集）
|
||
if fresh_count < total_needed:
|
||
print(f"[batch] 简报池未用 {fresh_count}/{len(briefs)} 条 < 需要 {total_needed} 条 → "
|
||
f"直接用采集缓存热点生成新简报（采集池未用 {pool_fresh} 条充足，无需重新采集）…")
|
||
briefs = _rebuild_briefs_from_cache(country, config, project_root, cache_dir, total_needed)
|
||
|
||
# openai 模式：LLM 已安全改写，review（带 concept）也可用；mock 模式：review 留人工复核
|
||
allow_review = str((config.get("llm_screen") or {}).get("provider", "")).strip() != "mock"
|
||
from graph.loader import build_country_config
|
||
_cc = build_country_config(config, country, project_root)
|
||
exclude_topics = list((_cc.get("exclude_topics") or []) or []) # 国家配置的黑名单热点
|
||
assigned = assign_hotspots(briefs, used, total_needed, allow_review=allow_review,
|
||
exclude_topics=exclude_topics)
|
||
if not assigned:
|
||
print("[batch] 无可用热点分配")
|
||
return {"product": [], "briefs": briefs, "errors": [{"node": "batch", "message": "无可用热点"}]}
|
||
if len(assigned) < total_needed:
|
||
print(f"[batch] ⚠ 热点不足：简报 {len(briefs)} 条，仅分配到 {len(assigned)}/{total_needed} 个热点（已用去重后剩余热点少，第 {len(assigned)+1} 个起无热点）")
|
||
else:
|
||
print(f"[batch] 缓存热点 {len(briefs)} 条 → 分配 {len(assigned)} 个热点")
|
||
|
||
# 款号与热点一一绑定：第 i 个款号用第 i 个热点（缓存模式；完整流水线由 product 自行绑定）
|
||
for i, t in enumerate(picked_tasks):
|
||
if i < len(assigned):
|
||
t["topic"] = assigned[i].get("topic", "")
|
||
if log_q:
|
||
for i, t in enumerate(picked_tasks):
|
||
tp = t.get("topic", "")
|
||
log_q.put(("log", f"[batch] 款号 {t.get('spu')} ← 热点「{tp}」\n"))
|
||
|
||
# 直接构造 state 调 product_node（跳过 seed/fetch 等重跑）
|
||
from graph.nodes.product_node import product_node
|
||
from graph.loader import build_country_config
|
||
import time as _time
|
||
cc = build_country_config(config, country, project_root)
|
||
config.setdefault("product", {})["spu_tasks"] = picked_tasks
|
||
if count > 0:
|
||
config.setdefault("product", {})["spu_count"] = count
|
||
state: Dict[str, Any] = {
|
||
"country": country,
|
||
"config": config,
|
||
"country_config": cc,
|
||
"prompts_dir": str(project_root / "prompts" / country),
|
||
"output_dir": str(output_dir),
|
||
"briefs": briefs,
|
||
"designs": [],
|
||
"composite": [],
|
||
"errors": [],
|
||
"stats": {},
|
||
"task_timestamp": task_timestamp or _time.strftime("%Y%m%d%H%M%S"),
|
||
"oss_seq": 0,
|
||
}
|
||
out = product_node(state)
|
||
# 缓存模式也走压缩+上传+种草图节点（与 graph 全流程一致）
|
||
if out.get("product"):
|
||
from graph.nodes.oss_upload_node import oss_upload_node
|
||
out2 = oss_upload_node({**state, "product": out.get("product"),
|
||
"stats": out.get("stats") or {},
|
||
"errors": out.get("errors") or []})
|
||
out["oss"] = out2.get("oss") or []
|
||
out["product"] = out2.get("product") or out.get("product")
|
||
out["stats"] = out2.get("stats") or out.get("stats") or {}
|
||
if out2.get("oss_seq") is not None:
|
||
state["oss_seq"] = out2["oss_seq"]
|
||
|
||
from graph.nodes.seed_shot_node import seed_shot_node
|
||
out3 = seed_shot_node({**state, "product": out.get("product"),
|
||
"stats": out.get("stats") or {},
|
||
"errors": out.get("errors") or []})
|
||
out["seed_shots"] = out3.get("seed_shots") or []
|
||
out["product"] = out3.get("product") or out.get("product")
|
||
out["stats"] = out3.get("stats") or out.get("stats") or {}
|
||
if out3.get("oss_seq") is not None:
|
||
state["oss_seq"] = out3["oss_seq"]
|
||
|
||
from graph.nodes.template_export_node import template_export_node
|
||
out4 = template_export_node({**state, "product": out.get("product"),
|
||
"stats": out.get("stats") or {},
|
||
"errors": out.get("errors") or []})
|
||
out["product"] = out4.get("product") or out.get("product")
|
||
out["stats"] = out4.get("stats") or out.get("stats") or {}
|
||
return out
|