POD 趋势感知 Agent：缓存热点模式 + 三图合成 + 热点去重/风格去重 + review 兜底
- 缓存热点批量流程（有采集缓存不触发 Google） - 简报不足直接从采集缓存生成（轻量补齐） - 三图合成（模特/印花/底图）+ 底图压缩 <2MB - 热点去重→风格去重自动切换 + 不适合类目 review 兜底 - 透明背景（background=transparent）+ 提示词清洗（敏感词/背景描述） - 任务前 basemap 校验 + 模板国家校验 + 模特任务级分配
This commit is contained in:
@@ -0,0 +1,253 @@
|
||||
"""归一化、跨源融合、合规黑名单过滤、人名过滤（从原 src/scoring.py 迁移到 graph 包）。
|
||||
|
||||
所有函数纯逻辑、无 IO，便于节点内调用与单元测试。
|
||||
"""
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
DEFAULT_NAME_PATTERNS = [r"^[A-Z][a-z]+(?: [A-Z][a-z]+){1,2}$"]
|
||||
DEFAULT_EXTRA_NAMES = [
|
||||
"taylor swift", "trump", "biden", "kardashian", "lebron", "charlie sheen",
|
||||
"bernie sanders", "elon musk", "beyonce", "drake", "rihanna", "justin bieber",
|
||||
"ariana grande", "selena gomez", "eminem", "kanye", "travis scott", "messi",
|
||||
"ronaldo", "harry styles", "bts", "blackpink", "pewdiepie", "mrbeast",
|
||||
"kamala harris", "joe biden", "donald trump", "kim kardashian", "pearl jam",
|
||||
"nirvana", "michael jackson", "madonna", "britney spears", "lady gaga",
|
||||
"justin timberlake", "tom cruise", "brad pitt", "keanu reeves", "robert downey",
|
||||
"cristiano ronaldo", "lionel messi", "billie eilish", "the weeknd", "post malone",
|
||||
"kendrick lamar", "joe rogan", "andrew tate", "elon", "musk", "obama", "clinton",
|
||||
"springsteen", "reiner", "eliza lopes", "camilla", "noah kahan", "gina carano",
|
||||
# —— 常见人名扩充（歌手/演员/运动员/政客/企业家/网红/王室，子串匹配，避免真人印花）——
|
||||
"ed sheeran", "dua lipa", "adele", "bruno mars", "shakira", "elton john",
|
||||
"david bowie", "freddie mercury", "whitney houston", "celine dion", "olivia rodrigo",
|
||||
"sabrina carpenter", "chappell roan", "ice spice", "nicki minaj", "cardi b",
|
||||
"doja cat", "sza", "lil nas x", "bad bunny", "shawn mendes", "zayn malik",
|
||||
"dwayne johnson", "johnny depp", "leonardo dicaprio", "chris hemsworth", "chris evans",
|
||||
"tom holland", "zendaya", "jennifer lawrence", "emma watson", "scarlett johansson",
|
||||
"miley cyrus", "hugh jackman", "nicole kidman", "cate blanchett", "steve irwin",
|
||||
"kylie minogue", "morgan freeman", "will smith", "denzel washington", "angelina jolie",
|
||||
"jennifer aniston", "george clooney", "robert pattinson", "daniel radcliffe", "emma stone",
|
||||
"ryan reynolds", "ryan gosling", "kobe bryant", "michael jordan", "serena williams",
|
||||
"venus williams", "tiger woods", "usain bolt", "tom brady", "patrick mahomes",
|
||||
"stephen curry", "kevin durant", "lewis hamilton", "max verstappen", "novak djokovic",
|
||||
"rafael nadal", "roger federer", "conor mcgregor", "putin", "zelensky",
|
||||
"boris johnson", "rishi sunak", "narendra modi", "justin trudeau", "emmanuel macron",
|
||||
"olaf scholz", "bill gates", "jeff bezos", "mark zuckerberg", "logan paul", "jake paul",
|
||||
"ksi", "charli damelio", "addison rae", "kylie jenner", "kendall jenner",
|
||||
"queen elizabeth", "king charles", "prince william", "prince harry", "meghan markle",
|
||||
"princess diana",
|
||||
]
|
||||
DEFAULT_EXEMPTIONS = [
|
||||
"new album", "best seller", "top gear", "red cross", "black cat", "blue moon",
|
||||
"green day", "red hot chili peppers", "cold play", "one direction", "little mix",
|
||||
"west life", "back street", "new york", "los angeles", "san diego", "new orleans",
|
||||
"san francisco", "las vegas", "white house", "high school", "middle earth",
|
||||
]
|
||||
|
||||
|
||||
def filter_person_names(
|
||||
rows: List[Dict],
|
||||
extra_names: Optional[List[str]] = None,
|
||||
patterns: Optional[List[str]] = None,
|
||||
exemptions: Optional[List[str]] = None,
|
||||
pattern_sources: Optional[set] = None,
|
||||
) -> Tuple[List[Dict], List[Dict]]:
|
||||
"""剔除真实人物（明星/政客/名人），避免肖像权风险。返回 (kept, dropped)。"""
|
||||
extra = [e.lower() for e in (extra_names if extra_names is not None else DEFAULT_EXTRA_NAMES)]
|
||||
pats = patterns if patterns is not None else DEFAULT_NAME_PATTERNS
|
||||
exempt = [e.lower() for e in (exemptions if exemptions is not None else DEFAULT_EXEMPTIONS)]
|
||||
compiled = [re.compile(p) for p in pats]
|
||||
pattern_sources = set(pattern_sources) if pattern_sources is not None else {"gt_trending"}
|
||||
|
||||
kept, dropped = [], []
|
||||
for r in rows:
|
||||
topic = str(r.get("topic", "")).strip()
|
||||
tl = topic.lower()
|
||||
if any(e and e in tl for e in exempt):
|
||||
kept.append(r)
|
||||
continue
|
||||
reason = None
|
||||
hit_name = [n for n in extra if n and n in tl]
|
||||
if hit_name:
|
||||
reason = f"命中人名名单: {hit_name}"
|
||||
elif r.get("source") in pattern_sources and any(p.search(topic) for p in compiled):
|
||||
reason = "匹配人名模式(疑似真实人物)"
|
||||
if reason:
|
||||
r2 = dict(r)
|
||||
r2["drop_reason"] = reason
|
||||
dropped.append(r2)
|
||||
else:
|
||||
kept.append(r)
|
||||
return kept, dropped
|
||||
|
||||
|
||||
def filter_design_relevance(
|
||||
rows: List[Dict],
|
||||
drop_patterns: Optional[List[str]] = None,
|
||||
keep_patterns: Optional[List[str]] = None,
|
||||
) -> Tuple[List[Dict], List[Dict]]:
|
||||
"""丢弃不可作印花主体的泛新闻/科技/赛事词。返回 (kept, dropped)。"""
|
||||
drop = [re.compile(p, re.I) for p in (drop_patterns or [])]
|
||||
keep = [re.compile(p, re.I) for p in (keep_patterns or [])]
|
||||
kept, dropped = [], []
|
||||
for r in rows:
|
||||
topic = str(r.get("topic", "")).strip()
|
||||
tl = topic.lower()
|
||||
if keep and not any(p.search(tl) for p in keep):
|
||||
r2 = dict(r)
|
||||
r2["drop_reason"] = "未命中设计相关性白名单"
|
||||
dropped.append(r2)
|
||||
continue
|
||||
if drop and any(p.search(tl) for p in drop):
|
||||
r2 = dict(r)
|
||||
r2["drop_reason"] = "非印花设计主体(泛新闻/科技/赛事)"
|
||||
dropped.append(r2)
|
||||
continue
|
||||
kept.append(r)
|
||||
return kept, dropped
|
||||
|
||||
|
||||
# 查询噪声：非“可印花设计概念”的检索问句 / 命名清单 / 损坏碎片，应直接丢弃而非标 safe。
|
||||
_QUERY_NOISE_LEAD = re.compile(
|
||||
r"^(what|who|how|why|when|where|which|is|are|was|were|do|does|did|can|will|"
|
||||
r"should|would|may|might|has|have|whose|whom)\b", re.I)
|
||||
# 任意位置的疑问词：覆盖 "punk sprite what does it do" 这类词序在中的问句
|
||||
_QUERY_NOISE_WH_ANY = re.compile(r"\b(what|who|how|why|when|where|which)\b", re.I)
|
||||
# names/surnames：覆盖 "cottagecore surnames" 这类变体
|
||||
_QUERY_NOISE_NAMES = re.compile(
|
||||
r"\b((?:boy|girl|baby|pet|dog|cat|last|first|middle)?\s*names?|surnames)"
|
||||
r"(?:\s+(?:ideas|list))?\b$", re.I)
|
||||
# 损坏/拼接碎片：数字前缀可选，覆盖 "gothic remake review"（无数字）与 "gothic remake metacritic"
|
||||
_QUERY_NOISE_CORRUPT = re.compile(
|
||||
r"\b(?:\d{1,2}\s+)?(remake|review|version|copy|edit|replica|metacritic)\b", re.I)
|
||||
_QUERY_NOISE_WORDS = [re.compile(p, re.I) for p in
|
||||
[r"\bstory\b", r"\bmeaning\b", r"\bdefinition\b",
|
||||
r"\btutorial\b", r"\bguide\b", r"\bquests?\b"]]
|
||||
|
||||
|
||||
# —— 新闻类热点过滤（突发新闻不适合做印花主题，各国语言词表）——
|
||||
NEWS_WORDS_GLOBAL = [
|
||||
"weather", "forecast", "typhoon", "earthquake", "tsunami", "hurricane",
|
||||
"missile", "election", "vote", "prime minister", "president", "minister",
|
||||
"cabinet", "senate", "congress", "parliament", "shooting", "ceasefire",
|
||||
"nuclear", "summit", "hostage", "emergency", "warning", "breaking news",
|
||||
"stock market", "oil price", "inflation", "deadline", "live update",
|
||||
]
|
||||
NEWS_WORDS_JP = [
|
||||
"天気", "台風", "気象", "地震", "津波", "ミサイル", "首相", "大臣",
|
||||
"会見", "速報", "選挙", "防衛", "自衛隊", "警報", "注意報", "ニュース",
|
||||
"報道", "豪雨", "猛暑", "熱中症", "株価", "円相場", "物価", "国会",
|
||||
"衆院", "参院", "裁判", "逮捕", "捜査", "事故", "死亡", "追悼", "慰霊",
|
||||
]
|
||||
|
||||
|
||||
def filter_news(rows: List[Dict], country: str = "") -> Tuple[List[Dict], List[Dict]]:
|
||||
"""丢弃新闻类热点（天气/灾害/政治/事故等突发新闻，非印花主题）。按国家语言补充词表。"""
|
||||
words = list(NEWS_WORDS_GLOBAL)
|
||||
if str(country).upper() == "JP":
|
||||
words += NEWS_WORDS_JP
|
||||
elif str(country).upper() == "US":
|
||||
words += ["weather alert", "live coverage", "breaking"]
|
||||
kept, dropped = [], []
|
||||
for r in rows:
|
||||
tl = str(r.get("topic", "")).lower()
|
||||
hit = next((w for w in words if w in tl), None)
|
||||
if hit:
|
||||
r2 = dict(r)
|
||||
r2["drop_reason"] = f"新闻类热点(非印花主题): {hit}"
|
||||
dropped.append(r2)
|
||||
else:
|
||||
kept.append(r)
|
||||
return kept, dropped
|
||||
|
||||
|
||||
def filter_query_noise(
|
||||
rows: List[Dict],
|
||||
enabled: bool = True,
|
||||
) -> Tuple[List[Dict], List[Dict]]:
|
||||
"""丢弃“查询噪声/非设计概念”词（问句、命名清单、损坏碎片、模糊名词）。
|
||||
|
||||
返回 (kept, dropped)。这些词不是可印花主体，进入 screen 会被 Mock 误标 safe，
|
||||
故在过滤阶段就剔除，避免污染生图环节。
|
||||
"""
|
||||
if not enabled:
|
||||
return rows, []
|
||||
kept, dropped = [], []
|
||||
for r in rows:
|
||||
topic = str(r.get("topic", "")).strip()
|
||||
tl = topic.lower()
|
||||
reason = None
|
||||
if _QUERY_NOISE_WH_ANY.search(tl) or _QUERY_NOISE_LEAD.search(tl):
|
||||
reason = "查询问句(非设计概念)"
|
||||
elif _QUERY_NOISE_NAMES.search(tl):
|
||||
reason = "命名清单类查询(非设计概念)"
|
||||
elif _QUERY_NOISE_CORRUPT.search(tl):
|
||||
reason = "损坏/拼接的查询碎片"
|
||||
elif any(p.search(tl) for p in _QUERY_NOISE_WORDS):
|
||||
reason = "模糊名词(非设计概念)"
|
||||
if reason:
|
||||
r2 = dict(r)
|
||||
r2["drop_reason"] = reason
|
||||
dropped.append(r2)
|
||||
else:
|
||||
kept.append(r)
|
||||
return kept, dropped
|
||||
|
||||
|
||||
def normalize(rows: List[Dict], key: str = "raw_score") -> List[Dict]:
|
||||
"""min-max 归一化到 0-1，按 (source, kind) 分组分别归一化。"""
|
||||
if not rows:
|
||||
return rows
|
||||
groups = defaultdict(list)
|
||||
for r in rows:
|
||||
groups[(r.get("source", "_"), r.get("kind", "_"))].append(r)
|
||||
for grp in groups.values():
|
||||
vals = [r[key] for r in grp if r.get(key) is not None]
|
||||
if not vals:
|
||||
for r in grp:
|
||||
r["norm"] = 0.0
|
||||
continue
|
||||
lo, hi = min(vals), max(vals)
|
||||
span = (hi - lo) or 1.0
|
||||
for r in grp:
|
||||
v = r.get(key)
|
||||
r["norm"] = (v - lo) / span if v is not None else 0.0
|
||||
return rows
|
||||
|
||||
|
||||
def apply_blacklist(rows: List[Dict], blacklist: List[str]) -> Tuple[List[Dict], List[Dict]]:
|
||||
"""命中黑名单的词丢弃，返回 (保留, 丢弃)。"""
|
||||
if not blacklist:
|
||||
return rows, []
|
||||
bl = [b.lower() for b in blacklist]
|
||||
kept, dropped = [], []
|
||||
for r in rows:
|
||||
text = f"{r.get('topic', '')} {r.get('seed', '')}".lower()
|
||||
if any(b in text for b in bl):
|
||||
dropped.append(r)
|
||||
else:
|
||||
kept.append(r)
|
||||
return kept, dropped
|
||||
|
||||
|
||||
def combine(rows: List[Dict], weights: Dict[str, float]) -> List[Dict]:
|
||||
"""按 topic 跨源融合，权重来自 config。"""
|
||||
agg = {}
|
||||
for r in rows:
|
||||
t = r["topic"].lower().strip()
|
||||
if t not in agg:
|
||||
agg[t] = {"topic": r["topic"], "countries": set(), "sources": set(), "score": 0.0}
|
||||
w = weights.get(r["source"], 0.5)
|
||||
agg[t]["score"] += r.get("norm", 0.0) * w
|
||||
if r.get("country"):
|
||||
agg[t]["countries"].add(r["country"])
|
||||
agg[t]["sources"].add(r["source"])
|
||||
out = []
|
||||
for o in agg.values():
|
||||
o["countries"] = ",".join(sorted(o["countries"])) or "GLOBAL"
|
||||
o["sources"] = ",".join(sorted(o["sources"]))
|
||||
out.append(o)
|
||||
out.sort(key=lambda x: x["score"], reverse=True)
|
||||
return out
|
||||
Reference in New Issue
Block a user