3218485270
f493bde8a9
- 缓存热点批量流程（有采集缓存不触发 Google） - 简报不足直接从采集缓存生成（轻量补齐） - 三图合成（模特/印花/底图）+ 底图压缩 <2MB - 热点去重→风格去重自动切换 + 不适合类目 review 兜底 - 透明背景（background=transparent）+ 提示词清洗（敏感词/背景描述） - 任务前 basemap 校验 + 模板国家校验 + 模特任务级分配
254 lines
11 KiB
Python
254 lines
11 KiB
Python
"""归一化、跨源融合、合规黑名单过滤、人名过滤（从原 src/scoring.py 迁移到 graph 包）。
|
||
|
||
所有函数纯逻辑、无 IO，便于节点内调用与单元测试。
|
||
"""
|
||
import re
|
||
from collections import defaultdict
|
||
from typing import Dict, List, Optional, Tuple
|
||
|
||
|
||
DEFAULT_NAME_PATTERNS = [r"^[A-Z][a-z]+(?: [A-Z][a-z]+){1,2}$"]
|
||
DEFAULT_EXTRA_NAMES = [
|
||
"taylor swift", "trump", "biden", "kardashian", "lebron", "charlie sheen",
|
||
"bernie sanders", "elon musk", "beyonce", "drake", "rihanna", "justin bieber",
|
||
"ariana grande", "selena gomez", "eminem", "kanye", "travis scott", "messi",
|
||
"ronaldo", "harry styles", "bts", "blackpink", "pewdiepie", "mrbeast",
|
||
"kamala harris", "joe biden", "donald trump", "kim kardashian", "pearl jam",
|
||
"nirvana", "michael jackson", "madonna", "britney spears", "lady gaga",
|
||
"justin timberlake", "tom cruise", "brad pitt", "keanu reeves", "robert downey",
|
||
"cristiano ronaldo", "lionel messi", "billie eilish", "the weeknd", "post malone",
|
||
"kendrick lamar", "joe rogan", "andrew tate", "elon", "musk", "obama", "clinton",
|
||
"springsteen", "reiner", "eliza lopes", "camilla", "noah kahan", "gina carano",
|
||
# —— 常见人名扩充（歌手/演员/运动员/政客/企业家/网红/王室，子串匹配，避免真人印花）——
|
||
"ed sheeran", "dua lipa", "adele", "bruno mars", "shakira", "elton john",
|
||
"david bowie", "freddie mercury", "whitney houston", "celine dion", "olivia rodrigo",
|
||
"sabrina carpenter", "chappell roan", "ice spice", "nicki minaj", "cardi b",
|
||
"doja cat", "sza", "lil nas x", "bad bunny", "shawn mendes", "zayn malik",
|
||
"dwayne johnson", "johnny depp", "leonardo dicaprio", "chris hemsworth", "chris evans",
|
||
"tom holland", "zendaya", "jennifer lawrence", "emma watson", "scarlett johansson",
|
||
"miley cyrus", "hugh jackman", "nicole kidman", "cate blanchett", "steve irwin",
|
||
"kylie minogue", "morgan freeman", "will smith", "denzel washington", "angelina jolie",
|
||
"jennifer aniston", "george clooney", "robert pattinson", "daniel radcliffe", "emma stone",
|
||
"ryan reynolds", "ryan gosling", "kobe bryant", "michael jordan", "serena williams",
|
||
"venus williams", "tiger woods", "usain bolt", "tom brady", "patrick mahomes",
|
||
"stephen curry", "kevin durant", "lewis hamilton", "max verstappen", "novak djokovic",
|
||
"rafael nadal", "roger federer", "conor mcgregor", "putin", "zelensky",
|
||
"boris johnson", "rishi sunak", "narendra modi", "justin trudeau", "emmanuel macron",
|
||
"olaf scholz", "bill gates", "jeff bezos", "mark zuckerberg", "logan paul", "jake paul",
|
||
"ksi", "charli damelio", "addison rae", "kylie jenner", "kendall jenner",
|
||
"queen elizabeth", "king charles", "prince william", "prince harry", "meghan markle",
|
||
"princess diana",
|
||
]
|
||
DEFAULT_EXEMPTIONS = [
|
||
"new album", "best seller", "top gear", "red cross", "black cat", "blue moon",
|
||
"green day", "red hot chili peppers", "cold play", "one direction", "little mix",
|
||
"west life", "back street", "new york", "los angeles", "san diego", "new orleans",
|
||
"san francisco", "las vegas", "white house", "high school", "middle earth",
|
||
]
|
||
|
||
|
||
def filter_person_names(
|
||
rows: List[Dict],
|
||
extra_names: Optional[List[str]] = None,
|
||
patterns: Optional[List[str]] = None,
|
||
exemptions: Optional[List[str]] = None,
|
||
pattern_sources: Optional[set] = None,
|
||
) -> Tuple[List[Dict], List[Dict]]:
|
||
"""剔除真实人物（明星/政客/名人），避免肖像权风险。返回 (kept, dropped)。"""
|
||
extra = [e.lower() for e in (extra_names if extra_names is not None else DEFAULT_EXTRA_NAMES)]
|
||
pats = patterns if patterns is not None else DEFAULT_NAME_PATTERNS
|
||
exempt = [e.lower() for e in (exemptions if exemptions is not None else DEFAULT_EXEMPTIONS)]
|
||
compiled = [re.compile(p) for p in pats]
|
||
pattern_sources = set(pattern_sources) if pattern_sources is not None else {"gt_trending"}
|
||
|
||
kept, dropped = [], []
|
||
for r in rows:
|
||
topic = str(r.get("topic", "")).strip()
|
||
tl = topic.lower()
|
||
if any(e and e in tl for e in exempt):
|
||
kept.append(r)
|
||
continue
|
||
reason = None
|
||
hit_name = [n for n in extra if n and n in tl]
|
||
if hit_name:
|
||
reason = f"命中人名名单: {hit_name}"
|
||
elif r.get("source") in pattern_sources and any(p.search(topic) for p in compiled):
|
||
reason = "匹配人名模式(疑似真实人物)"
|
||
if reason:
|
||
r2 = dict(r)
|
||
r2["drop_reason"] = reason
|
||
dropped.append(r2)
|
||
else:
|
||
kept.append(r)
|
||
return kept, dropped
|
||
|
||
|
||
def filter_design_relevance(
|
||
rows: List[Dict],
|
||
drop_patterns: Optional[List[str]] = None,
|
||
keep_patterns: Optional[List[str]] = None,
|
||
) -> Tuple[List[Dict], List[Dict]]:
|
||
"""丢弃不可作印花主体的泛新闻/科技/赛事词。返回 (kept, dropped)。"""
|
||
drop = [re.compile(p, re.I) for p in (drop_patterns or [])]
|
||
keep = [re.compile(p, re.I) for p in (keep_patterns or [])]
|
||
kept, dropped = [], []
|
||
for r in rows:
|
||
topic = str(r.get("topic", "")).strip()
|
||
tl = topic.lower()
|
||
if keep and not any(p.search(tl) for p in keep):
|
||
r2 = dict(r)
|
||
r2["drop_reason"] = "未命中设计相关性白名单"
|
||
dropped.append(r2)
|
||
continue
|
||
if drop and any(p.search(tl) for p in drop):
|
||
r2 = dict(r)
|
||
r2["drop_reason"] = "非印花设计主体(泛新闻/科技/赛事)"
|
||
dropped.append(r2)
|
||
continue
|
||
kept.append(r)
|
||
return kept, dropped
|
||
|
||
|
||
# 查询噪声：非“可印花设计概念”的检索问句 / 命名清单 / 损坏碎片，应直接丢弃而非标 safe。
|
||
_QUERY_NOISE_LEAD = re.compile(
|
||
r"^(what|who|how|why|when|where|which|is|are|was|were|do|does|did|can|will|"
|
||
r"should|would|may|might|has|have|whose|whom)\b", re.I)
|
||
# 任意位置的疑问词：覆盖 "punk sprite what does it do" 这类词序在中的问句
|
||
_QUERY_NOISE_WH_ANY = re.compile(r"\b(what|who|how|why|when|where|which)\b", re.I)
|
||
# names/surnames：覆盖 "cottagecore surnames" 这类变体
|
||
_QUERY_NOISE_NAMES = re.compile(
|
||
r"\b((?:boy|girl|baby|pet|dog|cat|last|first|middle)?\s*names?|surnames)"
|
||
r"(?:\s+(?:ideas|list))?\b$", re.I)
|
||
# 损坏/拼接碎片：数字前缀可选，覆盖 "gothic remake review"（无数字）与 "gothic remake metacritic"
|
||
_QUERY_NOISE_CORRUPT = re.compile(
|
||
r"\b(?:\d{1,2}\s+)?(remake|review|version|copy|edit|replica|metacritic)\b", re.I)
|
||
_QUERY_NOISE_WORDS = [re.compile(p, re.I) for p in
|
||
[r"\bstory\b", r"\bmeaning\b", r"\bdefinition\b",
|
||
r"\btutorial\b", r"\bguide\b", r"\bquests?\b"]]
|
||
|
||
|
||
# —— 新闻类热点过滤（突发新闻不适合做印花主题，各国语言词表）——
|
||
NEWS_WORDS_GLOBAL = [
|
||
"weather", "forecast", "typhoon", "earthquake", "tsunami", "hurricane",
|
||
"missile", "election", "vote", "prime minister", "president", "minister",
|
||
"cabinet", "senate", "congress", "parliament", "shooting", "ceasefire",
|
||
"nuclear", "summit", "hostage", "emergency", "warning", "breaking news",
|
||
"stock market", "oil price", "inflation", "deadline", "live update",
|
||
]
|
||
NEWS_WORDS_JP = [
|
||
"天気", "台風", "気象", "地震", "津波", "ミサイル", "首相", "大臣",
|
||
"会見", "速報", "選挙", "防衛", "自衛隊", "警報", "注意報", "ニュース",
|
||
"報道", "豪雨", "猛暑", "熱中症", "株価", "円相場", "物価", "国会",
|
||
"衆院", "参院", "裁判", "逮捕", "捜査", "事故", "死亡", "追悼", "慰霊",
|
||
]
|
||
|
||
|
||
def filter_news(rows: List[Dict], country: str = "") -> Tuple[List[Dict], List[Dict]]:
|
||
"""丢弃新闻类热点（天气/灾害/政治/事故等突发新闻，非印花主题）。按国家语言补充词表。"""
|
||
words = list(NEWS_WORDS_GLOBAL)
|
||
if str(country).upper() == "JP":
|
||
words += NEWS_WORDS_JP
|
||
elif str(country).upper() == "US":
|
||
words += ["weather alert", "live coverage", "breaking"]
|
||
kept, dropped = [], []
|
||
for r in rows:
|
||
tl = str(r.get("topic", "")).lower()
|
||
hit = next((w for w in words if w in tl), None)
|
||
if hit:
|
||
r2 = dict(r)
|
||
r2["drop_reason"] = f"新闻类热点(非印花主题): {hit}"
|
||
dropped.append(r2)
|
||
else:
|
||
kept.append(r)
|
||
return kept, dropped
|
||
|
||
|
||
def filter_query_noise(
|
||
rows: List[Dict],
|
||
enabled: bool = True,
|
||
) -> Tuple[List[Dict], List[Dict]]:
|
||
"""丢弃“查询噪声/非设计概念”词（问句、命名清单、损坏碎片、模糊名词）。
|
||
|
||
返回 (kept, dropped)。这些词不是可印花主体，进入 screen 会被 Mock 误标 safe，
|
||
故在过滤阶段就剔除，避免污染生图环节。
|
||
"""
|
||
if not enabled:
|
||
return rows, []
|
||
kept, dropped = [], []
|
||
for r in rows:
|
||
topic = str(r.get("topic", "")).strip()
|
||
tl = topic.lower()
|
||
reason = None
|
||
if _QUERY_NOISE_WH_ANY.search(tl) or _QUERY_NOISE_LEAD.search(tl):
|
||
reason = "查询问句(非设计概念)"
|
||
elif _QUERY_NOISE_NAMES.search(tl):
|
||
reason = "命名清单类查询(非设计概念)"
|
||
elif _QUERY_NOISE_CORRUPT.search(tl):
|
||
reason = "损坏/拼接的查询碎片"
|
||
elif any(p.search(tl) for p in _QUERY_NOISE_WORDS):
|
||
reason = "模糊名词(非设计概念)"
|
||
if reason:
|
||
r2 = dict(r)
|
||
r2["drop_reason"] = reason
|
||
dropped.append(r2)
|
||
else:
|
||
kept.append(r)
|
||
return kept, dropped
|
||
|
||
|
||
def normalize(rows: List[Dict], key: str = "raw_score") -> List[Dict]:
|
||
"""min-max 归一化到 0-1，按 (source, kind) 分组分别归一化。"""
|
||
if not rows:
|
||
return rows
|
||
groups = defaultdict(list)
|
||
for r in rows:
|
||
groups[(r.get("source", "_"), r.get("kind", "_"))].append(r)
|
||
for grp in groups.values():
|
||
vals = [r[key] for r in grp if r.get(key) is not None]
|
||
if not vals:
|
||
for r in grp:
|
||
r["norm"] = 0.0
|
||
continue
|
||
lo, hi = min(vals), max(vals)
|
||
span = (hi - lo) or 1.0
|
||
for r in grp:
|
||
v = r.get(key)
|
||
r["norm"] = (v - lo) / span if v is not None else 0.0
|
||
return rows
|
||
|
||
|
||
def apply_blacklist(rows: List[Dict], blacklist: List[str]) -> Tuple[List[Dict], List[Dict]]:
|
||
"""命中黑名单的词丢弃，返回 (保留, 丢弃)。"""
|
||
if not blacklist:
|
||
return rows, []
|
||
bl = [b.lower() for b in blacklist]
|
||
kept, dropped = [], []
|
||
for r in rows:
|
||
text = f"{r.get('topic', '')} {r.get('seed', '')}".lower()
|
||
if any(b in text for b in bl):
|
||
dropped.append(r)
|
||
else:
|
||
kept.append(r)
|
||
return kept, dropped
|
||
|
||
|
||
def combine(rows: List[Dict], weights: Dict[str, float]) -> List[Dict]:
|
||
"""按 topic 跨源融合，权重来自 config。"""
|
||
agg = {}
|
||
for r in rows:
|
||
t = r["topic"].lower().strip()
|
||
if t not in agg:
|
||
agg[t] = {"topic": r["topic"], "countries": set(), "sources": set(), "score": 0.0}
|
||
w = weights.get(r["source"], 0.5)
|
||
agg[t]["score"] += r.get("norm", 0.0) * w
|
||
if r.get("country"):
|
||
agg[t]["countries"].add(r["country"])
|
||
agg[t]["sources"].add(r["source"])
|
||
out = []
|
||
for o in agg.values():
|
||
o["countries"] = ",".join(sorted(o["countries"])) or "GLOBAL"
|
||
o["sources"] = ",".join(sorted(o["sources"]))
|
||
out.append(o)
|
||
out.sort(key=lambda x: x["score"], reverse=True)
|
||
return out
|