"""归一化、跨源融合、合规黑名单过滤、人名过滤(从原 src/scoring.py 迁移到 graph 包)。 所有函数纯逻辑、无 IO,便于节点内调用与单元测试。 """ import re from collections import defaultdict from typing import Dict, List, Optional, Tuple DEFAULT_NAME_PATTERNS = [r"^[A-Z][a-z]+(?: [A-Z][a-z]+){1,2}$"] DEFAULT_EXTRA_NAMES = [ "taylor swift", "trump", "biden", "kardashian", "lebron", "charlie sheen", "bernie sanders", "elon musk", "beyonce", "drake", "rihanna", "justin bieber", "ariana grande", "selena gomez", "eminem", "kanye", "travis scott", "messi", "ronaldo", "harry styles", "bts", "blackpink", "pewdiepie", "mrbeast", "kamala harris", "joe biden", "donald trump", "kim kardashian", "pearl jam", "nirvana", "michael jackson", "madonna", "britney spears", "lady gaga", "justin timberlake", "tom cruise", "brad pitt", "keanu reeves", "robert downey", "cristiano ronaldo", "lionel messi", "billie eilish", "the weeknd", "post malone", "kendrick lamar", "joe rogan", "andrew tate", "elon", "musk", "obama", "clinton", "springsteen", "reiner", "eliza lopes", "camilla", "noah kahan", "gina carano", # —— 常见人名扩充(歌手/演员/运动员/政客/企业家/网红/王室,子串匹配,避免真人印花)—— "ed sheeran", "dua lipa", "adele", "bruno mars", "shakira", "elton john", "david bowie", "freddie mercury", "whitney houston", "celine dion", "olivia rodrigo", "sabrina carpenter", "chappell roan", "ice spice", "nicki minaj", "cardi b", "doja cat", "sza", "lil nas x", "bad bunny", "shawn mendes", "zayn malik", "dwayne johnson", "johnny depp", "leonardo dicaprio", "chris hemsworth", "chris evans", "tom holland", "zendaya", "jennifer lawrence", "emma watson", "scarlett johansson", "miley cyrus", "hugh jackman", "nicole kidman", "cate blanchett", "steve irwin", "kylie minogue", "morgan freeman", "will smith", "denzel washington", "angelina jolie", "jennifer aniston", "george clooney", "robert pattinson", "daniel radcliffe", "emma stone", "ryan reynolds", "ryan gosling", "kobe bryant", "michael jordan", "serena williams", "venus williams", "tiger woods", "usain bolt", "tom brady", "patrick mahomes", "stephen curry", "kevin durant", "lewis hamilton", "max verstappen", "novak djokovic", "rafael nadal", "roger federer", "conor mcgregor", "putin", "zelensky", "boris johnson", "rishi sunak", "narendra modi", "justin trudeau", "emmanuel macron", "olaf scholz", "bill gates", "jeff bezos", "mark zuckerberg", "logan paul", "jake paul", "ksi", "charli damelio", "addison rae", "kylie jenner", "kendall jenner", "queen elizabeth", "king charles", "prince william", "prince harry", "meghan markle", "princess diana", ] DEFAULT_EXEMPTIONS = [ "new album", "best seller", "top gear", "red cross", "black cat", "blue moon", "green day", "red hot chili peppers", "cold play", "one direction", "little mix", "west life", "back street", "new york", "los angeles", "san diego", "new orleans", "san francisco", "las vegas", "white house", "high school", "middle earth", ] def filter_person_names( rows: List[Dict], extra_names: Optional[List[str]] = None, patterns: Optional[List[str]] = None, exemptions: Optional[List[str]] = None, pattern_sources: Optional[set] = None, ) -> Tuple[List[Dict], List[Dict]]: """剔除真实人物(明星/政客/名人),避免肖像权风险。返回 (kept, dropped)。""" extra = [e.lower() for e in (extra_names if extra_names is not None else DEFAULT_EXTRA_NAMES)] pats = patterns if patterns is not None else DEFAULT_NAME_PATTERNS exempt = [e.lower() for e in (exemptions if exemptions is not None else DEFAULT_EXEMPTIONS)] compiled = [re.compile(p) for p in pats] pattern_sources = set(pattern_sources) if pattern_sources is not None else {"gt_trending"} kept, dropped = [], [] for r in rows: topic = str(r.get("topic", "")).strip() tl = topic.lower() if any(e and e in tl for e in exempt): kept.append(r) continue reason = None hit_name = [n for n in extra if n and n in tl] if hit_name: reason = f"命中人名名单: {hit_name}" elif r.get("source") in pattern_sources and any(p.search(topic) for p in compiled): reason = "匹配人名模式(疑似真实人物)" if reason: r2 = dict(r) r2["drop_reason"] = reason dropped.append(r2) else: kept.append(r) return kept, dropped def filter_design_relevance( rows: List[Dict], drop_patterns: Optional[List[str]] = None, keep_patterns: Optional[List[str]] = None, ) -> Tuple[List[Dict], List[Dict]]: """丢弃不可作印花主体的泛新闻/科技/赛事词。返回 (kept, dropped)。""" drop = [re.compile(p, re.I) for p in (drop_patterns or [])] keep = [re.compile(p, re.I) for p in (keep_patterns or [])] kept, dropped = [], [] for r in rows: topic = str(r.get("topic", "")).strip() tl = topic.lower() if keep and not any(p.search(tl) for p in keep): r2 = dict(r) r2["drop_reason"] = "未命中设计相关性白名单" dropped.append(r2) continue if drop and any(p.search(tl) for p in drop): r2 = dict(r) r2["drop_reason"] = "非印花设计主体(泛新闻/科技/赛事)" dropped.append(r2) continue kept.append(r) return kept, dropped # 查询噪声:非“可印花设计概念”的检索问句 / 命名清单 / 损坏碎片,应直接丢弃而非标 safe。 _QUERY_NOISE_LEAD = re.compile( r"^(what|who|how|why|when|where|which|is|are|was|were|do|does|did|can|will|" r"should|would|may|might|has|have|whose|whom)\b", re.I) # 任意位置的疑问词:覆盖 "punk sprite what does it do" 这类词序在中的问句 _QUERY_NOISE_WH_ANY = re.compile(r"\b(what|who|how|why|when|where|which)\b", re.I) # names/surnames:覆盖 "cottagecore surnames" 这类变体 _QUERY_NOISE_NAMES = re.compile( r"\b((?:boy|girl|baby|pet|dog|cat|last|first|middle)?\s*names?|surnames)" r"(?:\s+(?:ideas|list))?\b$", re.I) # 损坏/拼接碎片:数字前缀可选,覆盖 "gothic remake review"(无数字)与 "gothic remake metacritic" _QUERY_NOISE_CORRUPT = re.compile( r"\b(?:\d{1,2}\s+)?(remake|review|version|copy|edit|replica|metacritic)\b", re.I) _QUERY_NOISE_WORDS = [re.compile(p, re.I) for p in [r"\bstory\b", r"\bmeaning\b", r"\bdefinition\b", r"\btutorial\b", r"\bguide\b", r"\bquests?\b"]] # —— 新闻类热点过滤(突发新闻不适合做印花主题,各国语言词表)—— NEWS_WORDS_GLOBAL = [ "weather", "forecast", "typhoon", "earthquake", "tsunami", "hurricane", "missile", "election", "vote", "prime minister", "president", "minister", "cabinet", "senate", "congress", "parliament", "shooting", "ceasefire", "nuclear", "summit", "hostage", "emergency", "warning", "breaking news", "stock market", "oil price", "inflation", "deadline", "live update", ] NEWS_WORDS_JP = [ "天気", "台風", "気象", "地震", "津波", "ミサイル", "首相", "大臣", "会見", "速報", "選挙", "防衛", "自衛隊", "警報", "注意報", "ニュース", "報道", "豪雨", "猛暑", "熱中症", "株価", "円相場", "物価", "国会", "衆院", "参院", "裁判", "逮捕", "捜査", "事故", "死亡", "追悼", "慰霊", ] def filter_news(rows: List[Dict], country: str = "") -> Tuple[List[Dict], List[Dict]]: """丢弃新闻类热点(天气/灾害/政治/事故等突发新闻,非印花主题)。按国家语言补充词表。""" words = list(NEWS_WORDS_GLOBAL) if str(country).upper() == "JP": words += NEWS_WORDS_JP elif str(country).upper() == "US": words += ["weather alert", "live coverage", "breaking"] kept, dropped = [], [] for r in rows: tl = str(r.get("topic", "")).lower() hit = next((w for w in words if w in tl), None) if hit: r2 = dict(r) r2["drop_reason"] = f"新闻类热点(非印花主题): {hit}" dropped.append(r2) else: kept.append(r) return kept, dropped def filter_query_noise( rows: List[Dict], enabled: bool = True, ) -> Tuple[List[Dict], List[Dict]]: """丢弃“查询噪声/非设计概念”词(问句、命名清单、损坏碎片、模糊名词)。 返回 (kept, dropped)。这些词不是可印花主体,进入 screen 会被 Mock 误标 safe, 故在过滤阶段就剔除,避免污染生图环节。 """ if not enabled: return rows, [] kept, dropped = [], [] for r in rows: topic = str(r.get("topic", "")).strip() tl = topic.lower() reason = None if _QUERY_NOISE_WH_ANY.search(tl) or _QUERY_NOISE_LEAD.search(tl): reason = "查询问句(非设计概念)" elif _QUERY_NOISE_NAMES.search(tl): reason = "命名清单类查询(非设计概念)" elif _QUERY_NOISE_CORRUPT.search(tl): reason = "损坏/拼接的查询碎片" elif any(p.search(tl) for p in _QUERY_NOISE_WORDS): reason = "模糊名词(非设计概念)" if reason: r2 = dict(r) r2["drop_reason"] = reason dropped.append(r2) else: kept.append(r) return kept, dropped def normalize(rows: List[Dict], key: str = "raw_score") -> List[Dict]: """min-max 归一化到 0-1,按 (source, kind) 分组分别归一化。""" if not rows: return rows groups = defaultdict(list) for r in rows: groups[(r.get("source", "_"), r.get("kind", "_"))].append(r) for grp in groups.values(): vals = [r[key] for r in grp if r.get(key) is not None] if not vals: for r in grp: r["norm"] = 0.0 continue lo, hi = min(vals), max(vals) span = (hi - lo) or 1.0 for r in grp: v = r.get(key) r["norm"] = (v - lo) / span if v is not None else 0.0 return rows def apply_blacklist(rows: List[Dict], blacklist: List[str]) -> Tuple[List[Dict], List[Dict]]: """命中黑名单的词丢弃,返回 (保留, 丢弃)。""" if not blacklist: return rows, [] bl = [b.lower() for b in blacklist] kept, dropped = [], [] for r in rows: text = f"{r.get('topic', '')} {r.get('seed', '')}".lower() if any(b in text for b in bl): dropped.append(r) else: kept.append(r) return kept, dropped def combine(rows: List[Dict], weights: Dict[str, float]) -> List[Dict]: """按 topic 跨源融合,权重来自 config。""" agg = {} for r in rows: t = r["topic"].lower().strip() if t not in agg: agg[t] = {"topic": r["topic"], "countries": set(), "sources": set(), "score": 0.0} w = weights.get(r["source"], 0.5) agg[t]["score"] += r.get("norm", 0.0) * w if r.get("country"): agg[t]["countries"].add(r["country"]) agg[t]["sources"].add(r["source"]) out = [] for o in agg.values(): o["countries"] = ",".join(sorted(o["countries"])) or "GLOBAL" o["sources"] = ",".join(sorted(o["sources"])) out.append(o) out.sort(key=lambda x: x["score"], reverse=True) return out