yeuimu
0033b7a4d2
- 新增 logutil：DEBUG 全量滚动日志文件 + 控制台 INFO 简洁输出 + GUI 队列处理器 - providers：每次尝试失败记录 HTTP 状态/响应体，网络异常 logger.exception 留完整堆栈 - pipeline：单任务异常不拖垮整批，缓存命中/阶段耗时/停止过程全程留痕 - CLI/GUI 顶层兜底：未预期异常打印完整 traceback 并指引日志文件；GUI 新增「打开日志」按钮 - 新增 test_logutil（文件落盘/级别过滤/异常堆栈/队列），共 33 个测试全部通过
200 lines
9.1 KiB
Python
200 lines
9.1 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""视觉模型提供方：豆包（火山方舟）与 DeepSeek，均为 OpenAI 兼容 chat/completions。
|
||
|
||
统一的返回结构：
|
||
{"file", "provider", "status"(ok/parse_fail/error), "category", "attr",
|
||
"logic", "raw", "usage": {"prompt", "completion", "cached"}, "finish_reason"}
|
||
"""
|
||
import asyncio
|
||
import base64
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
|
||
import aiohttp
|
||
|
||
from .config import AppConfig
|
||
|
||
logger = logging.getLogger("violation_detector.providers")
|
||
|
||
MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||
".webp": "image/webp", ".gif": "image/gif", ".bmp": "image/bmp",
|
||
".tiff": "image/tiff"}
|
||
IMAGE_EXTS = set(MIME)
|
||
|
||
# 17 类标准名称：按分类号归一化模型输出的写法差异
|
||
CANON = {
|
||
1: "1. 烟草、毒品、赌博、犹太", 2: "2. 酒精", 3: "3. 色情", 4: "4. LGBT",
|
||
5: "5. 负向敏感类信息（血腥暴力）、战争（含动漫里的战争）、武器类、纳粹",
|
||
6: "6. 政治相关（包含军队组织等相关内容）", 7: "7. 种族歧视和文化差异",
|
||
8: "8. 旗帜和徽章", 9: "9. 宗教 - 其他教", 10: "10. 宗教 - 伊斯兰教",
|
||
11: "11. 宗教 - 基督教", 12: "12. 侵权 - 除人物外的其他侵权",
|
||
13: "13. 侵权 - 人物相关（如肖像权、姓名权等）",
|
||
14: "14. 脏话、侮辱性、谩骂攻击类文字图案（包含冒犯和争议性内容）",
|
||
15: "15. 图案消极低俗或者暴力（包含冒犯和争议性内容）",
|
||
16: "16. 欧区国家禁售", 17: "17. 冒犯和争议性内容",
|
||
}
|
||
|
||
|
||
def encode_image(path: str) -> str:
|
||
ext = os.path.splitext(path)[1].lower()
|
||
if ext not in MIME:
|
||
raise ValueError(f"不支持的图片格式: {path}")
|
||
with open(path, "rb") as f:
|
||
b64 = base64.b64encode(f.read()).decode("utf-8")
|
||
return f"data:{MIME[ext]};base64,{b64}"
|
||
|
||
|
||
def norm_category(cat: str) -> str:
|
||
c = str(cat).strip()
|
||
m = re.match(r"^(\d+)", c)
|
||
return CANON.get(int(m.group(1)), c) if m else c
|
||
|
||
|
||
def parse_judgment(raw: str):
|
||
"""解析模型返回的三字段 JSON，返回 (attr, logic, category) 或 None。"""
|
||
if not raw:
|
||
return None
|
||
text = raw.strip()
|
||
m = re.search(r"```(?:json)?\s*(.*?)```", text, re.S)
|
||
if m:
|
||
text = m.group(1).strip()
|
||
s, e = text.find("{"), text.rfind("}")
|
||
if s == -1 or e <= s:
|
||
return None
|
||
try:
|
||
obj = json.loads(text[s:e + 1])
|
||
except json.JSONDecodeError:
|
||
return None
|
||
keys = ["文字/图形属性", "侵权/违规逻辑", "违规分类"]
|
||
if not all(k in obj for k in keys):
|
||
return None
|
||
return (str(obj["文字/图形属性"]).strip(), str(obj["侵权/违规逻辑"]).strip(),
|
||
str(obj["违规分类"]).strip())
|
||
|
||
|
||
def _norm_usage(u: dict) -> dict:
|
||
u = u or {}
|
||
return {"prompt": u.get("prompt_tokens"), "completion": u.get("completion_tokens"),
|
||
"cached": (u.get("prompt_tokens_details") or {}).get("cached_tokens", 0)}
|
||
|
||
|
||
def _result(fname, provider, content, data, path_for_err=""):
|
||
ch = (data.get("choices") or [{}])[0]
|
||
parsed = parse_judgment(content)
|
||
base = {"file": fname, "provider": provider, "finish_reason": ch.get("finish_reason"),
|
||
"usage": _norm_usage(data.get("usage"))}
|
||
if parsed:
|
||
attr, logic, cat = parsed
|
||
base.update({"status": "ok", "attr": attr, "logic": logic,
|
||
"category": norm_category(cat), "raw": content})
|
||
else:
|
||
base.update({"status": "parse_fail", "attr": "", "logic": "",
|
||
"category": "违规不明", "raw": content or ""})
|
||
return base
|
||
|
||
|
||
async def detect_ark(session, sem, cfg: AppConfig, prompt: str, path: str, stop=None):
|
||
"""豆包（火山方舟）视觉检测：图片在前、文本在后，强制 JSON 输出。"""
|
||
fname = os.path.basename(path)
|
||
url = cfg.ark_base_url.rstrip("/") + "/chat/completions"
|
||
headers = {"Authorization": f"Bearer {cfg.ark_api_key}",
|
||
"Content-Type": "application/json",
|
||
"x-is-encrypted": "true"}
|
||
payload = {
|
||
"model": cfg.ark_model,
|
||
"messages": [{
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "image_url", "image_url": {"url": encode_image(path)}},
|
||
{"type": "text", "text": prompt},
|
||
],
|
||
}],
|
||
"response_format": {"type": "json_object"},
|
||
}
|
||
|
||
last_err = ""
|
||
for attempt in range(1, cfg.retries + 1):
|
||
if stop and stop():
|
||
logger.info("豆包检测 %s：用户停止", fname)
|
||
return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
|
||
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
|
||
try:
|
||
async with sem:
|
||
async with session.post(url, json=payload, headers=headers) as r:
|
||
if r.status == 200:
|
||
data = await r.json()
|
||
content = data["choices"][0]["message"]["content"]
|
||
return _result(fname, "doubao", content, data)
|
||
body = (await r.text())[:2000]
|
||
last_err = f"HTTP {r.status}: {body[:300]}"
|
||
logger.warning("豆包检测 %s 第%d/%d次尝试失败：HTTP %s\n响应体：%s",
|
||
fname, attempt, cfg.retries, r.status, body)
|
||
except Exception: # noqa: BLE001 网络层异常：记录完整堆栈后重试
|
||
last_err = "网络/协议异常（详见日志）"
|
||
logger.exception("豆包检测 %s 第%d/%d次尝试抛出异常", fname, attempt, cfg.retries)
|
||
if attempt < cfg.retries:
|
||
logger.info("豆包检测 %s：%.0f 秒后重试", fname, 3 * attempt)
|
||
await asyncio.sleep(3 * attempt)
|
||
logger.error("豆包检测 %s 最终失败：%s", fname, last_err)
|
||
return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
|
||
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
|
||
|
||
|
||
async def detect_deepseek(session, sem, cfg: AppConfig, prompt: str, path: str, stop=None):
|
||
"""DeepSeek 视觉检测：temperature=0 + JSON 模式。
|
||
注意该模型为推理模型，reasoning 消耗 completion 预算，max_tokens 须留足。"""
|
||
fname = os.path.basename(path)
|
||
url = cfg.deepseek_base_url.rstrip("/") + "/chat/completions"
|
||
headers = {"Authorization": f"Bearer {cfg.deepseek_api_key}",
|
||
"Content-Type": "application/json"}
|
||
payload = {
|
||
"model": cfg.deepseek_model,
|
||
"messages": [{
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "text", "text": prompt},
|
||
{"type": "image_url", "image_url": {"url": encode_image(path)}},
|
||
],
|
||
}],
|
||
"temperature": 0,
|
||
"response_format": {"type": "json_object"},
|
||
"max_tokens": cfg.max_tokens,
|
||
}
|
||
|
||
last_err = ""
|
||
for attempt in range(1, cfg.retries + 1):
|
||
if stop and stop():
|
||
logger.info("DeepSeek 检测 %s：用户停止", fname)
|
||
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
|
||
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
|
||
try:
|
||
async with sem:
|
||
async with session.post(url, json=payload, headers=headers) as r:
|
||
if r.status == 200:
|
||
data = await r.json()
|
||
content = data["choices"][0]["message"]["content"]
|
||
if content and content.strip():
|
||
return _result(fname, "deepseek", content, data)
|
||
# 推理烧尽预算会得到空正文，按可重试错误处理
|
||
fr = data["choices"][0].get("finish_reason")
|
||
last_err = f"empty content (finish_reason={fr}, max_tokens={cfg.max_tokens})"
|
||
logger.warning("DeepSeek 检测 %s 第%d/%d次尝试返回空正文"
|
||
"（finish_reason=%s，推理可能耗尽 max_tokens=%s）",
|
||
fname, attempt, cfg.retries, fr, cfg.max_tokens)
|
||
else:
|
||
body = (await r.text())[:2000]
|
||
last_err = f"HTTP {r.status}: {body[:300]}"
|
||
logger.warning("DeepSeek 检测 %s 第%d/%d次尝试失败：HTTP %s\n响应体：%s",
|
||
fname, attempt, cfg.retries, r.status, body)
|
||
except Exception: # noqa: BLE001 网络层异常：记录完整堆栈后重试
|
||
last_err = "网络/协议异常（详见日志）"
|
||
logger.exception("DeepSeek 检测 %s 第%d/%d次尝试抛出异常", fname, attempt, cfg.retries)
|
||
if attempt < cfg.retries:
|
||
logger.info("DeepSeek 检测 %s：%.0f 秒后重试", fname, 3 * attempt)
|
||
await asyncio.sleep(3 * attempt)
|
||
logger.error("DeepSeek 检测 %s 最终失败：%s", fname, last_err)
|
||
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
|
||
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
|