logging: 全链路日志与完整异常堆栈

- 新增 logutil:DEBUG 全量滚动日志文件 + 控制台 INFO 简洁输出 + GUI 队列处理器
- providers:每次尝试失败记录 HTTP 状态/响应体,网络异常 logger.exception 留完整堆栈
- pipeline:单任务异常不拖垮整批,缓存命中/阶段耗时/停止过程全程留痕
- CLI/GUI 顶层兜底:未预期异常打印完整 traceback 并指引日志文件;GUI 新增「打开日志」按钮
- 新增 test_logutil(文件落盘/级别过滤/异常堆栈/队列),共 33 个测试全部通过
This commit is contained in:
yeuimu
2026-09-02 16:01:44 +08:00
parent 08d26710b4
commit 0033b7a4d2
8 changed files with 271 additions and 53 deletions
+3
View File
@@ -10,6 +10,9 @@ tmp/
# 检测缓存(断点续跑数据,含运行记录) # 检测缓存(断点续跑数据,含运行记录)
runs/ runs/
# 运行日志
logs/
# 检测输出(生成物) # 检测输出(生成物)
output/ output/
检测*/ 检测*/
+14
View File
@@ -94,6 +94,20 @@ max_tokens = 5000
└── legacy/ # 早期脚本与实验数据 └── legacy/ # 早期脚本与实验数据
``` ```
## 日志与排错
所有运行过程写入日志文件(exe 同目录 / 源码项目根目录下的 `logs/violation_detector_日期.log`):
- 文件记录 DEBUG 级全量:运行配置、缓存命中、每次请求的重试与 HTTP 响应体、异常完整堆栈;
- 控制台/GUI 只显示 INFO 级简洁信息,异常时同样附带完整 traceback;
- GUI 右下角「打开日志」按钮直达日志文件;CLI 出错时会在结尾打印日志文件路径。
```bash
# 跑测试(33 个单元测试,无需网络与 API Key)
uv run --with pytest --with aiohttp --with openpyxl --no-project \
env PYTHONPATH=src pytest tests -q
```
## 检测规则与已知特性 ## 检测规则与已知特性
- 分类按提示词 17 类标准输出唯一分类,自动归一化写法差异。 - 分类按提示词 17 类标准输出唯一分类,自动归一化写法差异。
+19 -1
View File
@@ -2,13 +2,17 @@
"""CLI 入口:python -m violation_detector <图片文件夹> [选项]""" """CLI 入口:python -m violation_detector <图片文件夹> [选项]"""
import argparse import argparse
import asyncio import asyncio
import logging
import sys import sys
from . import __version__ from . import __version__
from .config import DEFAULT_CONFIG, load_config, missing_fields from .config import DEFAULT_CONFIG, load_config, missing_fields
from .logutil import log_file, setup_logging
from .pipeline import run_detection from .pipeline import run_detection
from .report import build_report, organize_output from .report import build_report, organize_output
logger = logging.getLogger("violation_detector.cli")
def build_parser() -> argparse.ArgumentParser: def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser( p = argparse.ArgumentParser(
@@ -32,6 +36,8 @@ def build_parser() -> argparse.ArgumentParser:
def main(argv=None) -> int: def main(argv=None) -> int:
args = build_parser().parse_args(argv) args = build_parser().parse_args(argv)
log_path = setup_logging(console=True)
logger.info("商品图合规检测工具 v%s 启动,日志文件:%s", __version__, log_path)
cfg = load_config(args.config) cfg = load_config(args.config)
if args.prompt: if args.prompt:
cfg.prompt_file = args.prompt cfg.prompt_file = args.prompt
@@ -53,13 +59,25 @@ def main(argv=None) -> int:
rows, summary = asyncio.run(run_detection( rows, summary = asyncio.run(run_detection(
args.folder, cfg, mode=args.mode, verify=not args.no_verify)) args.folder, cfg, mode=args.mode, verify=not args.no_verify))
except FileNotFoundError as e: except FileNotFoundError as e:
print(f"错误:{e}") logger.error("路径或文件不存在:%s", e)
return 1
except KeyboardInterrupt:
logger.warning("用户中断")
return 130
except Exception: # noqa: BLE001 顶层兜底:完整堆栈进日志与控制台
logger.exception("检测过程发生未预期异常")
print(f"\n检测失败,完整日志:{log_file()}")
return 1 return 1
try:
out_dir = args.output or args.folder out_dir = args.output or args.folder
ts_dir = organize_output(rows, args.folder, out_dir) ts_dir = organize_output(rows, args.folder, out_dir)
xlsx = build_report(rows, str(ts_dir), {"mode": args.mode, **{ xlsx = build_report(rows, str(ts_dir), {"mode": args.mode, **{
k: summary.get(k) for k in ("ds_calls", "ds_peak_cost", "ds_idle_cost")}}) k: summary.get(k) for k in ("ds_calls", "ds_peak_cost", "ds_idle_cost")}})
except Exception: # noqa: BLE001
logger.exception("输出组织/报表生成失败")
print(f"\n报表生成失败,完整日志:{log_file()}")
return 1
print(f"\n分类输出目录:{ts_dir}") print(f"\n分类输出目录:{ts_dir}")
print(f"报表已生成:{xlsx}") print(f"报表已生成:{xlsx}")
return 0 return 0
+27 -12
View File
@@ -6,6 +6,7 @@
检测在后台线程运行 asyncio 事件循环;日志与进度经线程安全队列由主线程刷新。 检测在后台线程运行 asyncio 事件循环;日志与进度经线程安全队列由主线程刷新。
""" """
import asyncio import asyncio
import logging
import os import os
import queue import queue
import threading import threading
@@ -14,9 +15,12 @@ from tkinter import filedialog, messagebox, scrolledtext, ttk
from . import __version__ from . import __version__
from .config import load_config, missing_fields, save_config from .config import load_config, missing_fields, save_config
from .logutil import QueueHandler, log_file, setup_logging
from .pipeline import run_detection from .pipeline import run_detection
from .report import build_report, organize_output from .report import build_report, organize_output
logger = logging.getLogger("violation_detector")
class FirstRunDialog: class FirstRunDialog:
"""首次配置对话框:补齐级联模式必填的四项模型配置并写入 config.ini。""" """首次配置对话框:补齐级联模式必填的四项模型配置并写入 config.ini。"""
@@ -68,6 +72,8 @@ class App:
root.geometry("880x640") root.geometry("880x640")
self.cfg = load_config() self.cfg = load_config()
self.q: queue.Queue = queue.Queue() self.q: queue.Queue = queue.Queue()
setup_logging(console=False)
logger.addHandler(QueueHandler(self.q))
self.stop_event = threading.Event() self.stop_event = threading.Event()
self.worker: threading.Thread | None = None self.worker: threading.Thread | None = None
self.last_xlsx = "" self.last_xlsx = ""
@@ -110,6 +116,7 @@ class App:
self.btn_open.pack(side="left", padx=8) self.btn_open.pack(side="left", padx=8)
self.btn_folder = ttk.Button(ctrl, text="打开输出目录", command=self.open_outdir, state="disabled") self.btn_folder = ttk.Button(ctrl, text="打开输出目录", command=self.open_outdir, state="disabled")
self.btn_folder.pack(side="left", padx=8) self.btn_folder.pack(side="left", padx=8)
ttk.Button(ctrl, text="打开日志", command=self.open_log).pack(side="right")
self.progress = ttk.Progressbar(frm, mode="determinate") self.progress = ttk.Progressbar(frm, mode="determinate")
self.progress.pack(fill="x", pady=(6, 2)) self.progress.pack(fill="x", pady=(6, 2))
@@ -162,7 +169,13 @@ class App:
xlsx, outdir, summary = payload xlsx, outdir, summary = payload
self.btn_run.configure(state="normal") self.btn_run.configure(state="normal")
self.btn_stop.configure(state="disabled") self.btn_stop.configure(state="disabled")
if xlsx: if not xlsx:
messagebox.showerror(
"检测失败",
f"检测未能完成,错误详情已在上方日志窗口显示,\n"
f"完整堆栈已写入日志文件:\n{log_file()}\n\n"
f"(也可点\"打开日志\"查看)")
continue
self.btn_open.configure(state="normal") self.btn_open.configure(state="normal")
self.btn_folder.configure(state="normal") self.btn_folder.configure(state="normal")
self.last_xlsx, self.last_outdir = xlsx, outdir self.last_xlsx, self.last_outdir = xlsx, outdir
@@ -172,7 +185,7 @@ class App:
"检测完成", "检测完成",
f"图片 {summary.get('total', 0)} 张已处理,已按分类归档。\n\n分类分布:\n{top}\n\n" f"图片 {summary.get('total', 0)} 张已处理,已按分类归档。\n\n分类分布:\n{top}\n\n"
f"DeepSeek 成本≈¥{summary.get('ds_peak_cost', 0):.2f}(高峰)/ " f"DeepSeek 成本≈¥{summary.get('ds_peak_cost', 0):.2f}(高峰)/ "
f"¥{summary.get('ds_idle_cost', 0):.2f}(空闲)\n输出目录:{outdir or '未生成'}") f"¥{summary.get('ds_idle_cost', 0):.2f}(空闲)\n输出目录:{outdir}")
except queue.Empty: except queue.Empty:
pass pass
self.root.after(100, self.poll_queue) self.root.after(100, self.poll_queue)
@@ -186,6 +199,11 @@ class App:
if d and os.path.isdir(d): if d and os.path.isdir(d):
os.startfile(d) # noqa: S606 os.startfile(d) # noqa: S606
def open_log(self):
lf = log_file()
if lf and os.path.isfile(lf):
os.startfile(lf) # noqa: S606
# ---------- 检测控制 ---------- # ---------- 检测控制 ----------
def start(self): def start(self):
folder = self.var_folder.get().strip() folder = self.var_folder.get().strip()
@@ -217,12 +235,9 @@ class App:
self.btn_open.configure(state="disabled") self.btn_open.configure(state="disabled")
self.btn_folder.configure(state="disabled") self.btn_folder.configure(state="disabled")
self.stop_event.clear() self.stop_event.clear()
self._ui_log(f"===== 开始检测:{folder} =====") logger.info("===== 开始检测:%s(模式 cascade,输出:%s=====", folder, out_dir)
self.var_progress_label.set("检测中…") self.var_progress_label.set("检测中…")
def log(msg):
self.q.put(("log", str(msg)))
def progress(done, total, _label=None): def progress(done, total, _label=None):
self.q.put(("progress", done, total)) self.q.put(("progress", done, total))
@@ -232,19 +247,19 @@ class App:
xlsx, outdir, summary = "", "", {} xlsx, outdir, summary = "", "", {}
try: try:
rows, summary = loop.run_until_complete(run_detection( rows, summary = loop.run_until_complete(run_detection(
folder, cfg, log=log, progress=progress, folder, cfg, progress=progress,
stop=self.stop_event.is_set, mode="cascade", verify=True)) stop=self.stop_event.is_set, mode="cascade", verify=True))
self.q.put(("log", "正在整理输出(分类归档 + 生成报表)…")) logger.info("正在整理输出(分类归档 + 生成报表)…")
ts_dir = organize_output(rows, folder, out_dir) ts_dir = organize_output(rows, folder, out_dir)
xlsx = build_report(rows, str(ts_dir), xlsx = build_report(rows, str(ts_dir),
{"mode": "cascade", **{k: summary.get(k) for k in {"mode": "cascade", **{k: summary.get(k) for k in
("ds_calls", "ds_peak_cost", ("ds_calls", "ds_peak_cost",
"ds_idle_cost")}}) "ds_idle_cost")}})
outdir = str(ts_dir) outdir = str(ts_dir)
log(f"输出目录:{ts_dir}") logger.info("输出目录:%s", ts_dir)
log(f"报表:{xlsx}") logger.info("报表:%s", xlsx)
except Exception as e: # noqa: BLE001 except Exception: # noqa: BLE001 完整堆栈进日志窗口与日志文件
log(f"检测失败{type(e).__name__}: {e}") logger.exception("检测失败")
finally: finally:
loop.close() loop.close()
self.q.put(("done", xlsx, outdir, summary)) self.q.put(("done", xlsx, outdir, summary))
+68
View File
@@ -0,0 +1,68 @@
# -*- coding: utf-8 -*-
"""统一日志:滚动文件(DEBUG 全量)+ 控制台(INFO 简洁)+ GUI 队列。
- 文件记录完整堆栈与调试细节,出错时按日志文件排查;
- 控制台/GUI 只看简洁信息,异常同样会附带完整 traceback。
"""
import logging
import sys
from datetime import datetime
from pathlib import Path
from .config import app_dir
LOGGER_NAME = "violation_detector"
LOG_DIR = app_dir() / "logs"
_log_file: Path | None = None
def setup_logging(console: bool = True, log_dir: Path | None = None) -> Path:
"""初始化日志(幂等),返回日志文件路径。
console=True 时向 stdout 输出 INFO 级简洁信息;文件始终记录 DEBUG 级全量。
"""
global _log_file
target_dir = Path(log_dir) if log_dir else LOG_DIR
target_dir.mkdir(parents=True, exist_ok=True)
_log_file = target_dir / f"{LOGGER_NAME}_{datetime.now().strftime('%Y%m%d')}.log"
root = logging.getLogger(LOGGER_NAME)
root.setLevel(logging.DEBUG)
root.handlers.clear()
root.propagate = False
detail_fmt = logging.Formatter(
"%(asctime)s.%(msecs)03d %(levelname)-7s [%(name)s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
fh = logging.FileHandler(_log_file, encoding="utf-8")
fh.setLevel(logging.DEBUG)
fh.setFormatter(detail_fmt)
root.addHandler(fh)
if console:
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
ch.setFormatter(logging.Formatter("%(message)s"))
root.addHandler(ch)
return _log_file
def log_file() -> Path | None:
"""当前日志文件路径(未初始化时为 None)。"""
return _log_file
class QueueHandler(logging.Handler):
"""把日志记录推入线程安全队列(GUI 主线程轮询刷新到日志窗口)。"""
def __init__(self, q):
super().__init__(level=logging.INFO)
self.q = q
self.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s",
datefmt="%H:%M:%S"))
def emit(self, record):
try:
self.q.put(("log", self.format(record)))
except Exception: # noqa: BLE001 日志通道绝不影响业务
self.handleError(record)
+49 -24
View File
@@ -7,11 +7,15 @@ mode:
cascade 默认,豆包初筛 + DeepSeek 复检 cascade 默认,豆包初筛 + DeepSeek 复检
doubao 仅豆包 doubao 仅豆包
deepseek 仅 DeepSeek(全量直接进 DeepSeek,含无违规两票复核) deepseek 仅 DeepSeek(全量直接进 DeepSeek,含无违规两票复核)
所有过程与异常通过 logging 输出(文件记录 DEBUG 级全量与完整堆栈)。
""" """
import asyncio import asyncio
import json import json
import logging
import os import os
import re import re
import time
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
@@ -20,6 +24,8 @@ import aiohttp
from .config import PRICE, AppConfig, RUNS_DIR, load_prompt from .config import PRICE, AppConfig, RUNS_DIR, load_prompt
from .providers import IMAGE_EXTS, detect_ark, detect_deepseek from .providers import IMAGE_EXTS, detect_ark, detect_deepseek
logger = logging.getLogger("violation_detector.pipeline")
STATUS_LABEL = {"ok": "", "parse_fail": "模型返回无法解析为标准JSON", STATUS_LABEL = {"ok": "", "parse_fail": "模型返回无法解析为标准JSON",
"error": "调用失败(网络/服务异常)"} "error": "调用失败(网络/服务异常)"}
@@ -75,12 +81,15 @@ def _load_cache(path: Path) -> dict:
try: try:
return json.loads(path.read_text(encoding="utf-8")) return json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError: except json.JSONDecodeError:
pass logger.exception("缓存文件损坏,忽略并重建:%s", path)
return {} return {}
def _save_cache(path: Path, data: dict): def _save_cache(path: Path, data: dict):
try:
path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8") path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
except OSError:
logger.exception("缓存写入失败(不影响检测结果):%s", path)
def _is_peak_now() -> bool: def _is_peak_now() -> bool:
@@ -109,12 +118,16 @@ def deepseek_cost(results: dict) -> tuple:
async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_path, async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_path,
log, progress, stop): progress, stop):
"""执行一个检测阶段:跳过缓存中已完成的结果,边跑边落盘。""" """执行一个检测阶段:跳过缓存中已完成的结果,边跑边落盘。"""
cache = _load_cache(cache_path) cache = _load_cache(cache_path)
todo = [f for f in files if f not in cache or cache[f].get("status") != "ok"] todo = [f for f in files if f not in cache or cache[f].get("status") != "ok"]
total = len(files) total = len(files)
done = total - len(todo) done = total - len(todo)
if todo:
logger.info(" 缓存命中 %d/%d,本次需检测 %d", done, total, len(todo))
else:
logger.info(" 全部 %d 张命中缓存,跳过请求", total)
if progress: if progress:
progress(done, total) progress(done, total)
@@ -125,19 +138,21 @@ async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_pat
for fut in asyncio.as_completed(tasks): for fut in asyncio.as_completed(tasks):
try: try:
res = await fut res = await fut
except Exception as e: # noqa: BLE001 单任务异常不拖垮整批 except Exception: # noqa: BLE001 单任务异常不拖垮整批,完整堆栈进日志
logger.exception("检测任务抛出未捕获异常(按检测异常记录)")
res = {"file": "?", "provider": "?", "status": "error", "attr": "", res = {"file": "?", "provider": "?", "status": "error", "attr": "",
"logic": "", "category": "检测异常", "logic": "", "category": "检测异常",
"raw": f"{type(e).__name__}: {e}", "usage": {}} "raw": "未捕获异常(详见日志)", "usage": {}}
cache[res["file"]] = res cache[res["file"]] = res
done += 1 done += 1
cat = res["category"] if res["status"] == "ok" else f"[{res['status']}]" cat = res["category"] if res["status"] == "ok" else f"[{res['status']}]"
log(f" ({done}/{total}) {res['file']} -> {cat}") logger.info(" (%d/%d) %s -> %s", done, total, res["file"], cat)
if progress: if progress:
progress(done, total) progress(done, total)
if done % 5 == 0: if done % 5 == 0:
_save_cache(cache_path, cache) _save_cache(cache_path, cache)
if stop and stop(): if stop and stop():
logger.warning("收到停止请求,取消剩余 %d 个任务", len(tasks) - done)
for t in tasks: for t in tasks:
t.cancel() t.cancel()
break break
@@ -146,7 +161,7 @@ async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_pat
async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg, async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
log, progress, stop, cache_path): progress, stop, cache_path):
"""对阶段二判为无违规的图片各追加两票独立复核。""" """对阶段二判为无违规的图片各追加两票独立复核。"""
verify = [f for f in files verify = [f for f in files
if f in stage2 and stage2[f].get("status") == "ok" if f in stage2 and stage2[f].get("status") == "ok"
@@ -154,7 +169,7 @@ async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
and "votes" not in stage2[f]] and "votes" not in stage2[f]]
if not verify: if not verify:
return return
log(f"复核阶段:{len(verify)} 张 DeepSeek 无违规图片,各追加 2 票") logger.info("复核阶段:%d 张 DeepSeek 无违规图片,各追加 2 票", len(verify))
async def run_verify(fname): async def run_verify(fname):
out = [] out = []
@@ -168,10 +183,15 @@ async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
tasks = [asyncio.ensure_future(run_verify(f)) for f in verify] tasks = [asyncio.ensure_future(run_verify(f)) for f in verify]
vdone = 0 vdone = 0
for fut in asyncio.as_completed(tasks): for fut in asyncio.as_completed(tasks):
try:
fname, extra = await fut fname, extra = await fut
except Exception: # noqa: BLE001
logger.exception("复核任务抛出未捕获异常,跳过该图")
continue
runs = [stage2[fname]] + [r for r in extra if r.get("status") == "ok"] runs = [stage2[fname]] + [r for r in extra if r.get("status") == "ok"]
if len(runs) < 2: if len(runs) < 2:
stage2[fname]["vote_note"] = "复核调用失败,维持单票结论" stage2[fname]["vote_note"] = "复核调用失败,维持单票结论"
logger.error("复核调用失败:%s", fname)
else: else:
final, note = decide_final(runs) final, note = decide_final(runs)
final = dict(final) final = dict(final)
@@ -179,22 +199,26 @@ async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
final["vote_note"] = note final["vote_note"] = note
stage2[fname] = final stage2[fname] = final
vdone += 1 vdone += 1
log(f" 复核 ({vdone}/{len(verify)}) {fname} -> {stage2[fname]['category']}" logger.info(" 复核 (%d/%d) %s -> %s%s", vdone, len(verify), fname,
f"{stage2[fname].get('vote_note', '')}") stage2[fname]["category"], stage2[fname].get("vote_note", ""))
if progress: if progress:
progress(vdone, len(verify)) progress(vdone, len(verify))
_save_cache(cache_path, stage2) _save_cache(cache_path, stage2)
return stage2 return stage2
async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=None, async def run_detection(folder, cfg: AppConfig, progress=None, stop=None,
mode="cascade", verify=True): mode="cascade", verify=True):
"""执行级联检测,返回 (rows, summary)。rows 已按文件名自然排序。""" """执行级联检测,返回 (rows, summary)。rows 已按文件名自然排序。"""
t0 = time.monotonic()
prompt = load_prompt(cfg.prompt_path) prompt = load_prompt(cfg.prompt_path)
files = list_images(folder) files = list_images(folder)
if not files: if not files:
raise FileNotFoundError(f"文件夹中没有图片: {folder}") raise FileNotFoundError(f"文件夹中没有图片: {folder}")
log(f"{len(files)} 张图片,模式:{mode},提示词:{cfg.prompt_path}") logger.info("%d 张图片,模式:%s,提示词:%s", len(files), mode, cfg.prompt_path)
logger.debug("运行配置:ark=%s workers=%d | deepseek=%s workers=%d max_tokens=%d "
"| recheck=%s", cfg.ark_model, cfg.ark_workers, cfg.deepseek_model,
cfg.deepseek_workers, cfg.max_tokens, cfg.recheck_categories)
ark_cache_path = _cache_path("ark", folder) ark_cache_path = _cache_path("ark", folder)
ds_cache_path = _cache_path("deepseek", folder) ds_cache_path = _cache_path("deepseek", folder)
@@ -206,10 +230,10 @@ async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=N
# ---- 阶段一:豆包初筛(cascade/doubao 模式)---- # ---- 阶段一:豆包初筛(cascade/doubao 模式)----
stage1 = {} stage1 = {}
if mode in ("cascade", "doubao"): if mode in ("cascade", "doubao"):
log(f"[阶段一] 豆包初筛(并发 {cfg.ark_workers}") logger.info("[阶段一] 豆包初筛(并发 %d", cfg.ark_workers)
sem1 = asyncio.Semaphore(cfg.ark_workers) sem1 = asyncio.Semaphore(cfg.ark_workers)
stage1 = await _run_stage(detect_ark, session, sem1, cfg, prompt, folder, stage1 = await _run_stage(detect_ark, session, sem1, cfg, prompt, folder,
files, ark_cache_path, log, progress, stop) files, ark_cache_path, progress, stop)
# ---- 阶段二:DeepSeek 复检 ---- # ---- 阶段二:DeepSeek 复检 ----
stage2 = {} stage2 = {}
@@ -218,25 +242,25 @@ async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=N
if stage1.get(f, {}).get("category") in cfg.recheck_categories if stage1.get(f, {}).get("category") in cfg.recheck_categories
or stage1.get(f, {}).get("status") != "ok"] or stage1.get(f, {}).get("status") != "ok"]
if recheck: if recheck:
log(f"[阶段二] DeepSeek 复检 {len(recheck)} 张(初筛为" logger.info("[阶段二] DeepSeek 复检 %d 张(初筛为%s,并发 %d",
f"{'/'.join(cfg.recheck_categories)},并发 {cfg.deepseek_workers}") len(recheck), "/".join(cfg.recheck_categories),
cfg.deepseek_workers)
sem2 = asyncio.Semaphore(cfg.deepseek_workers) sem2 = asyncio.Semaphore(cfg.deepseek_workers)
stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt, stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt,
folder, recheck, ds_cache_path, log, folder, recheck, ds_cache_path, progress, stop)
progress, stop)
else: else:
log("[阶段二] 无需复检的图片") logger.info("[阶段二] 无需复检的图片")
elif mode == "deepseek": elif mode == "deepseek":
log(f"[阶段二] DeepSeek 全量检测(并发 {cfg.deepseek_workers}") logger.info("[阶段二] DeepSeek 全量检测(并发 %d", cfg.deepseek_workers)
sem2 = asyncio.Semaphore(cfg.deepseek_workers) sem2 = asyncio.Semaphore(cfg.deepseek_workers)
stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt, folder, stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt, folder,
files, ds_cache_path, log, progress, stop) files, ds_cache_path, progress, stop)
# ---- 无违规两票复核 ---- # ---- 无违规两票复核 ----
if stage2 and verify and not (stop and stop()): if stage2 and verify and not (stop and stop()):
sem2 = asyncio.Semaphore(cfg.deepseek_workers) sem2 = asyncio.Semaphore(cfg.deepseek_workers)
await _verify_clean(session, sem2, prompt, folder, files, stage2, cfg, await _verify_clean(session, sem2, prompt, folder, files, stage2, cfg,
log, progress, stop, ds_cache_path) progress, stop, ds_cache_path)
# ---- 合并结果 ---- # ---- 合并结果 ----
rows = [] rows = []
@@ -263,7 +287,8 @@ async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=N
_save_cache(_cache_path("final", folder), _save_cache(_cache_path("final", folder),
{"rows": rows, "summary": summary, {"rows": rows, "summary": summary,
"generated_at": datetime.now().isoformat(timespec="seconds")}) "generated_at": datetime.now().isoformat(timespec="seconds")})
log(f"完成:{json.dumps(summary['counts'], ensure_ascii=False)}") logger.info("完成:%s", json.dumps(summary["counts"], ensure_ascii=False))
log(f"DeepSeek 用量:{summary['ds_calls']} 次调用,输出 {comp} token" logger.info("DeepSeek 用量:%d 次调用,输出 %d token成本≈¥%.2f(高峰)/¥%.2f(空闲)",
f"成本≈¥{peak:.2f}(高峰)/¥{idle:.2f}(空闲)") summary["ds_calls"], summary["ds_output_tokens"], peak, idle)
logger.info("总耗时 %.1f", time.monotonic() - t0)
return rows, summary return rows, summary
+28 -7
View File
@@ -8,6 +8,7 @@
import asyncio import asyncio
import base64 import base64
import json import json
import logging
import os import os
import re import re
@@ -15,6 +16,8 @@ import aiohttp
from .config import AppConfig from .config import AppConfig
logger = logging.getLogger("violation_detector.providers")
MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
".webp": "image/webp", ".gif": "image/gif", ".bmp": "image/bmp", ".webp": "image/webp", ".gif": "image/gif", ".bmp": "image/bmp",
".tiff": "image/tiff"} ".tiff": "image/tiff"}
@@ -114,6 +117,7 @@ async def detect_ark(session, sem, cfg: AppConfig, prompt: str, path: str, stop=
last_err = "" last_err = ""
for attempt in range(1, cfg.retries + 1): for attempt in range(1, cfg.retries + 1):
if stop and stop(): if stop and stop():
logger.info("豆包检测 %s:用户停止", fname)
return {"file": fname, "provider": "doubao", "status": "error", "attr": "", return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}} "logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
try: try:
@@ -123,11 +127,17 @@ async def detect_ark(session, sem, cfg: AppConfig, prompt: str, path: str, stop=
data = await r.json() data = await r.json()
content = data["choices"][0]["message"]["content"] content = data["choices"][0]["message"]["content"]
return _result(fname, "doubao", content, data) return _result(fname, "doubao", content, data)
last_err = f"HTTP {r.status}: {(await r.text())[:300]}" body = (await r.text())[:2000]
except Exception as e: # noqa: BLE001 last_err = f"HTTP {r.status}: {body[:300]}"
last_err = f"{type(e).__name__}: {e}" logger.warning("豆包检测 %s%d/%d次尝试失败:HTTP %s\n响应体:%s",
fname, attempt, cfg.retries, r.status, body)
except Exception: # noqa: BLE001 网络层异常:记录完整堆栈后重试
last_err = "网络/协议异常(详见日志)"
logger.exception("豆包检测 %s%d/%d次尝试抛出异常", fname, attempt, cfg.retries)
if attempt < cfg.retries: if attempt < cfg.retries:
logger.info("豆包检测 %s%.0f 秒后重试", fname, 3 * attempt)
await asyncio.sleep(3 * attempt) await asyncio.sleep(3 * attempt)
logger.error("豆包检测 %s 最终失败:%s", fname, last_err)
return {"file": fname, "provider": "doubao", "status": "error", "attr": "", return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}} "logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
@@ -156,6 +166,7 @@ async def detect_deepseek(session, sem, cfg: AppConfig, prompt: str, path: str,
last_err = "" last_err = ""
for attempt in range(1, cfg.retries + 1): for attempt in range(1, cfg.retries + 1):
if stop and stop(): if stop and stop():
logger.info("DeepSeek 检测 %s:用户停止", fname)
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "", return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}} "logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
try: try:
@@ -167,12 +178,22 @@ async def detect_deepseek(session, sem, cfg: AppConfig, prompt: str, path: str,
if content and content.strip(): if content and content.strip():
return _result(fname, "deepseek", content, data) return _result(fname, "deepseek", content, data)
# 推理烧尽预算会得到空正文,按可重试错误处理 # 推理烧尽预算会得到空正文,按可重试错误处理
last_err = "empty content (reasoning exhausted max_tokens?)" fr = data["choices"][0].get("finish_reason")
last_err = f"empty content (finish_reason={fr}, max_tokens={cfg.max_tokens})"
logger.warning("DeepSeek 检测 %s%d/%d次尝试返回空正文"
"finish_reason=%s,推理可能耗尽 max_tokens=%s",
fname, attempt, cfg.retries, fr, cfg.max_tokens)
else: else:
last_err = f"HTTP {r.status}: {(await r.text())[:300]}" body = (await r.text())[:2000]
except Exception as e: # noqa: BLE001 last_err = f"HTTP {r.status}: {body[:300]}"
last_err = f"{type(e).__name__}: {e}" logger.warning("DeepSeek 检测 %s%d/%d次尝试失败:HTTP %s\n响应体:%s",
fname, attempt, cfg.retries, r.status, body)
except Exception: # noqa: BLE001 网络层异常:记录完整堆栈后重试
last_err = "网络/协议异常(详见日志)"
logger.exception("DeepSeek 检测 %s%d/%d次尝试抛出异常", fname, attempt, cfg.retries)
if attempt < cfg.retries: if attempt < cfg.retries:
logger.info("DeepSeek 检测 %s%.0f 秒后重试", fname, 3 * attempt)
await asyncio.sleep(3 * attempt) await asyncio.sleep(3 * attempt)
logger.error("DeepSeek 检测 %s 最终失败:%s", fname, last_err)
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "", return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}} "logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
+54
View File
@@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
"""logutil 模块测试:文件落盘、级别过滤、异常堆栈、队列处理器。"""
import logging
import queue
from violation_detector.logutil import LOGGER_NAME, QueueHandler, setup_logging
def get_logger():
return logging.getLogger(LOGGER_NAME)
def test_setup_creates_file_and_logs(tmp_path):
path = setup_logging(console=False, log_dir=tmp_path)
assert path.exists()
get_logger().info("你好日志")
for h in get_logger().handlers:
h.flush()
text = path.read_text(encoding="utf-8")
assert "你好日志" in text
assert "[violation_detector]" in text # 详细格式含模块名
def test_exception_logs_full_traceback(tmp_path):
path = setup_logging(console=False, log_dir=tmp_path)
try:
raise ValueError("爆炸点")
except ValueError:
get_logger().exception("捕获到异常")
for h in get_logger().handlers:
h.flush()
text = path.read_text(encoding="utf-8")
assert "Traceback (most recent call last)" in text
assert "ValueError: 爆炸点" in text
assert "test_exception_logs_full_traceback" in text # 堆栈含调用位置
def test_debug_goes_to_file_not_queue(tmp_path):
"""文件记录 DEBUG 全量;队列处理器(GUI)只收 INFO 以上。"""
path = setup_logging(console=False, log_dir=tmp_path)
q = queue.Queue()
get_logger().addHandler(QueueHandler(q))
get_logger().debug("调试细节")
get_logger().warning("告警信息")
msgs = []
while not q.empty():
msgs.append(q.get_nowait()[1])
# 队列里只有 WARNINGDEBUG 只进文件
assert any("告警信息" in m for m in msgs)
assert not any("调试细节" in m for m in msgs)
for h in get_logger().handlers:
if isinstance(h, logging.FileHandler):
h.stream.flush()
assert "调试细节" in path.read_text(encoding="utf-8")