logging: 全链路日志与完整异常堆栈
- 新增 logutil:DEBUG 全量滚动日志文件 + 控制台 INFO 简洁输出 + GUI 队列处理器 - providers:每次尝试失败记录 HTTP 状态/响应体,网络异常 logger.exception 留完整堆栈 - pipeline:单任务异常不拖垮整批,缓存命中/阶段耗时/停止过程全程留痕 - CLI/GUI 顶层兜底:未预期异常打印完整 traceback 并指引日志文件;GUI 新增「打开日志」按钮 - 新增 test_logutil(文件落盘/级别过滤/异常堆栈/队列),共 33 个测试全部通过
This commit is contained in:
@@ -10,6 +10,9 @@ tmp/
|
|||||||
# 检测缓存(断点续跑数据,含运行记录)
|
# 检测缓存(断点续跑数据,含运行记录)
|
||||||
runs/
|
runs/
|
||||||
|
|
||||||
|
# 运行日志
|
||||||
|
logs/
|
||||||
|
|
||||||
# 检测输出(生成物)
|
# 检测输出(生成物)
|
||||||
output/
|
output/
|
||||||
检测*/
|
检测*/
|
||||||
|
|||||||
@@ -94,6 +94,20 @@ max_tokens = 5000
|
|||||||
└── legacy/ # 早期脚本与实验数据
|
└── legacy/ # 早期脚本与实验数据
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## 日志与排错
|
||||||
|
|
||||||
|
所有运行过程写入日志文件(exe 同目录 / 源码项目根目录下的 `logs/violation_detector_日期.log`):
|
||||||
|
|
||||||
|
- 文件记录 DEBUG 级全量:运行配置、缓存命中、每次请求的重试与 HTTP 响应体、异常完整堆栈;
|
||||||
|
- 控制台/GUI 只显示 INFO 级简洁信息,异常时同样附带完整 traceback;
|
||||||
|
- GUI 右下角「打开日志」按钮直达日志文件;CLI 出错时会在结尾打印日志文件路径。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 跑测试(33 个单元测试,无需网络与 API Key)
|
||||||
|
uv run --with pytest --with aiohttp --with openpyxl --no-project \
|
||||||
|
env PYTHONPATH=src pytest tests -q
|
||||||
|
```
|
||||||
|
|
||||||
## 检测规则与已知特性
|
## 检测规则与已知特性
|
||||||
|
|
||||||
- 分类按提示词 17 类标准输出唯一分类,自动归一化写法差异。
|
- 分类按提示词 17 类标准输出唯一分类,自动归一化写法差异。
|
||||||
|
|||||||
@@ -2,13 +2,17 @@
|
|||||||
"""CLI 入口:python -m violation_detector <图片文件夹> [选项]"""
|
"""CLI 入口:python -m violation_detector <图片文件夹> [选项]"""
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from . import __version__
|
from . import __version__
|
||||||
from .config import DEFAULT_CONFIG, load_config, missing_fields
|
from .config import DEFAULT_CONFIG, load_config, missing_fields
|
||||||
|
from .logutil import log_file, setup_logging
|
||||||
from .pipeline import run_detection
|
from .pipeline import run_detection
|
||||||
from .report import build_report, organize_output
|
from .report import build_report, organize_output
|
||||||
|
|
||||||
|
logger = logging.getLogger("violation_detector.cli")
|
||||||
|
|
||||||
|
|
||||||
def build_parser() -> argparse.ArgumentParser:
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
p = argparse.ArgumentParser(
|
p = argparse.ArgumentParser(
|
||||||
@@ -32,6 +36,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
|
|
||||||
def main(argv=None) -> int:
|
def main(argv=None) -> int:
|
||||||
args = build_parser().parse_args(argv)
|
args = build_parser().parse_args(argv)
|
||||||
|
log_path = setup_logging(console=True)
|
||||||
|
logger.info("商品图合规检测工具 v%s 启动,日志文件:%s", __version__, log_path)
|
||||||
cfg = load_config(args.config)
|
cfg = load_config(args.config)
|
||||||
if args.prompt:
|
if args.prompt:
|
||||||
cfg.prompt_file = args.prompt
|
cfg.prompt_file = args.prompt
|
||||||
@@ -53,13 +59,25 @@ def main(argv=None) -> int:
|
|||||||
rows, summary = asyncio.run(run_detection(
|
rows, summary = asyncio.run(run_detection(
|
||||||
args.folder, cfg, mode=args.mode, verify=not args.no_verify))
|
args.folder, cfg, mode=args.mode, verify=not args.no_verify))
|
||||||
except FileNotFoundError as e:
|
except FileNotFoundError as e:
|
||||||
print(f"错误:{e}")
|
logger.error("路径或文件不存在:%s", e)
|
||||||
|
return 1
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
logger.warning("用户中断")
|
||||||
|
return 130
|
||||||
|
except Exception: # noqa: BLE001 顶层兜底:完整堆栈进日志与控制台
|
||||||
|
logger.exception("检测过程发生未预期异常")
|
||||||
|
print(f"\n检测失败,完整日志:{log_file()}")
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
try:
|
||||||
out_dir = args.output or args.folder
|
out_dir = args.output or args.folder
|
||||||
ts_dir = organize_output(rows, args.folder, out_dir)
|
ts_dir = organize_output(rows, args.folder, out_dir)
|
||||||
xlsx = build_report(rows, str(ts_dir), {"mode": args.mode, **{
|
xlsx = build_report(rows, str(ts_dir), {"mode": args.mode, **{
|
||||||
k: summary.get(k) for k in ("ds_calls", "ds_peak_cost", "ds_idle_cost")}})
|
k: summary.get(k) for k in ("ds_calls", "ds_peak_cost", "ds_idle_cost")}})
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
logger.exception("输出组织/报表生成失败")
|
||||||
|
print(f"\n报表生成失败,完整日志:{log_file()}")
|
||||||
|
return 1
|
||||||
print(f"\n分类输出目录:{ts_dir}")
|
print(f"\n分类输出目录:{ts_dir}")
|
||||||
print(f"报表已生成:{xlsx}")
|
print(f"报表已生成:{xlsx}")
|
||||||
return 0
|
return 0
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
检测在后台线程运行 asyncio 事件循环;日志与进度经线程安全队列由主线程刷新。
|
检测在后台线程运行 asyncio 事件循环;日志与进度经线程安全队列由主线程刷新。
|
||||||
"""
|
"""
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import queue
|
import queue
|
||||||
import threading
|
import threading
|
||||||
@@ -14,9 +15,12 @@ from tkinter import filedialog, messagebox, scrolledtext, ttk
|
|||||||
|
|
||||||
from . import __version__
|
from . import __version__
|
||||||
from .config import load_config, missing_fields, save_config
|
from .config import load_config, missing_fields, save_config
|
||||||
|
from .logutil import QueueHandler, log_file, setup_logging
|
||||||
from .pipeline import run_detection
|
from .pipeline import run_detection
|
||||||
from .report import build_report, organize_output
|
from .report import build_report, organize_output
|
||||||
|
|
||||||
|
logger = logging.getLogger("violation_detector")
|
||||||
|
|
||||||
|
|
||||||
class FirstRunDialog:
|
class FirstRunDialog:
|
||||||
"""首次配置对话框:补齐级联模式必填的四项模型配置并写入 config.ini。"""
|
"""首次配置对话框:补齐级联模式必填的四项模型配置并写入 config.ini。"""
|
||||||
@@ -68,6 +72,8 @@ class App:
|
|||||||
root.geometry("880x640")
|
root.geometry("880x640")
|
||||||
self.cfg = load_config()
|
self.cfg = load_config()
|
||||||
self.q: queue.Queue = queue.Queue()
|
self.q: queue.Queue = queue.Queue()
|
||||||
|
setup_logging(console=False)
|
||||||
|
logger.addHandler(QueueHandler(self.q))
|
||||||
self.stop_event = threading.Event()
|
self.stop_event = threading.Event()
|
||||||
self.worker: threading.Thread | None = None
|
self.worker: threading.Thread | None = None
|
||||||
self.last_xlsx = ""
|
self.last_xlsx = ""
|
||||||
@@ -110,6 +116,7 @@ class App:
|
|||||||
self.btn_open.pack(side="left", padx=8)
|
self.btn_open.pack(side="left", padx=8)
|
||||||
self.btn_folder = ttk.Button(ctrl, text="打开输出目录", command=self.open_outdir, state="disabled")
|
self.btn_folder = ttk.Button(ctrl, text="打开输出目录", command=self.open_outdir, state="disabled")
|
||||||
self.btn_folder.pack(side="left", padx=8)
|
self.btn_folder.pack(side="left", padx=8)
|
||||||
|
ttk.Button(ctrl, text="打开日志", command=self.open_log).pack(side="right")
|
||||||
|
|
||||||
self.progress = ttk.Progressbar(frm, mode="determinate")
|
self.progress = ttk.Progressbar(frm, mode="determinate")
|
||||||
self.progress.pack(fill="x", pady=(6, 2))
|
self.progress.pack(fill="x", pady=(6, 2))
|
||||||
@@ -162,7 +169,13 @@ class App:
|
|||||||
xlsx, outdir, summary = payload
|
xlsx, outdir, summary = payload
|
||||||
self.btn_run.configure(state="normal")
|
self.btn_run.configure(state="normal")
|
||||||
self.btn_stop.configure(state="disabled")
|
self.btn_stop.configure(state="disabled")
|
||||||
if xlsx:
|
if not xlsx:
|
||||||
|
messagebox.showerror(
|
||||||
|
"检测失败",
|
||||||
|
f"检测未能完成,错误详情已在上方日志窗口显示,\n"
|
||||||
|
f"完整堆栈已写入日志文件:\n{log_file()}\n\n"
|
||||||
|
f"(也可点\"打开日志\"查看)")
|
||||||
|
continue
|
||||||
self.btn_open.configure(state="normal")
|
self.btn_open.configure(state="normal")
|
||||||
self.btn_folder.configure(state="normal")
|
self.btn_folder.configure(state="normal")
|
||||||
self.last_xlsx, self.last_outdir = xlsx, outdir
|
self.last_xlsx, self.last_outdir = xlsx, outdir
|
||||||
@@ -172,7 +185,7 @@ class App:
|
|||||||
"检测完成",
|
"检测完成",
|
||||||
f"图片 {summary.get('total', 0)} 张已处理,已按分类归档。\n\n分类分布:\n{top}\n\n"
|
f"图片 {summary.get('total', 0)} 张已处理,已按分类归档。\n\n分类分布:\n{top}\n\n"
|
||||||
f"DeepSeek 成本≈¥{summary.get('ds_peak_cost', 0):.2f}(高峰)/ "
|
f"DeepSeek 成本≈¥{summary.get('ds_peak_cost', 0):.2f}(高峰)/ "
|
||||||
f"¥{summary.get('ds_idle_cost', 0):.2f}(空闲)\n输出目录:{outdir or '未生成'}")
|
f"¥{summary.get('ds_idle_cost', 0):.2f}(空闲)\n输出目录:{outdir}")
|
||||||
except queue.Empty:
|
except queue.Empty:
|
||||||
pass
|
pass
|
||||||
self.root.after(100, self.poll_queue)
|
self.root.after(100, self.poll_queue)
|
||||||
@@ -186,6 +199,11 @@ class App:
|
|||||||
if d and os.path.isdir(d):
|
if d and os.path.isdir(d):
|
||||||
os.startfile(d) # noqa: S606
|
os.startfile(d) # noqa: S606
|
||||||
|
|
||||||
|
def open_log(self):
|
||||||
|
lf = log_file()
|
||||||
|
if lf and os.path.isfile(lf):
|
||||||
|
os.startfile(lf) # noqa: S606
|
||||||
|
|
||||||
# ---------- 检测控制 ----------
|
# ---------- 检测控制 ----------
|
||||||
def start(self):
|
def start(self):
|
||||||
folder = self.var_folder.get().strip()
|
folder = self.var_folder.get().strip()
|
||||||
@@ -217,12 +235,9 @@ class App:
|
|||||||
self.btn_open.configure(state="disabled")
|
self.btn_open.configure(state="disabled")
|
||||||
self.btn_folder.configure(state="disabled")
|
self.btn_folder.configure(state="disabled")
|
||||||
self.stop_event.clear()
|
self.stop_event.clear()
|
||||||
self._ui_log(f"===== 开始检测:{folder} =====")
|
logger.info("===== 开始检测:%s(模式 cascade,输出:%s)=====", folder, out_dir)
|
||||||
self.var_progress_label.set("检测中…")
|
self.var_progress_label.set("检测中…")
|
||||||
|
|
||||||
def log(msg):
|
|
||||||
self.q.put(("log", str(msg)))
|
|
||||||
|
|
||||||
def progress(done, total, _label=None):
|
def progress(done, total, _label=None):
|
||||||
self.q.put(("progress", done, total))
|
self.q.put(("progress", done, total))
|
||||||
|
|
||||||
@@ -232,19 +247,19 @@ class App:
|
|||||||
xlsx, outdir, summary = "", "", {}
|
xlsx, outdir, summary = "", "", {}
|
||||||
try:
|
try:
|
||||||
rows, summary = loop.run_until_complete(run_detection(
|
rows, summary = loop.run_until_complete(run_detection(
|
||||||
folder, cfg, log=log, progress=progress,
|
folder, cfg, progress=progress,
|
||||||
stop=self.stop_event.is_set, mode="cascade", verify=True))
|
stop=self.stop_event.is_set, mode="cascade", verify=True))
|
||||||
self.q.put(("log", "正在整理输出(分类归档 + 生成报表)…"))
|
logger.info("正在整理输出(分类归档 + 生成报表)…")
|
||||||
ts_dir = organize_output(rows, folder, out_dir)
|
ts_dir = organize_output(rows, folder, out_dir)
|
||||||
xlsx = build_report(rows, str(ts_dir),
|
xlsx = build_report(rows, str(ts_dir),
|
||||||
{"mode": "cascade", **{k: summary.get(k) for k in
|
{"mode": "cascade", **{k: summary.get(k) for k in
|
||||||
("ds_calls", "ds_peak_cost",
|
("ds_calls", "ds_peak_cost",
|
||||||
"ds_idle_cost")}})
|
"ds_idle_cost")}})
|
||||||
outdir = str(ts_dir)
|
outdir = str(ts_dir)
|
||||||
log(f"输出目录:{ts_dir}")
|
logger.info("输出目录:%s", ts_dir)
|
||||||
log(f"报表:{xlsx}")
|
logger.info("报表:%s", xlsx)
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception: # noqa: BLE001 完整堆栈进日志窗口与日志文件
|
||||||
log(f"检测失败:{type(e).__name__}: {e}")
|
logger.exception("检测失败")
|
||||||
finally:
|
finally:
|
||||||
loop.close()
|
loop.close()
|
||||||
self.q.put(("done", xlsx, outdir, summary))
|
self.q.put(("done", xlsx, outdir, summary))
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""统一日志:滚动文件(DEBUG 全量)+ 控制台(INFO 简洁)+ GUI 队列。
|
||||||
|
|
||||||
|
- 文件记录完整堆栈与调试细节,出错时按日志文件排查;
|
||||||
|
- 控制台/GUI 只看简洁信息,异常同样会附带完整 traceback。
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .config import app_dir
|
||||||
|
|
||||||
|
LOGGER_NAME = "violation_detector"
|
||||||
|
LOG_DIR = app_dir() / "logs"
|
||||||
|
_log_file: Path | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def setup_logging(console: bool = True, log_dir: Path | None = None) -> Path:
|
||||||
|
"""初始化日志(幂等),返回日志文件路径。
|
||||||
|
|
||||||
|
console=True 时向 stdout 输出 INFO 级简洁信息;文件始终记录 DEBUG 级全量。
|
||||||
|
"""
|
||||||
|
global _log_file
|
||||||
|
target_dir = Path(log_dir) if log_dir else LOG_DIR
|
||||||
|
target_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
_log_file = target_dir / f"{LOGGER_NAME}_{datetime.now().strftime('%Y%m%d')}.log"
|
||||||
|
|
||||||
|
root = logging.getLogger(LOGGER_NAME)
|
||||||
|
root.setLevel(logging.DEBUG)
|
||||||
|
root.handlers.clear()
|
||||||
|
root.propagate = False
|
||||||
|
|
||||||
|
detail_fmt = logging.Formatter(
|
||||||
|
"%(asctime)s.%(msecs)03d %(levelname)-7s [%(name)s] %(message)s",
|
||||||
|
datefmt="%Y-%m-%d %H:%M:%S")
|
||||||
|
fh = logging.FileHandler(_log_file, encoding="utf-8")
|
||||||
|
fh.setLevel(logging.DEBUG)
|
||||||
|
fh.setFormatter(detail_fmt)
|
||||||
|
root.addHandler(fh)
|
||||||
|
|
||||||
|
if console:
|
||||||
|
ch = logging.StreamHandler(sys.stdout)
|
||||||
|
ch.setLevel(logging.INFO)
|
||||||
|
ch.setFormatter(logging.Formatter("%(message)s"))
|
||||||
|
root.addHandler(ch)
|
||||||
|
return _log_file
|
||||||
|
|
||||||
|
|
||||||
|
def log_file() -> Path | None:
|
||||||
|
"""当前日志文件路径(未初始化时为 None)。"""
|
||||||
|
return _log_file
|
||||||
|
|
||||||
|
|
||||||
|
class QueueHandler(logging.Handler):
|
||||||
|
"""把日志记录推入线程安全队列(GUI 主线程轮询刷新到日志窗口)。"""
|
||||||
|
|
||||||
|
def __init__(self, q):
|
||||||
|
super().__init__(level=logging.INFO)
|
||||||
|
self.q = q
|
||||||
|
self.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s",
|
||||||
|
datefmt="%H:%M:%S"))
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
try:
|
||||||
|
self.q.put(("log", self.format(record)))
|
||||||
|
except Exception: # noqa: BLE001 日志通道绝不影响业务
|
||||||
|
self.handleError(record)
|
||||||
@@ -7,11 +7,15 @@ mode:
|
|||||||
cascade 默认,豆包初筛 + DeepSeek 复检
|
cascade 默认,豆包初筛 + DeepSeek 复检
|
||||||
doubao 仅豆包
|
doubao 仅豆包
|
||||||
deepseek 仅 DeepSeek(全量直接进 DeepSeek,含无违规两票复核)
|
deepseek 仅 DeepSeek(全量直接进 DeepSeek,含无违规两票复核)
|
||||||
|
|
||||||
|
所有过程与异常通过 logging 输出(文件记录 DEBUG 级全量与完整堆栈)。
|
||||||
"""
|
"""
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -20,6 +24,8 @@ import aiohttp
|
|||||||
from .config import PRICE, AppConfig, RUNS_DIR, load_prompt
|
from .config import PRICE, AppConfig, RUNS_DIR, load_prompt
|
||||||
from .providers import IMAGE_EXTS, detect_ark, detect_deepseek
|
from .providers import IMAGE_EXTS, detect_ark, detect_deepseek
|
||||||
|
|
||||||
|
logger = logging.getLogger("violation_detector.pipeline")
|
||||||
|
|
||||||
STATUS_LABEL = {"ok": "", "parse_fail": "模型返回无法解析为标准JSON",
|
STATUS_LABEL = {"ok": "", "parse_fail": "模型返回无法解析为标准JSON",
|
||||||
"error": "调用失败(网络/服务异常)"}
|
"error": "调用失败(网络/服务异常)"}
|
||||||
|
|
||||||
@@ -75,12 +81,15 @@ def _load_cache(path: Path) -> dict:
|
|||||||
try:
|
try:
|
||||||
return json.loads(path.read_text(encoding="utf-8"))
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
pass
|
logger.exception("缓存文件损坏,忽略并重建:%s", path)
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def _save_cache(path: Path, data: dict):
|
def _save_cache(path: Path, data: dict):
|
||||||
|
try:
|
||||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
logger.exception("缓存写入失败(不影响检测结果):%s", path)
|
||||||
|
|
||||||
|
|
||||||
def _is_peak_now() -> bool:
|
def _is_peak_now() -> bool:
|
||||||
@@ -109,12 +118,16 @@ def deepseek_cost(results: dict) -> tuple:
|
|||||||
|
|
||||||
|
|
||||||
async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_path,
|
async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_path,
|
||||||
log, progress, stop):
|
progress, stop):
|
||||||
"""执行一个检测阶段:跳过缓存中已完成的结果,边跑边落盘。"""
|
"""执行一个检测阶段:跳过缓存中已完成的结果,边跑边落盘。"""
|
||||||
cache = _load_cache(cache_path)
|
cache = _load_cache(cache_path)
|
||||||
todo = [f for f in files if f not in cache or cache[f].get("status") != "ok"]
|
todo = [f for f in files if f not in cache or cache[f].get("status") != "ok"]
|
||||||
total = len(files)
|
total = len(files)
|
||||||
done = total - len(todo)
|
done = total - len(todo)
|
||||||
|
if todo:
|
||||||
|
logger.info(" 缓存命中 %d/%d,本次需检测 %d 张", done, total, len(todo))
|
||||||
|
else:
|
||||||
|
logger.info(" 全部 %d 张命中缓存,跳过请求", total)
|
||||||
if progress:
|
if progress:
|
||||||
progress(done, total)
|
progress(done, total)
|
||||||
|
|
||||||
@@ -125,19 +138,21 @@ async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_pat
|
|||||||
for fut in asyncio.as_completed(tasks):
|
for fut in asyncio.as_completed(tasks):
|
||||||
try:
|
try:
|
||||||
res = await fut
|
res = await fut
|
||||||
except Exception as e: # noqa: BLE001 单任务异常不拖垮整批
|
except Exception: # noqa: BLE001 单任务异常不拖垮整批,完整堆栈进日志
|
||||||
|
logger.exception("检测任务抛出未捕获异常(按检测异常记录)")
|
||||||
res = {"file": "?", "provider": "?", "status": "error", "attr": "",
|
res = {"file": "?", "provider": "?", "status": "error", "attr": "",
|
||||||
"logic": "", "category": "检测异常",
|
"logic": "", "category": "检测异常",
|
||||||
"raw": f"{type(e).__name__}: {e}", "usage": {}}
|
"raw": "未捕获异常(详见日志)", "usage": {}}
|
||||||
cache[res["file"]] = res
|
cache[res["file"]] = res
|
||||||
done += 1
|
done += 1
|
||||||
cat = res["category"] if res["status"] == "ok" else f"[{res['status']}]"
|
cat = res["category"] if res["status"] == "ok" else f"[{res['status']}]"
|
||||||
log(f" ({done}/{total}) {res['file']} -> {cat}")
|
logger.info(" (%d/%d) %s -> %s", done, total, res["file"], cat)
|
||||||
if progress:
|
if progress:
|
||||||
progress(done, total)
|
progress(done, total)
|
||||||
if done % 5 == 0:
|
if done % 5 == 0:
|
||||||
_save_cache(cache_path, cache)
|
_save_cache(cache_path, cache)
|
||||||
if stop and stop():
|
if stop and stop():
|
||||||
|
logger.warning("收到停止请求,取消剩余 %d 个任务", len(tasks) - done)
|
||||||
for t in tasks:
|
for t in tasks:
|
||||||
t.cancel()
|
t.cancel()
|
||||||
break
|
break
|
||||||
@@ -146,7 +161,7 @@ async def _run_stage(detect, session, sem, cfg, prompt, folder, files, cache_pat
|
|||||||
|
|
||||||
|
|
||||||
async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
|
async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
|
||||||
log, progress, stop, cache_path):
|
progress, stop, cache_path):
|
||||||
"""对阶段二判为无违规的图片各追加两票独立复核。"""
|
"""对阶段二判为无违规的图片各追加两票独立复核。"""
|
||||||
verify = [f for f in files
|
verify = [f for f in files
|
||||||
if f in stage2 and stage2[f].get("status") == "ok"
|
if f in stage2 and stage2[f].get("status") == "ok"
|
||||||
@@ -154,7 +169,7 @@ async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
|
|||||||
and "votes" not in stage2[f]]
|
and "votes" not in stage2[f]]
|
||||||
if not verify:
|
if not verify:
|
||||||
return
|
return
|
||||||
log(f"复核阶段:{len(verify)} 张 DeepSeek 无违规图片,各追加 2 票")
|
logger.info("复核阶段:%d 张 DeepSeek 无违规图片,各追加 2 票", len(verify))
|
||||||
|
|
||||||
async def run_verify(fname):
|
async def run_verify(fname):
|
||||||
out = []
|
out = []
|
||||||
@@ -168,10 +183,15 @@ async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
|
|||||||
tasks = [asyncio.ensure_future(run_verify(f)) for f in verify]
|
tasks = [asyncio.ensure_future(run_verify(f)) for f in verify]
|
||||||
vdone = 0
|
vdone = 0
|
||||||
for fut in asyncio.as_completed(tasks):
|
for fut in asyncio.as_completed(tasks):
|
||||||
|
try:
|
||||||
fname, extra = await fut
|
fname, extra = await fut
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
logger.exception("复核任务抛出未捕获异常,跳过该图")
|
||||||
|
continue
|
||||||
runs = [stage2[fname]] + [r for r in extra if r.get("status") == "ok"]
|
runs = [stage2[fname]] + [r for r in extra if r.get("status") == "ok"]
|
||||||
if len(runs) < 2:
|
if len(runs) < 2:
|
||||||
stage2[fname]["vote_note"] = "复核调用失败,维持单票结论"
|
stage2[fname]["vote_note"] = "复核调用失败,维持单票结论"
|
||||||
|
logger.error("复核调用失败:%s", fname)
|
||||||
else:
|
else:
|
||||||
final, note = decide_final(runs)
|
final, note = decide_final(runs)
|
||||||
final = dict(final)
|
final = dict(final)
|
||||||
@@ -179,22 +199,26 @@ async def _verify_clean(session, sem, prompt, folder, files, stage2, cfg,
|
|||||||
final["vote_note"] = note
|
final["vote_note"] = note
|
||||||
stage2[fname] = final
|
stage2[fname] = final
|
||||||
vdone += 1
|
vdone += 1
|
||||||
log(f" 复核 ({vdone}/{len(verify)}) {fname} -> {stage2[fname]['category']}|"
|
logger.info(" 复核 (%d/%d) %s -> %s|%s", vdone, len(verify), fname,
|
||||||
f"{stage2[fname].get('vote_note', '')}")
|
stage2[fname]["category"], stage2[fname].get("vote_note", ""))
|
||||||
if progress:
|
if progress:
|
||||||
progress(vdone, len(verify))
|
progress(vdone, len(verify))
|
||||||
_save_cache(cache_path, stage2)
|
_save_cache(cache_path, stage2)
|
||||||
return stage2
|
return stage2
|
||||||
|
|
||||||
|
|
||||||
async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=None,
|
async def run_detection(folder, cfg: AppConfig, progress=None, stop=None,
|
||||||
mode="cascade", verify=True):
|
mode="cascade", verify=True):
|
||||||
"""执行级联检测,返回 (rows, summary)。rows 已按文件名自然排序。"""
|
"""执行级联检测,返回 (rows, summary)。rows 已按文件名自然排序。"""
|
||||||
|
t0 = time.monotonic()
|
||||||
prompt = load_prompt(cfg.prompt_path)
|
prompt = load_prompt(cfg.prompt_path)
|
||||||
files = list_images(folder)
|
files = list_images(folder)
|
||||||
if not files:
|
if not files:
|
||||||
raise FileNotFoundError(f"文件夹中没有图片: {folder}")
|
raise FileNotFoundError(f"文件夹中没有图片: {folder}")
|
||||||
log(f"共 {len(files)} 张图片,模式:{mode},提示词:{cfg.prompt_path}")
|
logger.info("共 %d 张图片,模式:%s,提示词:%s", len(files), mode, cfg.prompt_path)
|
||||||
|
logger.debug("运行配置:ark=%s workers=%d | deepseek=%s workers=%d max_tokens=%d "
|
||||||
|
"| recheck=%s", cfg.ark_model, cfg.ark_workers, cfg.deepseek_model,
|
||||||
|
cfg.deepseek_workers, cfg.max_tokens, cfg.recheck_categories)
|
||||||
|
|
||||||
ark_cache_path = _cache_path("ark", folder)
|
ark_cache_path = _cache_path("ark", folder)
|
||||||
ds_cache_path = _cache_path("deepseek", folder)
|
ds_cache_path = _cache_path("deepseek", folder)
|
||||||
@@ -206,10 +230,10 @@ async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=N
|
|||||||
# ---- 阶段一:豆包初筛(cascade/doubao 模式)----
|
# ---- 阶段一:豆包初筛(cascade/doubao 模式)----
|
||||||
stage1 = {}
|
stage1 = {}
|
||||||
if mode in ("cascade", "doubao"):
|
if mode in ("cascade", "doubao"):
|
||||||
log(f"[阶段一] 豆包初筛(并发 {cfg.ark_workers})")
|
logger.info("[阶段一] 豆包初筛(并发 %d)", cfg.ark_workers)
|
||||||
sem1 = asyncio.Semaphore(cfg.ark_workers)
|
sem1 = asyncio.Semaphore(cfg.ark_workers)
|
||||||
stage1 = await _run_stage(detect_ark, session, sem1, cfg, prompt, folder,
|
stage1 = await _run_stage(detect_ark, session, sem1, cfg, prompt, folder,
|
||||||
files, ark_cache_path, log, progress, stop)
|
files, ark_cache_path, progress, stop)
|
||||||
|
|
||||||
# ---- 阶段二:DeepSeek 复检 ----
|
# ---- 阶段二:DeepSeek 复检 ----
|
||||||
stage2 = {}
|
stage2 = {}
|
||||||
@@ -218,25 +242,25 @@ async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=N
|
|||||||
if stage1.get(f, {}).get("category") in cfg.recheck_categories
|
if stage1.get(f, {}).get("category") in cfg.recheck_categories
|
||||||
or stage1.get(f, {}).get("status") != "ok"]
|
or stage1.get(f, {}).get("status") != "ok"]
|
||||||
if recheck:
|
if recheck:
|
||||||
log(f"[阶段二] DeepSeek 复检 {len(recheck)} 张(初筛为"
|
logger.info("[阶段二] DeepSeek 复检 %d 张(初筛为%s,并发 %d)",
|
||||||
f"{'/'.join(cfg.recheck_categories)},并发 {cfg.deepseek_workers})")
|
len(recheck), "/".join(cfg.recheck_categories),
|
||||||
|
cfg.deepseek_workers)
|
||||||
sem2 = asyncio.Semaphore(cfg.deepseek_workers)
|
sem2 = asyncio.Semaphore(cfg.deepseek_workers)
|
||||||
stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt,
|
stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt,
|
||||||
folder, recheck, ds_cache_path, log,
|
folder, recheck, ds_cache_path, progress, stop)
|
||||||
progress, stop)
|
|
||||||
else:
|
else:
|
||||||
log("[阶段二] 无需复检的图片")
|
logger.info("[阶段二] 无需复检的图片")
|
||||||
elif mode == "deepseek":
|
elif mode == "deepseek":
|
||||||
log(f"[阶段二] DeepSeek 全量检测(并发 {cfg.deepseek_workers})")
|
logger.info("[阶段二] DeepSeek 全量检测(并发 %d)", cfg.deepseek_workers)
|
||||||
sem2 = asyncio.Semaphore(cfg.deepseek_workers)
|
sem2 = asyncio.Semaphore(cfg.deepseek_workers)
|
||||||
stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt, folder,
|
stage2 = await _run_stage(detect_deepseek, session, sem2, cfg, prompt, folder,
|
||||||
files, ds_cache_path, log, progress, stop)
|
files, ds_cache_path, progress, stop)
|
||||||
|
|
||||||
# ---- 无违规两票复核 ----
|
# ---- 无违规两票复核 ----
|
||||||
if stage2 and verify and not (stop and stop()):
|
if stage2 and verify and not (stop and stop()):
|
||||||
sem2 = asyncio.Semaphore(cfg.deepseek_workers)
|
sem2 = asyncio.Semaphore(cfg.deepseek_workers)
|
||||||
await _verify_clean(session, sem2, prompt, folder, files, stage2, cfg,
|
await _verify_clean(session, sem2, prompt, folder, files, stage2, cfg,
|
||||||
log, progress, stop, ds_cache_path)
|
progress, stop, ds_cache_path)
|
||||||
|
|
||||||
# ---- 合并结果 ----
|
# ---- 合并结果 ----
|
||||||
rows = []
|
rows = []
|
||||||
@@ -263,7 +287,8 @@ async def run_detection(folder, cfg: AppConfig, log=print, progress=None, stop=N
|
|||||||
_save_cache(_cache_path("final", folder),
|
_save_cache(_cache_path("final", folder),
|
||||||
{"rows": rows, "summary": summary,
|
{"rows": rows, "summary": summary,
|
||||||
"generated_at": datetime.now().isoformat(timespec="seconds")})
|
"generated_at": datetime.now().isoformat(timespec="seconds")})
|
||||||
log(f"完成:{json.dumps(summary['counts'], ensure_ascii=False)}")
|
logger.info("完成:%s", json.dumps(summary["counts"], ensure_ascii=False))
|
||||||
log(f"DeepSeek 用量:{summary['ds_calls']} 次调用,输出 {comp} token,"
|
logger.info("DeepSeek 用量:%d 次调用,输出 %d token,成本≈¥%.2f(高峰)/¥%.2f(空闲)",
|
||||||
f"成本≈¥{peak:.2f}(高峰)/¥{idle:.2f}(空闲)")
|
summary["ds_calls"], summary["ds_output_tokens"], peak, idle)
|
||||||
|
logger.info("总耗时 %.1f 秒", time.monotonic() - t0)
|
||||||
return rows, summary
|
return rows, summary
|
||||||
|
|||||||
@@ -8,6 +8,7 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
@@ -15,6 +16,8 @@ import aiohttp
|
|||||||
|
|
||||||
from .config import AppConfig
|
from .config import AppConfig
|
||||||
|
|
||||||
|
logger = logging.getLogger("violation_detector.providers")
|
||||||
|
|
||||||
MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||||
".webp": "image/webp", ".gif": "image/gif", ".bmp": "image/bmp",
|
".webp": "image/webp", ".gif": "image/gif", ".bmp": "image/bmp",
|
||||||
".tiff": "image/tiff"}
|
".tiff": "image/tiff"}
|
||||||
@@ -114,6 +117,7 @@ async def detect_ark(session, sem, cfg: AppConfig, prompt: str, path: str, stop=
|
|||||||
last_err = ""
|
last_err = ""
|
||||||
for attempt in range(1, cfg.retries + 1):
|
for attempt in range(1, cfg.retries + 1):
|
||||||
if stop and stop():
|
if stop and stop():
|
||||||
|
logger.info("豆包检测 %s:用户停止", fname)
|
||||||
return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
|
return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
|
||||||
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
|
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
|
||||||
try:
|
try:
|
||||||
@@ -123,11 +127,17 @@ async def detect_ark(session, sem, cfg: AppConfig, prompt: str, path: str, stop=
|
|||||||
data = await r.json()
|
data = await r.json()
|
||||||
content = data["choices"][0]["message"]["content"]
|
content = data["choices"][0]["message"]["content"]
|
||||||
return _result(fname, "doubao", content, data)
|
return _result(fname, "doubao", content, data)
|
||||||
last_err = f"HTTP {r.status}: {(await r.text())[:300]}"
|
body = (await r.text())[:2000]
|
||||||
except Exception as e: # noqa: BLE001
|
last_err = f"HTTP {r.status}: {body[:300]}"
|
||||||
last_err = f"{type(e).__name__}: {e}"
|
logger.warning("豆包检测 %s 第%d/%d次尝试失败:HTTP %s\n响应体:%s",
|
||||||
|
fname, attempt, cfg.retries, r.status, body)
|
||||||
|
except Exception: # noqa: BLE001 网络层异常:记录完整堆栈后重试
|
||||||
|
last_err = "网络/协议异常(详见日志)"
|
||||||
|
logger.exception("豆包检测 %s 第%d/%d次尝试抛出异常", fname, attempt, cfg.retries)
|
||||||
if attempt < cfg.retries:
|
if attempt < cfg.retries:
|
||||||
|
logger.info("豆包检测 %s:%.0f 秒后重试", fname, 3 * attempt)
|
||||||
await asyncio.sleep(3 * attempt)
|
await asyncio.sleep(3 * attempt)
|
||||||
|
logger.error("豆包检测 %s 最终失败:%s", fname, last_err)
|
||||||
return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
|
return {"file": fname, "provider": "doubao", "status": "error", "attr": "",
|
||||||
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
|
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
|
||||||
|
|
||||||
@@ -156,6 +166,7 @@ async def detect_deepseek(session, sem, cfg: AppConfig, prompt: str, path: str,
|
|||||||
last_err = ""
|
last_err = ""
|
||||||
for attempt in range(1, cfg.retries + 1):
|
for attempt in range(1, cfg.retries + 1):
|
||||||
if stop and stop():
|
if stop and stop():
|
||||||
|
logger.info("DeepSeek 检测 %s:用户停止", fname)
|
||||||
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
|
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
|
||||||
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
|
"logic": "", "category": "检测异常", "raw": "用户停止", "usage": {}}
|
||||||
try:
|
try:
|
||||||
@@ -167,12 +178,22 @@ async def detect_deepseek(session, sem, cfg: AppConfig, prompt: str, path: str,
|
|||||||
if content and content.strip():
|
if content and content.strip():
|
||||||
return _result(fname, "deepseek", content, data)
|
return _result(fname, "deepseek", content, data)
|
||||||
# 推理烧尽预算会得到空正文,按可重试错误处理
|
# 推理烧尽预算会得到空正文,按可重试错误处理
|
||||||
last_err = "empty content (reasoning exhausted max_tokens?)"
|
fr = data["choices"][0].get("finish_reason")
|
||||||
|
last_err = f"empty content (finish_reason={fr}, max_tokens={cfg.max_tokens})"
|
||||||
|
logger.warning("DeepSeek 检测 %s 第%d/%d次尝试返回空正文"
|
||||||
|
"(finish_reason=%s,推理可能耗尽 max_tokens=%s)",
|
||||||
|
fname, attempt, cfg.retries, fr, cfg.max_tokens)
|
||||||
else:
|
else:
|
||||||
last_err = f"HTTP {r.status}: {(await r.text())[:300]}"
|
body = (await r.text())[:2000]
|
||||||
except Exception as e: # noqa: BLE001
|
last_err = f"HTTP {r.status}: {body[:300]}"
|
||||||
last_err = f"{type(e).__name__}: {e}"
|
logger.warning("DeepSeek 检测 %s 第%d/%d次尝试失败:HTTP %s\n响应体:%s",
|
||||||
|
fname, attempt, cfg.retries, r.status, body)
|
||||||
|
except Exception: # noqa: BLE001 网络层异常:记录完整堆栈后重试
|
||||||
|
last_err = "网络/协议异常(详见日志)"
|
||||||
|
logger.exception("DeepSeek 检测 %s 第%d/%d次尝试抛出异常", fname, attempt, cfg.retries)
|
||||||
if attempt < cfg.retries:
|
if attempt < cfg.retries:
|
||||||
|
logger.info("DeepSeek 检测 %s:%.0f 秒后重试", fname, 3 * attempt)
|
||||||
await asyncio.sleep(3 * attempt)
|
await asyncio.sleep(3 * attempt)
|
||||||
|
logger.error("DeepSeek 检测 %s 最终失败:%s", fname, last_err)
|
||||||
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
|
return {"file": fname, "provider": "deepseek", "status": "error", "attr": "",
|
||||||
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
|
"logic": "", "category": "检测异常", "raw": last_err, "usage": {}}
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""logutil 模块测试:文件落盘、级别过滤、异常堆栈、队列处理器。"""
|
||||||
|
import logging
|
||||||
|
import queue
|
||||||
|
|
||||||
|
from violation_detector.logutil import LOGGER_NAME, QueueHandler, setup_logging
|
||||||
|
|
||||||
|
|
||||||
|
def get_logger():
|
||||||
|
return logging.getLogger(LOGGER_NAME)
|
||||||
|
|
||||||
|
|
||||||
|
def test_setup_creates_file_and_logs(tmp_path):
|
||||||
|
path = setup_logging(console=False, log_dir=tmp_path)
|
||||||
|
assert path.exists()
|
||||||
|
get_logger().info("你好日志")
|
||||||
|
for h in get_logger().handlers:
|
||||||
|
h.flush()
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
assert "你好日志" in text
|
||||||
|
assert "[violation_detector]" in text # 详细格式含模块名
|
||||||
|
|
||||||
|
|
||||||
|
def test_exception_logs_full_traceback(tmp_path):
|
||||||
|
path = setup_logging(console=False, log_dir=tmp_path)
|
||||||
|
try:
|
||||||
|
raise ValueError("爆炸点")
|
||||||
|
except ValueError:
|
||||||
|
get_logger().exception("捕获到异常")
|
||||||
|
for h in get_logger().handlers:
|
||||||
|
h.flush()
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
assert "Traceback (most recent call last)" in text
|
||||||
|
assert "ValueError: 爆炸点" in text
|
||||||
|
assert "test_exception_logs_full_traceback" in text # 堆栈含调用位置
|
||||||
|
|
||||||
|
|
||||||
|
def test_debug_goes_to_file_not_queue(tmp_path):
|
||||||
|
"""文件记录 DEBUG 全量;队列处理器(GUI)只收 INFO 以上。"""
|
||||||
|
path = setup_logging(console=False, log_dir=tmp_path)
|
||||||
|
q = queue.Queue()
|
||||||
|
get_logger().addHandler(QueueHandler(q))
|
||||||
|
get_logger().debug("调试细节")
|
||||||
|
get_logger().warning("告警信息")
|
||||||
|
msgs = []
|
||||||
|
while not q.empty():
|
||||||
|
msgs.append(q.get_nowait()[1])
|
||||||
|
# 队列里只有 WARNING,DEBUG 只进文件
|
||||||
|
assert any("告警信息" in m for m in msgs)
|
||||||
|
assert not any("调试细节" in m for m in msgs)
|
||||||
|
for h in get_logger().handlers:
|
||||||
|
if isinstance(h, logging.FileHandler):
|
||||||
|
h.stream.flush()
|
||||||
|
assert "调试细节" in path.read_text(encoding="utf-8")
|
||||||
Reference in New Issue
Block a user