v88 功能增强:产品落盘持久化 + 生图网关适配 + 模板导出优化
- 产品持久化:每完成一个产品立即追加写入 products_pending.jsonl,崩溃不丢已完成产品,finish 读盘合并后统一写模板 - 503 致命错误提前终止:compose/product/seed_shot 端到端识别,提前终止搜索分析,丢弃未完成简报,保留已完成落盘产品直接合成模板 - 模特分配:material_library 合格模特图按任务序号独立随机,同 SPU 多款不再共用同一模特 - 图像网关适配:execution_mode/background 默认不再传入 yunfei 等标准网关,base_url 需带 /v1;429/5xx/空响应退避重试 - Pinterest 分析:删除 term 注入与纯文本降级,失败直接放弃;图片上传前 PIL 完整性校验;suitable_for_print=False 过滤丢弃 - 模板导出:不再产生空白 xlsx,文件名=模板原文件名_已填写;写入前按货号末 3 位升序排序 - 删除对接文档.md,更新 README,gitignore 排除测试产物
This commit is contained in:
@@ -5,7 +5,9 @@ import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any, Dict
|
||||
from urllib.parse import urlparse, quote
|
||||
|
||||
import aiohttp
|
||||
@@ -218,6 +220,80 @@ def _session_user_data_dir() -> str:
|
||||
return ud
|
||||
|
||||
|
||||
def check_login_state() -> Dict[str, Any]:
|
||||
"""静态检测 Pinterest 登录态(不启动 Chrome,只读持久化 cookies 数据库)。
|
||||
|
||||
通过检查 .chrome_session/User Data 里的 Chrome cookies 数据库:
|
||||
- 无目录 / 无 cookies 文件 → no_session(从未登录过)
|
||||
- 有 cookies 但无 Pinterest 域 cookie → logged_out(未登录过 Pinterest)
|
||||
- 有 Pinterest 域 cookie 但缺关键登录 cookie(_auth/csrftoken)或已过期 → logged_out
|
||||
- 含 _auth 且未过期 → logged_in
|
||||
- 数据库无法解析 → unknown(调用方走运行时检测兜底)
|
||||
|
||||
注意:Chrome 在 Windows 上对 cookie 值做 DPAPI 加密,这里只判断
|
||||
cookie 的存在性 / 域名 / 过期时间,不解析密文值。
|
||||
"""
|
||||
ud = _session_user_data_dir()
|
||||
out: Dict[str, Any] = {
|
||||
"status": "unknown",
|
||||
"session_dir": ud,
|
||||
"cookies_file": "",
|
||||
"pinterest_cookies": 0,
|
||||
"has_auth_cookie": False,
|
||||
"expired": False,
|
||||
"detail": "",
|
||||
}
|
||||
if not os.path.isdir(ud):
|
||||
out["status"] = "no_session"
|
||||
out["detail"] = "未找到 Chrome 登录态目录(从未登录过 Pinterest)"
|
||||
return out
|
||||
cookies_file = None
|
||||
for rel in ("Default/Network/Cookies", "Default/Cookies"):
|
||||
cand = os.path.join(ud, rel)
|
||||
if os.path.isfile(cand) and os.path.getsize(cand) > 0:
|
||||
cookies_file = cand
|
||||
break
|
||||
if not cookies_file:
|
||||
out["status"] = "no_session"
|
||||
out["detail"] = "未找到 cookies 数据库(从未登录过 Pinterest)"
|
||||
return out
|
||||
out["cookies_file"] = cookies_file
|
||||
try:
|
||||
import sqlite3
|
||||
conn = sqlite3.connect(f"file:{cookies_file}?mode=ro&immutable=1", uri=True)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT host_key, name, expires_utc FROM cookies "
|
||||
"WHERE host_key LIKE '%pinterest.com%' OR host_key LIKE '%pinimg.com%'"
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
out["detail"] = f"cookies 读取失败: {e}"
|
||||
return out
|
||||
out["pinterest_cookies"] = len(rows)
|
||||
# WebKit epoch(1601-01-01 UTC)微秒偏移;expires_utc=0 表示会话 cookie(不判过期)
|
||||
WEBKIT_EPOCH_US = 11644473600000000
|
||||
now_us = int(time.time() * 1_000_000)
|
||||
auth_names = ("_auth", "csrftoken", "sessionFunnelEventLogged")
|
||||
for host, name, exp in rows:
|
||||
if name in auth_names:
|
||||
out["has_auth_cookie"] = True
|
||||
if exp and exp != 0 and exp < now_us:
|
||||
out["expired"] = True
|
||||
if out["has_auth_cookie"] and not out["expired"]:
|
||||
out["status"] = "logged_in"
|
||||
out["detail"] = f"检测到 Pinterest 登录态({len(rows)} 个 Pinterest 域 cookie,含 _auth)"
|
||||
elif out["pinterest_cookies"] > 0:
|
||||
out["status"] = "logged_out"
|
||||
out["detail"] = (f"存在 {len(rows)} 个 Pinterest 域 cookie,但缺少有效登录 cookie"
|
||||
f"(可能已过期或退出登录)")
|
||||
else:
|
||||
out["status"] = "logged_out"
|
||||
out["detail"] = "cookies 数据库中无 Pinterest 域 cookie(未登录过 Pinterest)"
|
||||
return out
|
||||
|
||||
|
||||
async def download_image(session: aiohttp.ClientSession, url: str, idx: int,
|
||||
sem: asyncio.Semaphore, save_dir: str, proxy: str | None):
|
||||
filename = get_filename_from_url(url, idx)
|
||||
@@ -284,7 +360,7 @@ async def human_scroll(page: Page, distance: int):
|
||||
await page.wait_for_timeout(random.randint(30, 100)) # 滚轮事件间隔
|
||||
|
||||
|
||||
async def _ensure_logged_in(page: Page, search_url: str) -> bool:
|
||||
async def _ensure_logged_in(page: Page, search_url: str, login_wait: bool = False) -> bool:
|
||||
"""独立登录检查方法:在已跳到搜索页的前提下确认 Pinterest 已登录。
|
||||
|
||||
判定策略(避免误判):
|
||||
@@ -292,7 +368,9 @@ async def _ensure_logged_in(page: Page, search_url: str) -> bool:
|
||||
才认定未登录;否则默认已登录(不依赖可能不匹配的已登录选择器)。
|
||||
- Pinterest 是 SPA,goto 后需等待渲染,否则瞬间误判未登录。
|
||||
- 已登录 → 立即返回 True,走直路。
|
||||
- 未登录(被弹回登录墙)→ 回退首页提示手动登录一次,登录成功后返回 True。
|
||||
- 未登录:
|
||||
login_wait=True → 回退首页提示手动登录一次,登录成功后返回 True;
|
||||
login_wait=False → 不阻塞,直接返回 False(由调用方跳过并告警)。
|
||||
- 一直未登录(用户关窗口/放弃)→ 返回 False。
|
||||
"""
|
||||
async def _has_login_wall() -> bool:
|
||||
@@ -327,6 +405,13 @@ async def _ensure_logged_in(page: Page, search_url: str) -> bool:
|
||||
print("✅ 已检测到登录态,直接开始搜索")
|
||||
return True
|
||||
|
||||
# 未登录:默认不阻塞(跳过并告警);login_wait=True 才回退首页等手动登录
|
||||
if not login_wait:
|
||||
print("⚠️ 未检测到 Pinterest 登录态(cookie 缺失或已过期)。"
|
||||
"请先在 pinterest_scraper/.chrome_session 目录登录 Pinterest 一次,"
|
||||
"或设置 pinterest.login_wait=true 让程序自动等待手动登录。")
|
||||
return False
|
||||
|
||||
# 未登录(搜索页被弹回登录墙):回退首页,提示手动登录一次,成功后立即继续
|
||||
print("⚠️ 当前未登录(搜索页被拦截)。请在弹出的浏览器窗口中手动登录,登录成功后将自动继续……")
|
||||
try:
|
||||
@@ -351,7 +436,8 @@ async def _ensure_logged_in(page: Page, search_url: str) -> bool:
|
||||
|
||||
async def scrape(keyword: str, count: int, headless: bool = False,
|
||||
proxy: str | None = None,
|
||||
cdp_url: str | None = None) -> set[str]:
|
||||
cdp_url: str | None = None,
|
||||
login_wait: bool = False) -> set[str]:
|
||||
imgs_url: set[str] = set()
|
||||
async with async_playwright() as p:
|
||||
if cdp_url:
|
||||
@@ -389,7 +475,7 @@ async def scrape(keyword: str, count: int, headless: bool = False,
|
||||
await page.goto(search_url, wait_until="domcontentloaded")
|
||||
|
||||
# 独立登录检查:已登录直接开始;未登录才回退首页等手动登录
|
||||
logged_in = await _ensure_logged_in(page, search_url)
|
||||
logged_in = await _ensure_logged_in(page, search_url, login_wait=login_wait)
|
||||
if not logged_in:
|
||||
# 关闭浏览器(持久化目录已保存任何已有状态),中止本次爬取
|
||||
if cdp_url:
|
||||
|
||||
Reference in New Issue
Block a user