v92-v105 多国适配 + 模板导出增强 + 生图可靠性优化
- 新增韩国(KR)适配:countries/pinterest 种子词、K-pop 提示词、UI 国家列表 - 爬虫跨运行持久化已采集 URL(.collected_urls.json),同关键词重复搜索采新图 - 模板导出增强:SPU 商品属性列名前缀剥离匹配、尺码下拉框 INDIRECT 动态引用、 SPU 字段映射(袖长/门襟/胸垫等)、季节/印花图案女装映射、建议售价统一必填、 SKU 分类/数量/单位直接填默认值 - 自定义模式:多模态提示词独立(custom_analyze_*)、生图提示词模板可配置 - Pinterest:空选品守卫、连续空分析保护(max_empty_analyze)、flat_prompt 配置覆盖 - 生图可靠性:开始/完成进度日志、异步任务轮询超时 60s→300s
This commit is contained in:
@@ -3,11 +3,15 @@
|
||||
用法:
|
||||
from pinterest_scraper.scraper import scrape_pinterest
|
||||
files = scrape_pinterest("vintage 70s", count=40, save_dir="output/pinterest_ref/US/vintage_70s")
|
||||
|
||||
增量续采:每个 save_dir 下维护 .collected_urls.json(该关键词历史已采集 URL 集合)。
|
||||
重复搜索同一关键词时自动跳过旧图、往下拉采集新图,避免反复下载顶部重复图片。
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Set
|
||||
|
||||
from pinterest_scraper.pinterest_image_capture import (
|
||||
detect_proxy,
|
||||
@@ -17,6 +21,27 @@ from pinterest_scraper.pinterest_image_capture import (
|
||||
)
|
||||
|
||||
|
||||
def _load_collected_urls(save_dir: str) -> Set[str]:
|
||||
"""读该关键词历史已采集 URL(.collected_urls.json),不存在/损坏返回空集。"""
|
||||
p = Path(save_dir) / ".collected_urls.json"
|
||||
if not p.exists():
|
||||
return set()
|
||||
try:
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
return {str(u) for u in data if str(u).strip()}
|
||||
except Exception: # noqa: BLE001
|
||||
return set()
|
||||
|
||||
|
||||
def _save_collected_urls(save_dir: str, urls: Set[str]) -> None:
|
||||
"""把已采集 URL 集合持久化到 .collected_urls.json(供下次增量续采)。"""
|
||||
try:
|
||||
p = Path(save_dir) / ".collected_urls.json"
|
||||
p.write_text(json.dumps(sorted(urls), ensure_ascii=False), encoding="utf-8")
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[pinterest] 已采集 URL 持久化失败: {e}")
|
||||
|
||||
|
||||
def scrape_pinterest(keyword: str, count: int = 40, save_dir: Optional[str] = None,
|
||||
proxy: Optional[str] = None, headless: bool = False,
|
||||
cdp_url: Optional[str] = None,
|
||||
@@ -27,6 +52,8 @@ def scrape_pinterest(keyword: str, count: int = 40, save_dir: Optional[str] = No
|
||||
- login_wait=False(默认):运行时检测到未登录时不阻塞,直接返回空列表(由调用方跳过并告警);
|
||||
login_wait=True:弹出 Chrome 窗口等待手动登录一次,登录态缓存在
|
||||
pinterest_scraper/.chrome_session,之后自动复用。
|
||||
- 增量续采:读取 save_dir/.collected_urls.json 历史已采集 URL,滚动时跳过旧图往下拉,
|
||||
只下载新图,并把新 URL 合并写回(种子词用尽复用同一关键词时自动采集更靠后的新图)。
|
||||
"""
|
||||
# 代理解析:显式传入 > 自动探测 > 直连
|
||||
if proxy is None:
|
||||
@@ -41,12 +68,22 @@ def scrape_pinterest(keyword: str, count: int = 40, save_dir: Optional[str] = No
|
||||
os.makedirs(out, exist_ok=True)
|
||||
print(f"[pinterest] 关键词: {keyword} | 数量: {count} | 保存目录: {out}")
|
||||
|
||||
imgs_url = asyncio.run(scrape(keyword, count, headless, proxy, cdp_url, login_wait))
|
||||
# 跨运行已采集 URL:重复搜索同一关键词时跳过旧图、往下拉续采
|
||||
known = _load_collected_urls(out)
|
||||
if known:
|
||||
print(f"[pinterest] 该关键词已采集过 {len(known)} 个 URL,将跳过旧图往下拉续采新图")
|
||||
|
||||
imgs_url = asyncio.run(scrape(keyword, count, headless, proxy, cdp_url, login_wait,
|
||||
known_urls=known))
|
||||
if not imgs_url:
|
||||
print(f"[pinterest] 未收集到图片(可能未登录或搜索无结果): {keyword}")
|
||||
print(f"[pinterest] 未收集到新图片(可能未登录、搜索无结果或该关键词已无新图): {keyword}")
|
||||
return []
|
||||
asyncio.run(download_all(imgs_url, out, proxy))
|
||||
|
||||
# 合并本次新采集 URL 到持久化集合,供下次增量续采
|
||||
known |= imgs_url
|
||||
_save_collected_urls(out, known)
|
||||
|
||||
files = [str(p) for p in sorted(Path(out).iterdir()) if p.is_file()]
|
||||
print(f"[pinterest] 完成,共下载 {len(files)} 张图片: {keyword}")
|
||||
print(f"[pinterest] 完成,本次新增 {len(imgs_url)} 张图片(累计已采集 {len(known)} 个 URL): {keyword}")
|
||||
return files
|
||||
|
||||
Reference in New Issue
Block a user