Files
inkreach-official-website/apps/api/src/sync/sync.service.ts
T

822 lines
29 KiB
TypeScript

import {
BadRequestException,
Injectable,
Logger,
NotFoundException,
} from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { Prisma } from '@prisma/client';
import { PrismaService } from '../prisma/prisma.service';
import {
SdsClientService,
SdsCategoryTreeNode,
SdsProduct,
SdsProductDetail,
} from './sds-client.service';
import { normalizeProductDetail } from './sds-product-detail.mapper';
import { FamilyRecomputeService } from '../product-families/family-recompute.service';
import { originGroupKey, parseOriginName } from '../product-families/origin-name.parser';
export interface CategorySyncResult {
inserted: number;
updated: number;
total: number;
deletedStale: number;
}
export interface ProductSyncResult {
inserted: number;
updated: number;
total: number;
leafCategories: number;
delisted: number;
}
/**
* Safety guards so a degenerate/partial SDS response never triggers a
* destructive operation (stale category deletion / mass delist marking).
* Upstream normally returns ~226 categories and ~150 leaf categories with
* hundreds of products — the floors below only trigger on abnormal responses.
*/
export const SYNC_GUARDS = {
MIN_CATEGORY_COUNT: 10,
MIN_CATEGORY_RATIO: 0.5,
MIN_LEAF_CATEGORIES: 10,
MIN_SEEN_GOODS: 50,
} as const;
/**
* True when the fetched category count is suspiciously small compared to the
* categories already synced from SDS, i.e. the upstream response is likely
* partial/degenerate. In that case stale deletion must be skipped.
*/
export function shouldSkipStaleDeletion(fetched: number, existingSds: number): boolean {
if (existingSds <= 0) return false;
return (
fetched < SYNC_GUARDS.MIN_CATEGORY_COUNT ||
fetched < SYNC_GUARDS.MIN_CATEGORY_RATIO * existingSds
);
}
/**
* True only when both the leaf-category count and the number of seen products
* are healthy enough to trust the "not seen upstream => delisted" conclusion.
*/
export function shouldRunDelistDetection(leafCategories: number, seenGoods: number): boolean {
return (
leafCategories >= SYNC_GUARDS.MIN_LEAF_CATEGORIES &&
seenGoods >= SYNC_GUARDS.MIN_SEEN_GOODS
);
}
@Injectable()
export class SyncService {
private readonly logger = new Logger(SyncService.name);
private running = { categories: false, products: false, details: false };
constructor(
private readonly prisma: PrismaService,
private readonly sds: SdsClientService,
private readonly familyRecompute: FamilyRecomputeService,
) {}
/**
* Hourly full sync — runs `syncCategories` first (since product
* sync depends on knowing which leaf categories exist) and then
* `syncProducts`.
*/
@Cron(CronExpression.EVERY_HOUR)
async hourlyCron(): Promise<void> {
try {
await this.syncCategories();
await this.syncProducts();
} catch (err) {
this.logger.error('Hourly cron sync failed', err as Error);
}
}
/** Fire-and-forget wrappers for manual triggers via HTTP. */
async startCategorySync(): Promise<{ message: string }> {
if (this.running.categories) {
return { message: 'Category sync already in progress' };
}
void this.syncCategories().catch((err) =>
this.logger.error('Category sync failed', err as Error),
);
return { message: 'Category sync started' };
}
async startProductSync(): Promise<{ message: string }> {
if (this.running.products) {
return { message: 'Product sync already in progress' };
}
void this.syncProducts().catch((err) =>
this.logger.error('Product sync failed', err as Error),
);
return { message: 'Product sync started' };
}
/** Refresh all active SDS product details once per day at 03:30. */
@Cron('0 30 3 * * *', { timeZone: 'Asia/Shanghai' })
async dailyProductDetailCron(): Promise<void> {
try {
await this.syncProductDetails();
} catch (err) {
this.logger.error('Daily product detail sync failed', err as Error);
}
}
async startProductDetailSync(): Promise<{ message: string }> {
if (this.running.details) {
return { message: 'Product detail sync already in progress' };
}
void this.syncProductDetails().catch((err) =>
this.logger.error('Product detail sync failed', err as Error),
);
return { message: 'Product detail sync started' };
}
/** Check if a sync type is currently running. */
isRunning(type: 'categories' | 'products' | 'details'): boolean {
return this.running[type];
}
async syncCategories(): Promise<CategorySyncResult> {
if (this.running.categories) {
throw new Error('Category sync already in progress');
}
this.running.categories = true;
const log = await this.prisma.syncLog.create({
data: { type: 'CATEGORIES', status: 'RUNNING' },
});
try {
const tree = await this.sds.fetchCategoryTree();
const flat = this.flattenCategoryTree(tree);
this.logger.log(`Fetched ${flat.length} SDS categories`);
const seenSdsIds = new Set(flat.map((n) => n.sdsId));
// Guard: if the upstream tree is suspiciously small vs what we already
// have from SDS, skip stale deletion entirely — a partial response must
// never wipe the category library.
const existingSdsCount = await this.prisma.category.count({
where: { sdsCategoryId: { not: null } },
});
const skipStaleDeletion = shouldSkipStaleDeletion(flat.length, existingSdsCount);
if (skipStaleDeletion) {
this.logger.warn(
`Skipping stale category deletion: fetched=${flat.length} existingSds=${existingSdsCount} ` +
`(below guard thresholds)`,
);
}
// Single transaction: upsert + wire parents + delete stale.
// SDS tree is the source of truth — anything not in the response gets deleted
// (unless the response looks degenerate, see guard above).
const { inserted, updated, deletedStale } = await this.prisma.$transaction(async (tx) => {
let ins = 0;
let upd = 0;
// 1. Upsert all SDS categories
for (const node of flat) {
const existing = await tx.category.findUnique({
where: { sdsCategoryId: node.sdsId },
});
if (!existing) {
await tx.category.create({
data: {
sdsCategoryId: node.sdsId,
categoryName: node.name,
categoryIcon: node.icon ?? null,
},
});
ins++;
} else {
await tx.category.update({
where: { id: existing.id },
data: {
categoryName: node.name,
categoryIcon: node.icon ?? null,
},
});
upd++;
}
}
// 2. Wire parent-child relationships
for (const node of flat) {
if (!node.parentSdsId) continue;
const child = await tx.category.findUnique({
where: { sdsCategoryId: node.sdsId },
});
const parent = await tx.category.findUnique({
where: { sdsCategoryId: node.parentSdsId },
});
if (child && parent && child.parentCategoryId !== parent.id) {
await tx.category.update({
where: { id: child.id },
data: { parentCategoryId: parent.id },
});
}
}
// 3. Delete stale categories (in DB but not in SDS response)
// Detach parent links first, then delete leaf-first to respect FK constraints.
// Skipped entirely when the response looks degenerate (see guard above).
let deletedStale = 0;
if (!skipStaleDeletion) {
const staleCats = await tx.category.findMany({
where: { sdsCategoryId: { notIn: [...seenSdsIds] } },
select: { id: true },
});
const staleIds = staleCats.map((c) => c.id);
// Protect categories that have configured goods — onDelete: Restrict
const goodsInStale = await tx.good.groupBy({
by: ['categoryId'],
where: { categoryId: { in: staleIds } },
});
const protectedIds = new Set(goodsInStale.map((g) => g.categoryId));
const deletableIds = staleIds.filter((id) => !protectedIds.has(id));
deletedStale = deletableIds.length;
// Detach all deletable categories from their parents
if (deletableIds.length > 0) {
await tx.category.updateMany({
where: { id: { in: deletableIds } },
data: { parentCategoryId: null },
});
// Also detach any non-deletable children pointing to deletable parents
await tx.category.updateMany({
where: { parentCategoryId: { in: deletableIds } },
data: { parentCategoryId: null },
});
// Delete leaf-first (repeatedly remove nodes with no children)
let remaining = [...deletableIds];
while (remaining.length > 0) {
const withChildren = await tx.category.findMany({
where: { parentCategoryId: { in: remaining } },
select: { parentCategoryId: true },
distinct: ['parentCategoryId'],
});
const hasChildSet = new Set(
withChildren.filter((c) => c.parentCategoryId).map((c) => c.parentCategoryId!.toString()),
);
const leaves = remaining.filter((id) => !hasChildSet.has(id.toString()));
if (leaves.length === 0) break; // safety: circular dependency
await tx.category.deleteMany({ where: { id: { in: leaves } } });
remaining = remaining.filter((id) => !leaves.some((l) => l === id));
}
}
}
return { inserted: ins, updated: upd, deletedStale };
});
await this.prisma.syncLog.update({
where: { id: log.id },
data: {
status: 'SUCCESS',
finishedAt: new Date(),
message: `inserted=${inserted} updated=${updated} total=${flat.length} staleDeleted=${deletedStale}`,
},
});
return { inserted, updated, total: flat.length, deletedStale };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
await this.prisma.syncLog.update({
where: { id: log.id },
data: {
status: 'FAILED',
finishedAt: new Date(),
message,
},
});
throw err;
} finally {
this.running.categories = false;
}
}
async syncProducts(): Promise<ProductSyncResult> {
if (this.running.products) {
throw new Error('Product sync already in progress');
}
this.running.products = true;
const log = await this.prisma.syncLog.create({
data: { type: 'PRODUCTS', status: 'RUNNING' },
});
try {
// Identify leaf categories — those with no children.
const all = await this.prisma.category.findMany({
select: { id: true, sdsCategoryId: true },
});
const parents = await this.prisma.category.findMany({
where: { parent: { isNot: null } },
select: { parentCategoryId: true },
});
const parentIds = new Set(parents.map((p) => p.parentCategoryId!));
const leafRows = all.filter((c) => !parentIds.has(c.id) && c.sdsCategoryId);
let inserted = 0;
let updated = 0;
let total = 0;
const seenSdsGoodIds = new Set<string>();
for (const leaf of leafRows) {
const sdsCategoryId = leaf.sdsCategoryId!;
let page = 1;
// eslint-disable-next-line no-constant-condition
while (true) {
const resp = await this.sds.fetchProductsPage(sdsCategoryId, page, 50);
const products = resp.items ?? resp.content ?? [];
if (products.length === 0) break;
for (const product of products) {
seenSdsGoodIds.add(String(product.id));
const upserted = await this.upsertOriginGood(product, sdsCategoryId);
if (upserted === 'inserted') inserted++;
else updated++;
total++;
}
if (products.length < 50) break;
page++;
if (page > 200) {
// Safety net — at most 10k products per category.
this.logger.warn(`Reached 200-page safety cap for ${sdsCategoryId}`);
break;
}
}
}
// Detect delisted products: mark origin goods not seen in upstream as delisted,
// and re-activate any previously delisted goods that reappeared.
// Guard: only trust this conclusion when the sync covered a healthy number of
// leaf categories and saw a healthy number of products — otherwise a partial
// sync must never mass-delist the product library.
let delistedCount = 0;
let reactivatedCount = 0;
const runDelist = shouldRunDelistDetection(leafRows.length, seenSdsGoodIds.size);
if (runDelist) {
const delistedResult = await this.prisma.originGood.updateMany({
where: {
source: 'SDS',
sdsGoodId: { notIn: [...seenSdsGoodIds] },
delisted: false,
},
data: { delisted: true },
});
const reactivatedResult = await this.prisma.originGood.updateMany({
where: {
source: 'SDS',
sdsGoodId: { in: [...seenSdsGoodIds] },
delisted: true,
},
data: { delisted: false },
});
delistedCount = delistedResult.count;
reactivatedCount = reactivatedResult.count;
} else {
this.logger.warn(
`Skipping delist detection: leafCategories=${leafRows.length} seenGoods=${seenSdsGoodIds.size} ` +
`(below guard thresholds)`,
);
}
await this.prisma.syncLog.update({
where: { id: log.id },
data: {
status: 'SUCCESS',
finishedAt: new Date(),
message: `inserted=${inserted} updated=${updated} total=${total} delisted=${delistedCount} reactivated=${reactivatedCount} leafCategories=${leafRows.length}`,
},
});
return {
inserted,
updated,
total,
leafCategories: leafRows.length,
delisted: delistedCount,
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
await this.prisma.syncLog.update({
where: { id: log.id },
data: {
status: 'FAILED',
finishedAt: new Date(),
message,
},
});
throw err;
} finally {
this.running.products = false;
}
}
async getStatus(limit = 20) {
return this.prisma.syncLog.findMany({
orderBy: { startedAt: 'desc' },
take: limit,
});
}
async syncProductDetails(): Promise<{
total: number;
synced: number;
failed: number;
}> {
if (this.running.details) {
throw new Error('Product detail sync already in progress');
}
this.running.details = true;
const log = await this.prisma.syncLog.create({
data: { type: 'PRODUCT_DETAILS', status: 'RUNNING' },
});
try {
const result = await this.syncAllProductDetails(async (progress) => {
await this.prisma.syncLog.update({
where: { id: log.id },
data: {
message: `processed=${progress.processed}/${progress.total} synced=${progress.synced} failed=${progress.failed}`,
},
});
});
await this.prisma.syncLog.update({
where: { id: log.id },
data: {
status: 'SUCCESS',
finishedAt: new Date(),
message: `total=${result.total} synced=${result.synced} failed=${result.failed}`,
},
});
return result;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
await this.prisma.syncLog.update({
where: { id: log.id },
data: { status: 'FAILED', finishedAt: new Date(), message },
});
throw error;
} finally {
this.running.details = false;
}
}
async syncOneProductDetail(goodId: string): Promise<{
goodId: string;
variants: number;
detailSyncedAt: string;
}> {
const originGood = await this.prisma.originGood.findUnique({
where: { sdsGoodId: goodId },
select: { id: true, source: true },
});
if (!originGood) {
throw new NotFoundException(`SDS product ${goodId} not found locally`);
}
if (originGood.source !== 'SDS') {
throw new BadRequestException('自定义商品不支持从 SDS 同步详情');
}
const upstream = await this.sds.fetchProductDetail(goodId);
const normalized = normalizeProductDetail(upstream);
await this.persistProductDetail(originGood.id, upstream);
return {
goodId,
variants: normalized.variants.length,
detailSyncedAt: new Date().toISOString(),
};
}
queueProductDetailSync(goodId: string): void {
void this.syncOneProductDetail(goodId).catch((error) => {
const message = error instanceof Error ? error.message : String(error);
this.logger.warn(`Queued detail sync failed for ${goodId}: ${message}`);
});
}
async syncConfiguredProductDetails(): Promise<{ synced: number; failed: number }> {
const result = await this.syncMatchingProductDetails({
delisted: false,
source: 'SDS',
goods: { some: {} },
});
return { synced: result.synced, failed: result.failed };
}
async syncAllProductDetails(
onProgress?: (progress: {
processed: number;
total: number;
synced: number;
failed: number;
}) => Promise<void>,
): Promise<{ total: number; synced: number; failed: number }> {
return this.syncMatchingProductDetails(
{ delisted: false, source: 'SDS' },
onProgress,
2,
);
}
private async syncMatchingProductDetails(
where: Prisma.OriginGoodWhereInput,
onProgress?: (progress: {
processed: number;
total: number;
synced: number;
failed: number;
}) => Promise<void>,
attempts = 1,
): Promise<{ total: number; synced: number; failed: number }> {
const originGoods = await this.prisma.originGood.findMany({
where,
select: { id: true, sdsGoodId: true },
orderBy: { id: 'asc' },
});
let synced = 0;
let failed = 0;
let processed = 0;
for (const originGood of originGoods) {
let lastError: unknown;
let succeeded = false;
for (let attempt = 1; attempt <= attempts; attempt++) {
try {
const upstream = await this.sds.fetchProductDetail(originGood.sdsGoodId);
await this.persistProductDetail(originGood.id, upstream);
synced++;
succeeded = true;
break;
} catch (error) {
lastError = error;
}
}
if (!succeeded) {
failed++;
const message = lastError instanceof Error ? lastError.message : String(lastError);
this.logger.warn(`Failed to sync SDS detail ${originGood.sdsGoodId}: ${message}`);
}
processed++;
if (onProgress && (processed % 10 === 0 || processed === originGoods.length)) {
await onProgress({ processed, total: originGoods.length, synced, failed });
}
}
return { total: originGoods.length, synced, failed };
}
async importProductDetail(upstream: SdsProductDetail): Promise<{
goodId: string;
variants: number;
sizeRows: number;
packageRows: number;
configuredGoods: number;
}> {
const goodId = String(upstream.id);
const normalized = normalizeProductDetail(upstream);
const originGood = await this.prisma.originGood.upsert({
where: { sdsGoodId: goodId },
create: {
sdsGoodId: goodId,
goodName: String(upstream.name ?? goodId),
goodImage: String(upstream.psd_img_url ?? upstream.img_url ?? upstream.blankDesignUrl ?? '') || null,
goodPrice:
upstream.min_price === undefined || upstream.min_price === null
? null
: new Prisma.Decimal(Number(upstream.min_price)),
},
update: {
goodName: upstream.name ? String(upstream.name) : undefined,
goodImage: String(upstream.psd_img_url ?? upstream.img_url ?? upstream.blankDesignUrl ?? '') || undefined,
goodPrice:
upstream.min_price === undefined || upstream.min_price === null
? undefined
: new Prisma.Decimal(Number(upstream.min_price)),
},
});
await this.persistProductDetail(originGood.id, upstream);
const configuredGoods = await this.prisma.good.count({
where: { originGoodId: originGood.id },
});
const sizeChart = normalized.sizeChart as { rows?: unknown[] } | null;
const packageSpecs = normalized.packageSpecs as { rows?: unknown[] } | null;
return {
goodId,
variants: normalized.variants.length,
sizeRows: sizeChart?.rows?.length ?? 0,
packageRows: packageSpecs?.rows?.length ?? 0,
configuredGoods,
};
}
private async persistProductDetail(originGoodId: bigint, upstream: SdsProductDetail): Promise<void> {
const normalized = normalizeProductDetail(upstream);
const { variants, ...detail } = normalized;
await this.prisma.$transaction(async (tx) => {
const json = (value: Prisma.InputJsonValue | null) => value ?? Prisma.DbNull;
// Backfill the origin good's price from upstream min_price when present
if (upstream.min_price !== undefined && upstream.min_price !== null) {
await tx.originGood.update({
where: { id: originGoodId },
data: { goodPrice: new Prisma.Decimal(Number(upstream.min_price)) },
});
}
await tx.originGoodDetail.upsert({
where: { originGoodId },
create: {
originGoodId,
...detail,
sizeChart: json(detail.sizeChart),
packageSpecs: json(detail.packageSpecs),
options: json(detail.options),
media: json(detail.media),
},
update: {
...detail,
sizeChart: json(detail.sizeChart),
packageSpecs: json(detail.packageSpecs),
options: json(detail.options),
media: json(detail.media),
syncedAt: new Date(),
},
});
const seenVariantIds: string[] = [];
for (const variant of variants) {
seenVariantIds.push(variant.sdsVariantId);
const { designData, ...data } = variant;
await tx.originGoodVariant.upsert({
where: {
originGoodId_sdsVariantId: {
originGoodId,
sdsVariantId: variant.sdsVariantId,
},
},
create: { originGoodId, ...data, designData: json(designData) },
update: { ...data, designData: json(designData) },
});
}
await tx.originGoodVariant.deleteMany({
where: {
originGoodId,
...(seenVariantIds.length ? { sdsVariantId: { notIn: seenVariantIds } } : {}),
},
});
});
// 族成员的详情/变体变化 → 异步重算该族(进程内去重)
await this.maybeEnqueueFamilyRecompute(originGoodId);
}
/** 详情同步后的族重算钩子:链接有族归属才入队 */
private async maybeEnqueueFamilyRecompute(originGoodId: bigint): Promise<void> {
const og = await this.prisma.originGood.findUnique({
where: { id: originGoodId },
select: { familyId: true },
});
if (og?.familyId) this.familyRecompute.enqueue(og.familyId);
}
/**
* 新链接自动挂族:优先按 SDS 分类(=产品模型)匹配已有族的成员;
* 无分类时回退名称 3 段键。恰好命中唯一族才挂载(多族/零族留给管理员裁决)。
* 锁定族(autoManaged=false)不吸收新成员,只置 stale 提示。
*/
private async tryAutoAttachToFamily(originGoodId: bigint, goodName: string): Promise<void> {
const self = await this.prisma.originGood.findUnique({
where: { id: originGoodId },
select: { sdsCategoryId: true },
});
let familyIds: Set<string>;
if (self?.sdsCategoryId) {
const siblings = await this.prisma.originGood.findMany({
where: { sdsCategoryId: self.sdsCategoryId, familyId: { not: null } },
select: { familyId: true },
distinct: ['familyId'],
});
familyIds = new Set(siblings.map((s) => s.familyId!.toString()));
} else {
const key = originGroupKey(goodName);
if (!key) return;
const candidates = await this.prisma.originGood.findMany({
where: { familyId: { not: null }, goodName: { startsWith: key } },
select: { familyId: true, goodName: true },
});
familyIds = new Set(
candidates
.filter((c) => originGroupKey(c.goodName) === key && c.familyId !== null)
.map((c) => c.familyId!.toString()),
);
}
if (familyIds.size !== 1) return;
const familyId = BigInt([...familyIds][0]);
const family = await this.prisma.productFamily.findUnique({
where: { id: familyId },
select: { autoManaged: true },
});
if (!family) return;
if (family.autoManaged) {
await this.prisma.originGood.update({
where: { id: originGoodId },
data: { familyId },
});
this.familyRecompute.enqueue(familyId);
} else {
await this.prisma.productFamily.update({
where: { id: familyId },
data: { stale: true },
});
}
}
/**
* Flattens the SDS nested tree into a list of `{ sdsId, parentSdsId?, name, icon? }`.
*/
flattenCategoryTree(
nodes: SdsCategoryTreeNode[],
parentSdsId?: string,
): Array<{ sdsId: string; parentSdsId?: string; name: string; icon?: string }> {
const out: Array<{ sdsId: string; parentSdsId?: string; name: string; icon?: string }> = [];
const walk = (node: SdsCategoryTreeNode, parent?: string) => {
const sdsId = String(node.id);
if (sdsId === '' || sdsId === 'undefined' || sdsId === 'null') return;
out.push({
sdsId,
parentSdsId: parent,
name: String(node.name ?? node.title ?? sdsId),
icon: node.icon ? String(node.icon) : undefined,
});
if (Array.isArray(node.children)) {
for (const child of node.children) walk(child, sdsId);
}
};
for (const root of nodes) walk(root, parentSdsId);
return out;
}
private async upsertOriginGood(
product: SdsProduct,
sdsCategoryId: string,
): Promise<'inserted' | 'updated'> {
const sdsGoodId = String(product.id);
const existing = await this.prisma.originGood.findUnique({
where: { sdsGoodId },
});
const goodName = String(product.name ?? product.title ?? sdsGoodId);
const goodImage = product.psd_img_url
? String(product.psd_img_url)
: product.blankDesignUrl
? String(product.blankDesignUrl)
: product.thumbImgUrl
? String(product.thumbImgUrl)
: product.show_img
? String(product.show_img)
: product.img_url
? String(product.img_url)
: product.pic
? String(product.pic)
: product.image
? String(product.image)
: null;
const priceValue = product.currentPrice ?? product.price;
let goodPrice: Prisma.Decimal | null = null;
if (priceValue !== undefined && priceValue !== null) {
const n = typeof priceValue === 'string' ? Number(priceValue) : priceValue;
if (Number.isFinite(n)) {
goodPrice = new Prisma.Decimal(n);
}
}
// 链接名结构化解析列(镜像纯度:全量覆盖,含空值)
const parsed = parseOriginName(goodName);
const parsedData = {
skuCode: parsed.skuCode,
logisticsLabel: parsed.logisticsLabel,
craftLabel: parsed.craftLabel,
warehouseLabel: parsed.warehouseLabel,
};
const data: Prisma.OriginGoodUncheckedUpdateInput = {
sdsCategoryId,
goodName,
goodImage,
goodPrice,
...parsedData,
};
if (!existing) {
const created = await this.prisma.originGood.create({
data: {
sdsGoodId,
sdsCategoryId,
goodName,
goodImage,
goodPrice,
...parsedData,
},
select: { id: true },
});
await this.tryAutoAttachToFamily(created.id, goodName);
return 'inserted';
}
await this.prisma.originGood.update({
where: { id: existing.id },
data,
});
return 'updated';
}
}