fix(receiving): 收货未入库提醒可靠性增强(Redis 抢锁判定修正 + app 级兜底锁、逐货主群异常隔离、失败回退客服群、pushSystemMessage 幂等去重)

This commit is contained in:
liufumingren 2026-09-24 03:26:18 +08:00
parent 52181c44bd
commit 305d27bc5f
4 changed files with 238 additions and 54 deletions

View File

@ -42,7 +42,7 @@ module.exports = {
ctx.logger.info(
`[receiving-pending-notice] 本轮提醒完成:开启仓库 ${summary.checkedWarehouseCount},`
+ `执行 ${summary.executedWarehouseCount},未入库 ${summary.pendingCount} 单,`
+ `发送 ${summary.sentCount},失败 ${summary.failedCount}`
+ `发送 ${summary.sentCount},失败 ${summary.failedCount},回退客服群 ${summary.fallbackCount || 0}`
);
}
} catch (error) {

View File

@ -239,9 +239,24 @@ function formatDateTime(ctx, value) {
* 通知时间(notice_time)与超时区间(min_hours / max_hours)均可在仓库设置中配置。
*/
class ReceivingPendingNoticeService extends MainService {
constructor(ctx) {
super(ctx);
this.memoryDailyLocks = new Set();
/**
* 获取进程内当日锁集合(挂在 app 上,跨 ctx/service 实例共享)。
*
* 定时任务每轮都会创建新的 service 实例,实例级 Set 无法阻挡后续轮次再次执行,
* 因此进程内降级锁必须使用 app 级共享集合(仅用于 Redis 命令失败/不可用时的降级场景)。
*
* @return {Set<string>} 进程内当日锁 key 集合
*/
getMemoryDailyLocks() {
const { app } = this;
if (!app) {
return new Set();
}
if (!(app._receivingPendingDailyLocks instanceof Set)) {
app._receivingPendingDailyLocks = new Set();
}
return app._receivingPendingDailyLocks;
}
/**
@ -509,6 +524,9 @@ class ReceivingPendingNoticeService extends MainService {
/**
* 发送单个货主批次(或未绑定货主批次)的未入库提醒。
*
* 货主批次主推其收货群;主推失败(含异常)时自动回退到仓库客服群
* (见 sendCustomerServiceFallback),保证个别群发送失败不影响其他货主群的通知。
*
* @param {Object} options 发送参数
* @param {number} options.warehouseId 仓库 ID
* @param {string} options.warehouseName 仓库名称
@ -517,11 +535,12 @@ class ReceivingPendingNoticeService extends MainService {
* @param {Array<Object>} options.rows 该货主的收货单列表
* @param {Object} options.config 规范化后的提醒配置
* @param {Date} options.now 当前时刻
* @return {Promise<Object|null>} 通知结果
* @return {Promise<Object|null>} 通知结果(主推结果 + 可选 fallback 回退结果)
*/
async sendPendingNotice({ warehouseId, warehouseName, storeId, storeName, rows, config, now }) {
const { ctx, app } = this;
const sendMethod = this.resolveNoticeSendMethod();
const hasStore = Number(storeId) > 0;
if (typeof sendMethod !== 'function') {
ctx.logger.warn('[receiving-pending-notice] 通知服务不可用,跳过发送');
return null;
@ -529,7 +548,6 @@ class ReceivingPendingNoticeService extends MainService {
const warehouseFrontendUrl = String(app.config.frontendUrl || '').trim();
const storeFrontendUrl = String(app.config.storeFrontendUrl || app.config.frontendUrl || '').trim();
const hasStore = Number(storeId) > 0;
const content = this.buildContent({
rows,
config,
@ -557,44 +575,150 @@ class ReceivingPendingNoticeService extends MainService {
})
: undefined;
return await sendMethod({
warehouseId,
// 未绑定货主时不带 storeId,由 notice 固定订阅回退到仓库客服群
storeId: hasStore ? Number(storeId) : undefined,
messageType: 'text',
businessName: 'receiving',
event: NOTICE_EVENT,
channel: 'warehouse',
title: `收货未入库提醒(${rows.length} 单)`,
content,
...(storeContent ? { storeContent } : {}),
// 幂等键:同仓库+同货主+同一天只落一条消息记录
businessId: `receiving-pending:${warehouseId}:${hasStore ? Number(storeId) : 0}:${buildDateKey(now)}`,
subject: {
id: warehouseId,
type: 'warehouse',
name: warehouseName,
},
sender: {
id: null,
name: '系统提醒',
},
metadata: {
// 主推发送:单批次异常只影响本批次(记录失败并尝试回退),不向上抛出中断其他货主群
let sendResult = null;
try {
sendResult = await sendMethod({
warehouseId,
// 未绑定货主时不带 storeId,由 notice 固定订阅回退到仓库客服群
storeId: hasStore ? Number(storeId) : undefined,
messageType: 'text',
businessName: 'receiving',
event: NOTICE_EVENT,
channel: 'warehouse',
title: `收货未入库提醒(${rows.length} 单)`,
content,
...(storeContent ? { storeContent } : {}),
// 幂等键:同仓库+同货主+同一天只落一条消息记录
businessId: `receiving-pending:${warehouseId}:${hasStore ? Number(storeId) : 0}:${buildDateKey(now)}`,
subject: {
id: warehouseId,
type: 'warehouse',
name: warehouseName,
},
sender: {
id: null,
name: '系统提醒',
},
metadata: {
warehouseId,
warehouseName,
storeId: hasStore ? Number(storeId) : null,
storeName: String(storeName || '').trim(),
minHours: config.minHours,
maxHours: config.maxHours,
receivingIds: rows.map(row => Number(row.id)).filter(id => id > 0),
receivingNumbers: rows.map(row => String(row.receiving_number || '')).filter(Boolean),
},
});
} catch (error) {
ctx.logger.error(`[receiving-pending-notice] 仓库 ${warehouseId} 货主 ${storeId} 批次发送异常: ${error.message}`);
sendResult = { success: false, message: `发送异常: ${error.message}` };
}
// 货主收货群主推失败(含异常)时回退仓库客服群,保证部分群失败不遗漏人工跟进
let fallbackResult = null;
if (hasStore && (!sendResult || sendResult.success !== true)) {
fallbackResult = await this.sendCustomerServiceFallback({
warehouseId,
warehouseName,
storeId: hasStore ? Number(storeId) : null,
storeName: String(storeName || '').trim(),
minHours: config.minHours,
maxHours: config.maxHours,
receivingIds: rows.map(row => Number(row.id)).filter(id => id > 0),
receivingNumbers: rows.map(row => String(row.receiving_number || '')).filter(Boolean),
},
storeId,
storeName,
rows,
config,
now,
});
}
return {
...(sendResult || { success: false, message: '通知发送无结果' }),
...(fallbackResult ? { fallback: fallbackResult } : {}),
};
}
/**
* 货主收货群主推失败时,回退通知仓库客服群人工跟进。
*
* 复用 notice 固定订阅回退链:不带 storeId 时 receiving_pending_inbound 的
* store:receiving 目标被跳过,fallbackTargets 命中 warehouse:customer_service。
* 幂等键使用独立前缀(receiving-pending-fallback),与主推送、其他货主的回退互不干扰。
*
* @param {Object} options 回退参数
* @param {number} options.warehouseId 仓库 ID
* @param {string} options.warehouseName 仓库名称
* @param {number} options.storeId 货主 ID
* @param {string} options.storeName 货主名称
* @param {Array<Object>} options.rows 该货主的收货单列表
* @param {Object} options.config 规范化后的提醒配置
* @param {Date} options.now 当前时刻
* @return {Promise<Object|null>} 回退通知结果(通知服务不可用时返回 null)
*/
async sendCustomerServiceFallback({ warehouseId, warehouseName, storeId, storeName, rows, config, now }) {
const { ctx, app } = this;
const sendMethod = this.resolveNoticeSendMethod();
if (typeof sendMethod !== 'function') {
return null;
}
const storeLabel = String(storeName || '').trim() || `货主${storeId}`;
const warehouseFrontendUrl = String(app.config.frontendUrl || '').trim();
const content = this.buildContent({
rows,
config,
warehouseName,
warehouseId,
storeId,
storeName,
forStore: false,
viewUrl: `${warehouseFrontendUrl}/storage/receiving`,
now,
ctx,
});
try {
return await sendMethod({
warehouseId,
// 不带 storeId:固定订阅 store:receiving 被跳过,回退链命中仓库客服群
storeId: undefined,
messageType: 'text',
businessName: 'receiving',
event: NOTICE_EVENT,
channel: 'warehouse',
title: `收货未入库提醒·货主群发送失败(${storeLabel})`,
content: `通知货主【${storeLabel}】收货群推送失败,请人工跟进:\n${content}`,
// 回退幂等键独立于主推送:同一货主同日只回退一条,且互不影响
businessId: `receiving-pending-fallback:${warehouseId}:${Number(storeId)}:${buildDateKey(now)}`,
subject: {
id: warehouseId,
type: 'warehouse',
name: warehouseName,
},
sender: {
id: null,
name: '系统提醒',
},
metadata: {
warehouseId,
warehouseName,
storeId: Number(storeId),
storeName: storeLabel,
fallbackReason: '货主收货群推送失败回退',
receivingIds: rows.map(row => Number(row.id)).filter(id => id > 0),
receivingNumbers: rows.map(row => String(row.receiving_number || '')).filter(Boolean),
},
});
} catch (error) {
ctx.logger.error(`[receiving-pending-notice] 仓库 ${warehouseId} 货主 ${storeId} 回退客服群发送异常: ${error.message}`);
return { success: false, message: `回退发送异常: ${error.message}` };
}
}
/**
* 扫描并通知单个仓库的超时未入库收货单。
*
* 各货主群独立发送:单个批次异常/失败只影响该批次(失败自动回退客服群),
* 不中断后续货主群的遍历与通知。
*
* @param {Object} options 执行参数
* @param {number} options.warehouseId 仓库 ID
* @param {string} options.warehouseName 仓库名称
@ -612,6 +736,7 @@ class ReceivingPendingNoticeService extends MainService {
groupCount: 0,
sentCount: 0,
failedCount: 0,
fallbackCount: 0,
};
if (rows.length === 0) {
@ -621,22 +746,41 @@ class ReceivingPendingNoticeService extends MainService {
const groups = this.groupByStore(rows);
for (const group of groups) {
const result = await this.sendPendingNotice({
warehouseId,
warehouseName,
storeId: group.storeId,
storeName: group.storeName,
rows: group.rows,
config,
now,
});
summary.groupCount += 1;
let result = null;
try {
result = await this.sendPendingNotice({
warehouseId,
warehouseName,
storeId: group.storeId,
storeName: group.storeName,
rows: group.rows,
config,
now,
});
} catch (error) {
// 单个货主批次异常不允许中断整仓遍历:该批次计入失败,继续通知其他货主群
ctx.logger.error(
`[receiving-pending-notice] 仓库 ${warehouseId} 货主 ${group.storeId} 批次异常,`
+ `跳过并继续后续货主: ${error.message}`
);
summary.failedCount += 1;
continue;
}
if (result && result.success) {
summary.sentCount += 1;
} else {
summary.failedCount += 1;
}
if (result && result.fallback && result.fallback.success) {
summary.fallbackCount += 1;
ctx.logger.warn(
`[receiving-pending-notice] 仓库 ${warehouseId} 货主 ${group.storeId} 收货群发送失败,已回退仓库客服群人工跟进`
);
}
}
ctx.logger.info(
@ -662,7 +806,11 @@ class ReceivingPendingNoticeService extends MainService {
}
/**
* 申请当日执行锁(Redis NX;Redis 不可用时降级为进程内集合)。
* 申请当日执行锁(Redis SET NX;Redis 命令失败/不可用时降级为 app 级进程内集合)。
*
* SET NX 语义:返回 'OK' 表示抢锁成功;返回 null 表示键已存在(当日锁已被本进程或
* 其他实例持有),必须直接判定为「未抢到锁」,不能落入进程内兜底——兜底仅用于 Redis
* 不可用的场景,否则每轮调度都会重新获得锁,导致宽限窗口内重复发送。
*
* @param {string} lockKey 锁 key
* @return {Promise<boolean>} 是否申请成功
@ -677,19 +825,19 @@ class ReceivingPendingNoticeService extends MainService {
if (result === 'OK') {
return true;
}
if (result !== null && result !== undefined) {
return false;
}
// null(键已存在)及其他异常返回值一律视为未抢到锁,禁止落入进程内兜底
return false;
} catch (error) {
ctx.logger.warn(`[receiving-pending-notice] 当日锁申请失败,降级进程内锁: ${error.message}`);
}
}
if (this.memoryDailyLocks.has(lockKey)) {
const memoryLocks = this.getMemoryDailyLocks();
if (memoryLocks.has(lockKey)) {
return false;
}
this.memoryDailyLocks.add(lockKey);
memoryLocks.add(lockKey);
return true;
}
@ -701,7 +849,7 @@ class ReceivingPendingNoticeService extends MainService {
*/
async releaseDailyLock(lockKey) {
const { ctx } = this;
this.memoryDailyLocks.delete(lockKey);
this.getMemoryDailyLocks().delete(lockKey);
const redisClient = this.getRedisClient();
if (redisClient && typeof redisClient.del === 'function') {
@ -739,6 +887,7 @@ class ReceivingPendingNoticeService extends MainService {
pendingCount: 0,
sentCount: 0,
failedCount: 0,
fallbackCount: 0,
results: [],
};
@ -764,6 +913,7 @@ class ReceivingPendingNoticeService extends MainService {
summary.pendingCount += result.pendingCount;
summary.sentCount += result.sentCount;
summary.failedCount += result.failedCount;
summary.fallbackCount += result.fallbackCount || 0;
summary.results.push(result);
} catch (error) {
// 执行失败释放锁,宽限窗口内的下一轮仍可重试

View File

@ -230,6 +230,9 @@ class WarehouseNotifyService extends Service {
* type / content / ext_id / group_id / group_name / user_id / timestamp。
* 推送前写入 wms_inbox_message(direction=outbound,source_type=system),
* 推送后按结果回写 push_status(2=已送达 / 3=失败)与 push_response。
* 幂等去重:同 ext_id 的消息已送达(push_status=2)时直接跳过推送,
* 避免定时任务宽限窗口重复执行/多实例并发导致同一群重复收到相同通知;
* 未送达(0/1/3)时复用同一消息记录重推。
*
* 向后兼容:不传 userIds 且传入 group 时,行为与旧版本完全一致
* (校验、幂等 ext_id、落库字段、推送体均不变)。
@ -306,11 +309,24 @@ class WarehouseNotifyService extends Service {
try {
const existing = await InboxMessage.findOne({
where: { ext_id: uniqueExtId },
attributes: [ 'id' ],
attributes: [ 'id', 'push_status' ],
paranoid: false,
});
if (existing) {
messageId = Number(existing.id);
// 幂等去重:同 ext_id 消息已送达(push_status=2)时跳过推送,
// 防止重复轮次/多实例并发导致同一群重复收到相同通知;
// 未送达(0/1/3)时保留重推机会(复用同一消息记录回写推送结果)
if (Number(existing.push_status) === 2) {
return {
success: true,
skipped: true,
reason: '消息已送达,跳过重复推送',
warehouseId,
method: 'notify.reply',
messageId,
};
}
} else {
const created = await InboxMessage.create({
ext_id: uniqueExtId,

View File

@ -1,4 +1,22 @@
[
{
"version": "2026.09.23-warehouse-receiving-pending-notice-reliability",
"date": "2026-09-23",
"type": "updated",
"summary": "收货未入库提醒(receiving_pending_inbound)可靠性增强:修复 10 分钟宽限窗口内每 5 分钟一轮导致的重复执行/重复通知(Redis SET NX 抢锁判定修正 + 进程内兜底锁改为 app 级共享集合);逐货主群异常隔离(单群发送异常不中断其他货主群);货主收货群推送失败自动回退仓库客服群人工跟进(独立幂等键与文案前缀);pushSystemMessage 推送幂等去重(同 ext_id 已送达跳过重推)",
"methods": [
"notify.reply"
],
"notes": [
"定时任务锁修复:app/service/systems/receiving_pending_notice.js#acquireDailyLock 中 Redis SET NX 返回 null(键已存在)直接判定未抢到锁,不再落入进程内兜底(兜底仅用于 Redis 不可用/命令异常);进程内兜底锁由 service 实例级改为 app._receivingPendingDailyLocks 共享集合(getMemoryDailyLocks),跨轮次/跨实例生效;app/schedule/notice_receiving_pending_inbound.js 汇总日志补充 fallbackCount。修复 10 分钟宽限窗口内每 5 分钟一轮导致的重复执行/重复通知",
"逐群异常隔离:notifyWarehouse 对每个货主批次 try/catch(sendPendingNotice 内部同样隔离主推异常),单群发送异常只计失败并继续后续货主群,确保所有货主群的收货群均被通知,部分群失败不遗漏其他群",
"失败自动回退客服群:新增 sendCustomerServiceFallback——货主收货群主推失败(含异常)时自动回退仓库客服群人工跟进,文案前缀「通知货主【X】收货群推送失败,请人工跟进:」;幂等键独立为 receiving-pending-fallback:{仓库}:{货主}:{YYYYMMDD},与主推送互不影响;执行汇总(summary)新增 fallbackCount",
"推送幂等去重:app/service/systems/warehouseNotify.js#pushSystemMessage 中同 ext_id 且 push_status=2(已送达)直接返回成功并跳过推送;push_status=0/1/3 仍复用同一 wms_inbox_message 记录重推,防止重复轮次/多实例并发导致同一群重复收到相同通知",
"事件路由与协议不变:仍为 receiving_pending_inbound + notify.reply(type=system),固定订阅 store:receiving(未命中回退 warehouse:customer_service);主推送幂等键仍为 receiving-pending:{仓库}:{货主}:{YYYYMMDD};接口文档(methods/system.json 的 setting.warehouse 字段、methods/warehouse_notify.json 的 notify.reply 协议)无需变更",
"单测:test/app/service/systems/receiving_pending_notice.test.js 新增 5 个用例(推送失败同日不重发+回退计数、回退独立幂等键与失败前缀、逐群异常隔离、Redis SET NX null 判定、app 级进程内锁跨实例共享),全部通过;npx eslint 无告警"
],
"module": "receiving"
},
{
"version": "2026-09-15-warehouse-02",
"date": "2026-09-15",