callback_logs / api_outbound_logs / request_logs 只增不减,DELETE 只标记删除、 磁盘不还。改为按 TO_DAYS(created_at) 做天分区,用 DROP PARTITION 物理删除。 - 迁移 20261107080000:三张日志表主键改为 (id, created_at)(MySQL 要求分区键 包含在主键里),分区布局 p_old + 最近 10 天 + 未来 2 天 + pmax - 新增 logRetention service:补未来分区(REORGANIZE pmax)→ DROP 过期分区; 表若未分区则退化为分批 DELETE + OPTIMIZE 兜底 - 新增每日任务 purgeApiLogs(03:30):clean_request_logs 改为 disable, 避免两条清理逻辑抢同一张表 - 保留天数默认 10 天,可用 LOG_RETENTION_DAYS 覆盖 - 文档补分区口径、为什么要 DROP PARTITION、上线注意 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
112 lines
3.9 KiB
JavaScript
112 lines
3.9 KiB
JavaScript
'use strict';
|
||
|
||
/**
|
||
* 日志表按天分区(callback_logs / api_outbound_logs / request_logs)。
|
||
*
|
||
* 目的:日志量大时 `DELETE` 只标记删除、不还盘;按天分区后 `DROP PARTITION` 是瞬时物理删除,
|
||
* 能立刻释放磁盘。分区键用 `TO_DAYS(created_at)`,分区名 `pYYYYMMDD`(表示「< 次日 0 点」)。
|
||
*
|
||
* MySQL 要求分区键必须包含在主键里,因此主键从 `(id)` 改为 `(id, created_at)`(id 仍自增)。
|
||
* 分区布局:`p_old`(保留期外的历史数据,由每日任务直接 DROP)+ 最近 10 天 + 未来 2 天 + `pmax`。
|
||
*/
|
||
|
||
/** 需要分区(=按天滚动的日志表) */
|
||
const TABLES = [ 'callback_logs', 'api_outbound_logs', 'request_logs' ];
|
||
|
||
/** 保留天数(与 app/service/logRetention.js 保持一致,可用 LOG_RETENTION_DAYS 覆盖) */
|
||
const KEEP_DAYS = Math.max(1, Number(process.env.LOG_RETENTION_DAYS) || 10);
|
||
|
||
/** 预建未来分区天数(留出余量,避免跨天时写入落到 pmax 后不好回收) */
|
||
const AHEAD_DAYS = 2;
|
||
|
||
/**
|
||
* 分区名里的日期格式(YYYYMMDD)
|
||
* @param {Date} date 日期
|
||
* @return {string} 形如 20261107
|
||
*/
|
||
function fmtCompact(date) {
|
||
const pad = value => String(value).padStart(2, '0');
|
||
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}`;
|
||
}
|
||
|
||
/**
|
||
* 日期格式(YYYY-MM-DD,用于 SQL 字面量)
|
||
* @param {Date} date 日期
|
||
* @return {string} 形如 2026-11-07
|
||
*/
|
||
function fmtDate(date) {
|
||
const pad = value => String(value).padStart(2, '0');
|
||
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
|
||
}
|
||
|
||
/**
|
||
* 当天 0 点
|
||
* @param {Date} date 日期
|
||
* @return {Date} 0 点
|
||
*/
|
||
function startOfDay(date) {
|
||
const copy = new Date(date);
|
||
copy.setHours(0, 0, 0, 0);
|
||
return copy;
|
||
}
|
||
|
||
/**
|
||
* 加减天数
|
||
* @param {Date} date 基准日期
|
||
* @param {number} days 天数(可为负)
|
||
* @return {Date} 结果
|
||
*/
|
||
function addDays(date, days) {
|
||
const copy = new Date(date);
|
||
copy.setDate(copy.getDate() + days);
|
||
return copy;
|
||
}
|
||
|
||
module.exports = {
|
||
async up(queryInterface) {
|
||
const sequelize = queryInterface.sequelize;
|
||
const today = startOfDay(new Date());
|
||
const firstKeep = addDays(today, -(KEEP_DAYS - 1));
|
||
|
||
for (const table of TABLES) {
|
||
const [ partitioned ] = await sequelize.query(
|
||
`SELECT COUNT(*) AS cnt FROM information_schema.PARTITIONS
|
||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${table}' AND PARTITION_NAME IS NOT NULL`,
|
||
{ type: sequelize.QueryTypes.SELECT }
|
||
);
|
||
if (Number(partitioned && partitioned.cnt) > 0) {
|
||
// 已分区(重复执行/手工改过)则不重复处理
|
||
continue;
|
||
}
|
||
|
||
// 分区键必须包含在主键里
|
||
await sequelize.query(`ALTER TABLE \`${table}\` DROP PRIMARY KEY, ADD PRIMARY KEY (id, created_at)`);
|
||
|
||
const parts = [ `PARTITION p_old VALUES LESS THAN (TO_DAYS('${fmtDate(firstKeep)}'))` ];
|
||
for (let i = 0; i <= KEEP_DAYS - 1 + AHEAD_DAYS; i += 1) {
|
||
const day = addDays(firstKeep, i);
|
||
parts.push(`PARTITION p${fmtCompact(day)} VALUES LESS THAN (TO_DAYS('${fmtDate(addDays(day, 1))}'))`);
|
||
}
|
||
parts.push('PARTITION pmax VALUES LESS THAN MAXVALUE');
|
||
|
||
await sequelize.query(
|
||
`ALTER TABLE \`${table}\` PARTITION BY RANGE (TO_DAYS(created_at)) (${parts.join(', ')})`
|
||
);
|
||
}
|
||
},
|
||
|
||
async down(queryInterface) {
|
||
const sequelize = queryInterface.sequelize;
|
||
for (const table of TABLES) {
|
||
const [ partitioned ] = await sequelize.query(
|
||
`SELECT COUNT(*) AS cnt FROM information_schema.PARTITIONS
|
||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${table}' AND PARTITION_NAME IS NOT NULL`,
|
||
{ type: sequelize.QueryTypes.SELECT }
|
||
);
|
||
if (!Number(partitioned && partitioned.cnt)) continue;
|
||
await sequelize.query(`ALTER TABLE \`${table}\` REMOVE PARTITIONING`);
|
||
await sequelize.query(`ALTER TABLE \`${table}\` DROP PRIMARY KEY, ADD PRIMARY KEY (id)`);
|
||
}
|
||
},
|
||
};
|