oms/backend/database/migrations/20261107080000-partition-log-tables-by-day.js
liufumingren 6facd9c6f3 perf(log): 日志表按天分区,只保留 10 天
callback_logs / api_outbound_logs / request_logs 只增不减,DELETE 只标记删除、
磁盘不还。改为按 TO_DAYS(created_at) 做天分区,用 DROP PARTITION 物理删除。

- 迁移 20261107080000:三张日志表主键改为 (id, created_at)(MySQL 要求分区键
  包含在主键里),分区布局 p_old + 最近 10 天 + 未来 2 天 + pmax
- 新增 logRetention service:补未来分区(REORGANIZE pmax)→ DROP 过期分区;
  表若未分区则退化为分批 DELETE + OPTIMIZE 兜底
- 新增每日任务 purgeApiLogs(03:30):clean_request_logs 改为 disable,
  避免两条清理逻辑抢同一张表
- 保留天数默认 10 天,可用 LOG_RETENTION_DAYS 覆盖
- 文档补分区口径、为什么要 DROP PARTITION、上线注意

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-07 08:43:16 +08:00

112 lines
3.9 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

'use strict';
/**
* 日志表按天分区(callback_logs / api_outbound_logs / request_logs)。
*
* 目的:日志量大时 `DELETE` 只标记删除、不还盘;按天分区后 `DROP PARTITION` 是瞬时物理删除,
* 能立刻释放磁盘。分区键用 `TO_DAYS(created_at)`,分区名 `pYYYYMMDD`(表示「< 次日 0 点」)。
*
* MySQL 要求分区键必须包含在主键里,因此主键从 `(id)` 改为 `(id, created_at)`(id 仍自增)。
* 分区布局:`p_old`(保留期外的历史数据,由每日任务直接 DROP)+ 最近 10 天 + 未来 2 天 + `pmax`。
*/
/** 需要分区(=按天滚动的日志表) */
const TABLES = [ 'callback_logs', 'api_outbound_logs', 'request_logs' ];
/** 保留天数(与 app/service/logRetention.js 保持一致,可用 LOG_RETENTION_DAYS 覆盖) */
const KEEP_DAYS = Math.max(1, Number(process.env.LOG_RETENTION_DAYS) || 10);
/** 预建未来分区天数(留出余量,避免跨天时写入落到 pmax 后不好回收) */
const AHEAD_DAYS = 2;
/**
* 分区名里的日期格式(YYYYMMDD)
* @param {Date} date 日期
* @return {string} 形如 20261107
*/
function fmtCompact(date) {
const pad = value => String(value).padStart(2, '0');
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}`;
}
/**
* 日期格式(YYYY-MM-DD,用于 SQL 字面量)
* @param {Date} date 日期
* @return {string} 形如 2026-11-07
*/
function fmtDate(date) {
const pad = value => String(value).padStart(2, '0');
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
}
/**
* 当天 0 点
* @param {Date} date 日期
* @return {Date} 0 点
*/
function startOfDay(date) {
const copy = new Date(date);
copy.setHours(0, 0, 0, 0);
return copy;
}
/**
* 加减天数
* @param {Date} date 基准日期
* @param {number} days 天数(可为负)
* @return {Date} 结果
*/
function addDays(date, days) {
const copy = new Date(date);
copy.setDate(copy.getDate() + days);
return copy;
}
module.exports = {
async up(queryInterface) {
const sequelize = queryInterface.sequelize;
const today = startOfDay(new Date());
const firstKeep = addDays(today, -(KEEP_DAYS - 1));
for (const table of TABLES) {
const [ partitioned ] = await sequelize.query(
`SELECT COUNT(*) AS cnt FROM information_schema.PARTITIONS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${table}' AND PARTITION_NAME IS NOT NULL`,
{ type: sequelize.QueryTypes.SELECT }
);
if (Number(partitioned && partitioned.cnt) > 0) {
// 已分区(重复执行/手工改过)则不重复处理
continue;
}
// 分区键必须包含在主键里
await sequelize.query(`ALTER TABLE \`${table}\` DROP PRIMARY KEY, ADD PRIMARY KEY (id, created_at)`);
const parts = [ `PARTITION p_old VALUES LESS THAN (TO_DAYS('${fmtDate(firstKeep)}'))` ];
for (let i = 0; i <= KEEP_DAYS - 1 + AHEAD_DAYS; i += 1) {
const day = addDays(firstKeep, i);
parts.push(`PARTITION p${fmtCompact(day)} VALUES LESS THAN (TO_DAYS('${fmtDate(addDays(day, 1))}'))`);
}
parts.push('PARTITION pmax VALUES LESS THAN MAXVALUE');
await sequelize.query(
`ALTER TABLE \`${table}\` PARTITION BY RANGE (TO_DAYS(created_at)) (${parts.join(', ')})`
);
}
},
async down(queryInterface) {
const sequelize = queryInterface.sequelize;
for (const table of TABLES) {
const [ partitioned ] = await sequelize.query(
`SELECT COUNT(*) AS cnt FROM information_schema.PARTITIONS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '${table}' AND PARTITION_NAME IS NOT NULL`,
{ type: sequelize.QueryTypes.SELECT }
);
if (!Number(partitioned && partitioned.cnt)) continue;
await sequelize.query(`ALTER TABLE \`${table}\` REMOVE PARTITIONING`);
await sequelize.query(`ALTER TABLE \`${table}\` DROP PRIMARY KEY, ADD PRIMARY KEY (id)`);
}
},
};