oms/backend/app/service/logRetention.js
liufumingren 6facd9c6f3 perf(log): 日志表按天分区,只保留 10 天
callback_logs / api_outbound_logs / request_logs 只增不减,DELETE 只标记删除、
磁盘不还。改为按 TO_DAYS(created_at) 做天分区,用 DROP PARTITION 物理删除。

- 迁移 20261107080000:三张日志表主键改为 (id, created_at)(MySQL 要求分区键
  包含在主键里),分区布局 p_old + 最近 10 天 + 未来 2 天 + pmax
- 新增 logRetention service:补未来分区(REORGANIZE pmax)→ DROP 过期分区;
  表若未分区则退化为分批 DELETE + OPTIMIZE 兜底
- 新增每日任务 purgeApiLogs(03:30):clean_request_logs 改为 disable,
  避免两条清理逻辑抢同一张表
- 保留天数默认 10 天,可用 LOG_RETENTION_DAYS 覆盖
- 文档补分区口径、为什么要 DROP PARTITION、上线注意

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-07 08:43:16 +08:00

237 lines
7.5 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

'use strict';
const Service = require('egg').Service;
/**
* 日志表保留策略(按天分区 + 每日滚动清理)
*
* 背景:日志表长期增长,MySQL 的 `DELETE` 只标记删除、磁盘不还;
* 按天分区后 `ALTER TABLE ... DROP PARTITION` 是瞬时物理删除,立刻释放空间。
*
* 约定(见迁移 20261107080000-partition-log-tables-by-day.js):
* - 分区键 `TO_DAYS(created_at)`,分区名 `pYYYYMMDD` 表示「< 次日 0 点」;
* - `p_old` 装保留期之前的历史数据,直接 DROP;
* - `pmax` 是兜底分区,每天用 `REORGANIZE PARTITION pmax` 补出新的日分区。
*
* 保留天数默认 10 天,可用 `LOG_RETENTION_DAYS` 覆盖。
*/
/** 需要滚动清理的日志表 */
const LOG_TABLES = [ 'callback_logs', 'api_outbound_logs', 'request_logs' ];
/** 预建未来分区天数 */
const AHEAD_DAYS = 2;
/** 非分区表的兜底清理:每批删除行数 */
const PURGE_BATCH = 5000;
/** 分区名:pYYYYMMDD / p_old / pmax */
const DAY_PARTITION_RE = /^p(\d{4})(\d{2})(\d{2})$/;
/**
* 日期格式(YYYY-MM-DD)
* @param {Date} date 日期
* @return {string} 形如 2026-11-07
*/
function fmtDate(date) {
const pad = value => String(value).padStart(2, '0');
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
}
/**
* 分区名里的日期格式(YYYYMMDD)
* @param {Date} date 日期
* @return {string} 形如 20261107
*/
function fmtCompact(date) {
return fmtDate(date).replace(/-/g, '');
}
/**
* 当天 0 点
* @param {Date} date 日期
* @return {Date} 0 点
*/
function startOfDay(date) {
const copy = new Date(date);
copy.setHours(0, 0, 0, 0);
return copy;
}
/**
* 加减天数
* @param {Date} date 日期
* @param {number} days 天数(可为负)
* @return {Date} 结果
*/
function addDays(date, days) {
const copy = new Date(date);
copy.setDate(copy.getDate() + days);
return copy;
}
class LogRetentionService extends Service {
/** 保留天数(默认 10 天) */
get keepDays() {
return Math.max(1, Number(process.env.LOG_RETENTION_DAYS) || 10);
}
/** 需要处理的日志表 */
get tables() {
return LOG_TABLES;
}
/**
* 列出某张表的分区
* @param {string} table 表名
* @return {Promise<Array<{name: string, rows: number}>>} 分区清单(按顺序)
*/
async listPartitions(table) {
const rows = await this.ctx.model.query(
`SELECT PARTITION_NAME AS name, TABLE_ROWS AS row_count
FROM information_schema.PARTITIONS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND PARTITION_NAME IS NOT NULL
ORDER BY PARTITION_ORDINAL_POSITION`,
{ replacements: [ table ], type: this.ctx.model.QueryTypes.SELECT }
);
return rows.map(row => ({ name: String(row.name), rows: Number(row.row_count) || 0 }));
}
/**
* 补建未来分区(把 pmax 拆出 pYYYYMMDD)
* @param {string} table 表名
* @param {string[]} existing 现有分区名
* @return {Promise<string[]>} 本次新建的分区名
*/
async ensureFuturePartitions(table, existing) {
const has = new Set(existing);
const created = [];
const today = startOfDay(new Date());
for (let i = 0; i <= AHEAD_DAYS; i += 1) {
const day = addDays(today, i);
const name = `p${fmtCompact(day)}`;
if (has.has(name)) continue;
// 已有更晚的日分区时跳过(避免上界冲突)
const laterExists = Array.from(has).some(item => {
const matched = DAY_PARTITION_RE.exec(item);
if (!matched) return false;
return `${matched[1]}${matched[2]}${matched[3]}` > fmtCompact(day);
});
if (laterExists) continue;
const upper = fmtDate(addDays(day, 1));
await this.ctx.model.query(
`ALTER TABLE \`${table}\` REORGANIZE PARTITION pmax INTO (
PARTITION ${name} VALUES LESS THAN (TO_DAYS('${upper}')),
PARTITION pmax VALUES LESS THAN MAXVALUE
)`
);
has.add(name);
created.push(name);
}
return created;
}
/**
* 删除过期分区(瞬时物理删除,释放磁盘)
* @param {string} table 表名
* @param {string[]} partitions 现有分区名
* @return {Promise<{dropped: string[], rows: number}>} 删除结果
*/
async dropExpiredPartitions(table, partitions) {
const today = startOfDay(new Date());
// 保留期第一天:早于它的日分区全部删除
const cutoff = addDays(today, -(this.keepDays - 1));
const cutoffCompact = fmtCompact(cutoff);
const dropped = [];
let rows = 0;
const listRows = await this.listPartitions(table);
const rowsByName = new Map(listRows.map(item => [ item.name, item.rows ]));
for (const name of partitions) {
let expired = false;
if (name === 'p_old') {
expired = true;
} else {
const matched = DAY_PARTITION_RE.exec(name);
if (matched) {
const compact = `${matched[1]}${matched[2]}${matched[3]}`;
expired = compact < cutoffCompact;
}
}
if (!expired) continue;
await this.ctx.model.query(`ALTER TABLE \`${table}\` DROP PARTITION ${name}`);
dropped.push(name);
rows += rowsByName.get(name) || 0;
}
return { dropped, rows };
}
/**
* 非分区表兜底清理:分批删除过期行后 OPTIMIZE 回收空间
* @param {string} table 表名
* @return {Promise<number>} 删除行数
*/
async purgeFlatTable(table) {
const cutoff = startOfDay(addDays(new Date(), -(this.keepDays - 1)));
let deleted = 0;
for (;;) {
const [ result ] = await this.ctx.model.query(
`DELETE FROM \`${table}\` WHERE created_at < ? ORDER BY id LIMIT ${PURGE_BATCH}`,
{ replacements: [ cutoff ] }
);
const affected = Number(result && result.affectedRows) || 0;
deleted += affected;
if (affected < PURGE_BATCH) break;
}
// DELETE 不还盘,OPTIMIZE 才能真正释放(表大时会重建表,放在每日低峰执行)
await this.ctx.model.query(`OPTIMIZE TABLE \`${table}\``);
return deleted;
}
/**
* 处理单张日志表:补分区 → 删过期分区(未分区则走兜底删除)
* @param {string} table 表名
* @return {Promise<Object>} 执行结果
*/
async runTable(table) {
const { ctx } = this;
try {
const partitions = await this.listPartitions(table);
if (!partitions.length) {
const deleted = await this.purgeFlatTable(table);
ctx.logger.info('[log-retention] %s 未分区,兜底删除 %d 行(保留 %d 天)', table, deleted, this.keepDays);
return { table, partitioned: false, deleted };
}
const created = await this.ensureFuturePartitions(table, partitions.map(item => item.name));
const { dropped, rows } = await this.dropExpiredPartitions(table, partitions.map(item => item.name));
ctx.logger.info(
'[log-retention] %s 新建分区 %s;删除分区 %s(约 %d 行,保留 %d 天)',
table,
created.join(',') || '-',
dropped.join(',') || '-',
rows,
this.keepDays
);
return { table, partitioned: true, created, dropped, rows };
} catch (err) {
ctx.logger.error('[log-retention] %s 处理失败:%s', table, err.message);
return { table, error: err.message };
}
}
/**
* 处理全部日志表
* @return {Promise<Array<Object>>} 各表结果
*/
async runAll() {
const results = [];
for (const table of this.tables) {
results.push(await this.runTable(table));
}
return results;
}
}
module.exports = LogRetentionService;