callback_logs / api_outbound_logs / request_logs 只增不减,DELETE 只标记删除、 磁盘不还。改为按 TO_DAYS(created_at) 做天分区,用 DROP PARTITION 物理删除。 - 迁移 20261107080000:三张日志表主键改为 (id, created_at)(MySQL 要求分区键 包含在主键里),分区布局 p_old + 最近 10 天 + 未来 2 天 + pmax - 新增 logRetention service:补未来分区(REORGANIZE pmax)→ DROP 过期分区; 表若未分区则退化为分批 DELETE + OPTIMIZE 兜底 - 新增每日任务 purgeApiLogs(03:30):clean_request_logs 改为 disable, 避免两条清理逻辑抢同一张表 - 保留天数默认 10 天,可用 LOG_RETENTION_DAYS 覆盖 - 文档补分区口径、为什么要 DROP PARTITION、上线注意 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
237 lines
7.5 KiB
JavaScript
237 lines
7.5 KiB
JavaScript
'use strict';
|
||
|
||
const Service = require('egg').Service;
|
||
|
||
/**
|
||
* 日志表保留策略(按天分区 + 每日滚动清理)
|
||
*
|
||
* 背景:日志表长期增长,MySQL 的 `DELETE` 只标记删除、磁盘不还;
|
||
* 按天分区后 `ALTER TABLE ... DROP PARTITION` 是瞬时物理删除,立刻释放空间。
|
||
*
|
||
* 约定(见迁移 20261107080000-partition-log-tables-by-day.js):
|
||
* - 分区键 `TO_DAYS(created_at)`,分区名 `pYYYYMMDD` 表示「< 次日 0 点」;
|
||
* - `p_old` 装保留期之前的历史数据,直接 DROP;
|
||
* - `pmax` 是兜底分区,每天用 `REORGANIZE PARTITION pmax` 补出新的日分区。
|
||
*
|
||
* 保留天数默认 10 天,可用 `LOG_RETENTION_DAYS` 覆盖。
|
||
*/
|
||
|
||
/** 需要滚动清理的日志表 */
|
||
const LOG_TABLES = [ 'callback_logs', 'api_outbound_logs', 'request_logs' ];
|
||
|
||
/** 预建未来分区天数 */
|
||
const AHEAD_DAYS = 2;
|
||
|
||
/** 非分区表的兜底清理:每批删除行数 */
|
||
const PURGE_BATCH = 5000;
|
||
|
||
/** 分区名:pYYYYMMDD / p_old / pmax */
|
||
const DAY_PARTITION_RE = /^p(\d{4})(\d{2})(\d{2})$/;
|
||
|
||
/**
|
||
* 日期格式(YYYY-MM-DD)
|
||
* @param {Date} date 日期
|
||
* @return {string} 形如 2026-11-07
|
||
*/
|
||
function fmtDate(date) {
|
||
const pad = value => String(value).padStart(2, '0');
|
||
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
|
||
}
|
||
|
||
/**
|
||
* 分区名里的日期格式(YYYYMMDD)
|
||
* @param {Date} date 日期
|
||
* @return {string} 形如 20261107
|
||
*/
|
||
function fmtCompact(date) {
|
||
return fmtDate(date).replace(/-/g, '');
|
||
}
|
||
|
||
/**
|
||
* 当天 0 点
|
||
* @param {Date} date 日期
|
||
* @return {Date} 0 点
|
||
*/
|
||
function startOfDay(date) {
|
||
const copy = new Date(date);
|
||
copy.setHours(0, 0, 0, 0);
|
||
return copy;
|
||
}
|
||
|
||
/**
|
||
* 加减天数
|
||
* @param {Date} date 日期
|
||
* @param {number} days 天数(可为负)
|
||
* @return {Date} 结果
|
||
*/
|
||
function addDays(date, days) {
|
||
const copy = new Date(date);
|
||
copy.setDate(copy.getDate() + days);
|
||
return copy;
|
||
}
|
||
|
||
class LogRetentionService extends Service {
|
||
/** 保留天数(默认 10 天) */
|
||
get keepDays() {
|
||
return Math.max(1, Number(process.env.LOG_RETENTION_DAYS) || 10);
|
||
}
|
||
|
||
/** 需要处理的日志表 */
|
||
get tables() {
|
||
return LOG_TABLES;
|
||
}
|
||
|
||
/**
|
||
* 列出某张表的分区
|
||
* @param {string} table 表名
|
||
* @return {Promise<Array<{name: string, rows: number}>>} 分区清单(按顺序)
|
||
*/
|
||
async listPartitions(table) {
|
||
const rows = await this.ctx.model.query(
|
||
`SELECT PARTITION_NAME AS name, TABLE_ROWS AS row_count
|
||
FROM information_schema.PARTITIONS
|
||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND PARTITION_NAME IS NOT NULL
|
||
ORDER BY PARTITION_ORDINAL_POSITION`,
|
||
{ replacements: [ table ], type: this.ctx.model.QueryTypes.SELECT }
|
||
);
|
||
return rows.map(row => ({ name: String(row.name), rows: Number(row.row_count) || 0 }));
|
||
}
|
||
|
||
/**
|
||
* 补建未来分区(把 pmax 拆出 pYYYYMMDD)
|
||
* @param {string} table 表名
|
||
* @param {string[]} existing 现有分区名
|
||
* @return {Promise<string[]>} 本次新建的分区名
|
||
*/
|
||
async ensureFuturePartitions(table, existing) {
|
||
const has = new Set(existing);
|
||
const created = [];
|
||
const today = startOfDay(new Date());
|
||
for (let i = 0; i <= AHEAD_DAYS; i += 1) {
|
||
const day = addDays(today, i);
|
||
const name = `p${fmtCompact(day)}`;
|
||
if (has.has(name)) continue;
|
||
// 已有更晚的日分区时跳过(避免上界冲突)
|
||
const laterExists = Array.from(has).some(item => {
|
||
const matched = DAY_PARTITION_RE.exec(item);
|
||
if (!matched) return false;
|
||
return `${matched[1]}${matched[2]}${matched[3]}` > fmtCompact(day);
|
||
});
|
||
if (laterExists) continue;
|
||
const upper = fmtDate(addDays(day, 1));
|
||
await this.ctx.model.query(
|
||
`ALTER TABLE \`${table}\` REORGANIZE PARTITION pmax INTO (
|
||
PARTITION ${name} VALUES LESS THAN (TO_DAYS('${upper}')),
|
||
PARTITION pmax VALUES LESS THAN MAXVALUE
|
||
)`
|
||
);
|
||
has.add(name);
|
||
created.push(name);
|
||
}
|
||
return created;
|
||
}
|
||
|
||
/**
|
||
* 删除过期分区(瞬时物理删除,释放磁盘)
|
||
* @param {string} table 表名
|
||
* @param {string[]} partitions 现有分区名
|
||
* @return {Promise<{dropped: string[], rows: number}>} 删除结果
|
||
*/
|
||
async dropExpiredPartitions(table, partitions) {
|
||
const today = startOfDay(new Date());
|
||
// 保留期第一天:早于它的日分区全部删除
|
||
const cutoff = addDays(today, -(this.keepDays - 1));
|
||
const cutoffCompact = fmtCompact(cutoff);
|
||
const dropped = [];
|
||
let rows = 0;
|
||
|
||
const listRows = await this.listPartitions(table);
|
||
const rowsByName = new Map(listRows.map(item => [ item.name, item.rows ]));
|
||
|
||
for (const name of partitions) {
|
||
let expired = false;
|
||
if (name === 'p_old') {
|
||
expired = true;
|
||
} else {
|
||
const matched = DAY_PARTITION_RE.exec(name);
|
||
if (matched) {
|
||
const compact = `${matched[1]}${matched[2]}${matched[3]}`;
|
||
expired = compact < cutoffCompact;
|
||
}
|
||
}
|
||
if (!expired) continue;
|
||
await this.ctx.model.query(`ALTER TABLE \`${table}\` DROP PARTITION ${name}`);
|
||
dropped.push(name);
|
||
rows += rowsByName.get(name) || 0;
|
||
}
|
||
return { dropped, rows };
|
||
}
|
||
|
||
/**
|
||
* 非分区表兜底清理:分批删除过期行后 OPTIMIZE 回收空间
|
||
* @param {string} table 表名
|
||
* @return {Promise<number>} 删除行数
|
||
*/
|
||
async purgeFlatTable(table) {
|
||
const cutoff = startOfDay(addDays(new Date(), -(this.keepDays - 1)));
|
||
let deleted = 0;
|
||
for (;;) {
|
||
const [ result ] = await this.ctx.model.query(
|
||
`DELETE FROM \`${table}\` WHERE created_at < ? ORDER BY id LIMIT ${PURGE_BATCH}`,
|
||
{ replacements: [ cutoff ] }
|
||
);
|
||
const affected = Number(result && result.affectedRows) || 0;
|
||
deleted += affected;
|
||
if (affected < PURGE_BATCH) break;
|
||
}
|
||
// DELETE 不还盘,OPTIMIZE 才能真正释放(表大时会重建表,放在每日低峰执行)
|
||
await this.ctx.model.query(`OPTIMIZE TABLE \`${table}\``);
|
||
return deleted;
|
||
}
|
||
|
||
/**
|
||
* 处理单张日志表:补分区 → 删过期分区(未分区则走兜底删除)
|
||
* @param {string} table 表名
|
||
* @return {Promise<Object>} 执行结果
|
||
*/
|
||
async runTable(table) {
|
||
const { ctx } = this;
|
||
try {
|
||
const partitions = await this.listPartitions(table);
|
||
if (!partitions.length) {
|
||
const deleted = await this.purgeFlatTable(table);
|
||
ctx.logger.info('[log-retention] %s 未分区,兜底删除 %d 行(保留 %d 天)', table, deleted, this.keepDays);
|
||
return { table, partitioned: false, deleted };
|
||
}
|
||
const created = await this.ensureFuturePartitions(table, partitions.map(item => item.name));
|
||
const { dropped, rows } = await this.dropExpiredPartitions(table, partitions.map(item => item.name));
|
||
ctx.logger.info(
|
||
'[log-retention] %s 新建分区 %s;删除分区 %s(约 %d 行,保留 %d 天)',
|
||
table,
|
||
created.join(',') || '-',
|
||
dropped.join(',') || '-',
|
||
rows,
|
||
this.keepDays
|
||
);
|
||
return { table, partitioned: true, created, dropped, rows };
|
||
} catch (err) {
|
||
ctx.logger.error('[log-retention] %s 处理失败:%s', table, err.message);
|
||
return { table, error: err.message };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 处理全部日志表
|
||
* @return {Promise<Array<Object>>} 各表结果
|
||
*/
|
||
async runAll() {
|
||
const results = [];
|
||
for (const table of this.tables) {
|
||
results.push(await this.runTable(table));
|
||
}
|
||
return results;
|
||
}
|
||
}
|
||
|
||
module.exports = LogRetentionService;
|