'use strict'; const Service = require('egg').Service; /** * 日志表保留策略(按天分区 + 每日滚动清理) * * 背景:日志表长期增长,MySQL 的 `DELETE` 只标记删除、磁盘不还; * 按天分区后 `ALTER TABLE ... DROP PARTITION` 是瞬时物理删除,立刻释放空间。 * * 约定(见迁移 20261107080000-partition-log-tables-by-day.js): * - 分区键 `TO_DAYS(created_at)`,分区名 `pYYYYMMDD` 表示「< 次日 0 点」; * - `p_old` 装保留期之前的历史数据,直接 DROP; * - `pmax` 是兜底分区,每天用 `REORGANIZE PARTITION pmax` 补出新的日分区。 * * 保留天数默认 10 天,可用 `LOG_RETENTION_DAYS` 覆盖。 */ /** 需要滚动清理的日志表 */ const LOG_TABLES = [ 'callback_logs', 'api_outbound_logs', 'request_logs' ]; /** 预建未来分区天数 */ const AHEAD_DAYS = 2; /** 非分区表的兜底清理:每批删除行数 */ const PURGE_BATCH = 5000; /** 分区名:pYYYYMMDD / p_old / pmax */ const DAY_PARTITION_RE = /^p(\d{4})(\d{2})(\d{2})$/; /** * 日期格式(YYYY-MM-DD) * @param {Date} date 日期 * @return {string} 形如 2026-11-07 */ function fmtDate(date) { const pad = value => String(value).padStart(2, '0'); return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; } /** * 分区名里的日期格式(YYYYMMDD) * @param {Date} date 日期 * @return {string} 形如 20261107 */ function fmtCompact(date) { return fmtDate(date).replace(/-/g, ''); } /** * 当天 0 点 * @param {Date} date 日期 * @return {Date} 0 点 */ function startOfDay(date) { const copy = new Date(date); copy.setHours(0, 0, 0, 0); return copy; } /** * 加减天数 * @param {Date} date 日期 * @param {number} days 天数(可为负) * @return {Date} 结果 */ function addDays(date, days) { const copy = new Date(date); copy.setDate(copy.getDate() + days); return copy; } class LogRetentionService extends Service { /** 保留天数(默认 10 天) */ get keepDays() { return Math.max(1, Number(process.env.LOG_RETENTION_DAYS) || 10); } /** 需要处理的日志表 */ get tables() { return LOG_TABLES; } /** * 列出某张表的分区 * @param {string} table 表名 * @return {Promise>} 分区清单(按顺序) */ async listPartitions(table) { const rows = await this.ctx.model.query( `SELECT PARTITION_NAME AS name, TABLE_ROWS AS row_count FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND PARTITION_NAME IS NOT NULL ORDER BY PARTITION_ORDINAL_POSITION`, { replacements: [ table ], type: this.ctx.model.QueryTypes.SELECT } ); return rows.map(row => ({ name: String(row.name), rows: Number(row.row_count) || 0 })); } /** * 补建未来分区(把 pmax 拆出 pYYYYMMDD) * @param {string} table 表名 * @param {string[]} existing 现有分区名 * @return {Promise} 本次新建的分区名 */ async ensureFuturePartitions(table, existing) { const has = new Set(existing); const created = []; const today = startOfDay(new Date()); for (let i = 0; i <= AHEAD_DAYS; i += 1) { const day = addDays(today, i); const name = `p${fmtCompact(day)}`; if (has.has(name)) continue; // 已有更晚的日分区时跳过(避免上界冲突) const laterExists = Array.from(has).some(item => { const matched = DAY_PARTITION_RE.exec(item); if (!matched) return false; return `${matched[1]}${matched[2]}${matched[3]}` > fmtCompact(day); }); if (laterExists) continue; const upper = fmtDate(addDays(day, 1)); await this.ctx.model.query( `ALTER TABLE \`${table}\` REORGANIZE PARTITION pmax INTO ( PARTITION ${name} VALUES LESS THAN (TO_DAYS('${upper}')), PARTITION pmax VALUES LESS THAN MAXVALUE )` ); has.add(name); created.push(name); } return created; } /** * 删除过期分区(瞬时物理删除,释放磁盘) * @param {string} table 表名 * @param {string[]} partitions 现有分区名 * @return {Promise<{dropped: string[], rows: number}>} 删除结果 */ async dropExpiredPartitions(table, partitions) { const today = startOfDay(new Date()); // 保留期第一天:早于它的日分区全部删除 const cutoff = addDays(today, -(this.keepDays - 1)); const cutoffCompact = fmtCompact(cutoff); const dropped = []; let rows = 0; const listRows = await this.listPartitions(table); const rowsByName = new Map(listRows.map(item => [ item.name, item.rows ])); for (const name of partitions) { let expired = false; if (name === 'p_old') { expired = true; } else { const matched = DAY_PARTITION_RE.exec(name); if (matched) { const compact = `${matched[1]}${matched[2]}${matched[3]}`; expired = compact < cutoffCompact; } } if (!expired) continue; await this.ctx.model.query(`ALTER TABLE \`${table}\` DROP PARTITION ${name}`); dropped.push(name); rows += rowsByName.get(name) || 0; } return { dropped, rows }; } /** * 非分区表兜底清理:分批删除过期行后 OPTIMIZE 回收空间 * @param {string} table 表名 * @return {Promise} 删除行数 */ async purgeFlatTable(table) { const cutoff = startOfDay(addDays(new Date(), -(this.keepDays - 1))); let deleted = 0; for (;;) { const [ result ] = await this.ctx.model.query( `DELETE FROM \`${table}\` WHERE created_at < ? ORDER BY id LIMIT ${PURGE_BATCH}`, { replacements: [ cutoff ] } ); const affected = Number(result && result.affectedRows) || 0; deleted += affected; if (affected < PURGE_BATCH) break; } // DELETE 不还盘,OPTIMIZE 才能真正释放(表大时会重建表,放在每日低峰执行) await this.ctx.model.query(`OPTIMIZE TABLE \`${table}\``); return deleted; } /** * 处理单张日志表:补分区 → 删过期分区(未分区则走兜底删除) * @param {string} table 表名 * @return {Promise} 执行结果 */ async runTable(table) { const { ctx } = this; try { const partitions = await this.listPartitions(table); if (!partitions.length) { const deleted = await this.purgeFlatTable(table); ctx.logger.info('[log-retention] %s 未分区,兜底删除 %d 行(保留 %d 天)', table, deleted, this.keepDays); return { table, partitioned: false, deleted }; } const created = await this.ensureFuturePartitions(table, partitions.map(item => item.name)); const { dropped, rows } = await this.dropExpiredPartitions(table, partitions.map(item => item.name)); ctx.logger.info( '[log-retention] %s 新建分区 %s;删除分区 %s(约 %d 行,保留 %d 天)', table, created.join(',') || '-', dropped.join(',') || '-', rows, this.keepDays ); return { table, partitioned: true, created, dropped, rows }; } catch (err) { ctx.logger.error('[log-retention] %s 处理失败:%s', table, err.message); return { table, error: err.message }; } } /** * 处理全部日志表 * @return {Promise>} 各表结果 */ async runAll() { const results = []; for (const table of this.tables) { results.push(await this.runTable(table)); } return results; } } module.exports = LogRetentionService;