chore(ops): 生产日志目录巡检与清理脚本,超过 4GiB 触发清理至 2GiB 以下,活跃日志与近期轮转保留,附本地 dry-run 自测桩

This commit is contained in:
liufumingren 2026-09-03 17:03:59 +08:00
parent 6354a6d8dc
commit fb8840ecdb
3 changed files with 449 additions and 0 deletions

View File

@ -58,6 +58,9 @@
"deploy:verify": "node ./script/deploy-ops.cjs production verify",
"deploy:cleanup-backups": "node ./script/deploy-ops.cjs production cleanup-backups",
"deploy:gateway-probe": "node ./script/deploy-ops.cjs production gateway-probe",
"ops:check-root-logs": "node ./script/check-root-logs.cjs",
"ops:clean-root-logs": "node ./script/check-root-logs.cjs production --clean",
"ops:clean-root-logs:apply": "node ./script/check-root-logs.cjs production --clean --apply",
"version:record": "node ./script/version-record.cjs",
"version:history": "node ./script/version-history.cjs --limit=50",
"deploy": "npm run deploy:prod",

386
script/check-root-logs.cjs Normal file
View File

@ -0,0 +1,386 @@
'use strict';
const path = require('node:path');
const {
ensureCommandAvailable,
loadDeployConfig,
quoteForBash,
runSsh,
} = require('./deploy-lib.cjs');
const DEFAULT_ENV = 'production';
const LOGS_DIR = '/root/logs/wms';
/** 正在写入的活跃日志文件名(不带日期后缀),绝对不允许删除 */
const ACTIVE_LOG_NAME = 'wms-web.log';
/** egg logrotator 按日期轮转后的历史日志文件名模式:wms-web.log.YYYY-MM-DD */
const ROTATED_LOG_PATTERN = /^wms-web\.log\.\d{4}-\d{2}-\d{2}$/;
/** 触发清理的目录总大小阈值:4 GiB */
const CLEAN_THRESHOLD_BYTES = 4 * 1024 * 1024 * 1024;
/** 清理目标水位:目录总大小降到 2 GiB 以下 */
const CLEAN_TARGET_BYTES = 2 * 1024 * 1024 * 1024;
/** 清理后至少保留的历史轮转文件数量 */
const MIN_KEEP_COUNT = 3;
/**
* 构建只读的远端日志目录检测脚本
* 仅使用 du / ls / df 等只读命令,不包含任何删除、清理或写入操作
*
* @return {string} 远端 bash 脚本内容
*/
function buildRemoteScript() {
const logsDir = quoteForBash(LOGS_DIR);
return [
'set -euo pipefail',
`LOGS_DIR=${logsDir}`,
'printf "===WMS_LOGS_SIZE===\\n"',
'if [ -d "$LOGS_DIR" ]; then',
' du -sh "$LOGS_DIR" 2>&1 | head -5 || true',
'else',
' echo "DIR_NOT_FOUND: $LOGS_DIR"',
'fi',
'printf "===TOP_LARGE_FILES===\\n"',
'if [ -d "$LOGS_DIR" ]; then',
' ls -lhS "$LOGS_DIR" 2>&1 | head -20 || true',
'else',
' echo "SKIP: $LOGS_DIR 不存在"',
'fi',
'printf "===TOP_LARGE_ITEMS===\\n"',
'if [ -d "$LOGS_DIR" ]; then',
' du -sh "$LOGS_DIR"/* 2>&1 | sort -rh | head -20 || true',
'else',
' echo "SKIP: $LOGS_DIR 不存在"',
'fi',
'printf "===ROOT_PARTITION===\\n"',
'df -h / 2>&1 || true',
].join('\n');
}
/**
* 通过 SSH 执行只读检测,返回远端输出
*
* @param {object} config - 部署配置对象
* @return {string} 远端日志目录检测结果
*/
function checkRootLogs(config) {
const remoteScript = buildRemoteScript();
return runSsh(config, remoteScript, { stdio: 'pipe' }).trim();
}
/**
* 将字节数格式化为人类可读的大小字符串
*
* @param {number} bytes - 字节数
* @return {string} 人类可读的大小,如 "3.2 GiB"
*/
function formatBytes(bytes) {
if (!Number.isFinite(bytes) || bytes < 0) {
return 'N/A';
}
const units = [ 'B', 'KiB', 'MiB', 'GiB', 'TiB' ];
let value = bytes;
let unitIndex = 0;
while (value >= 1024 && unitIndex < units.length - 1) {
value /= 1024;
unitIndex += 1;
}
const digits = unitIndex === 0 ? 0 : 1;
return `${value.toFixed(digits)} ${units[unitIndex]}`;
}
/**
* 将 epoch 秒数格式化为 ISO 时间字符串,便于人工核对 mtime
*
* @param {number} mtimeSeconds - Unix 时间戳(秒,支持小数)
* @return {string} ISO 格式时间字符串,无效时返回 UNKNOWN
*/
function formatMtime(mtimeSeconds) {
const date = new Date(Number(mtimeSeconds) * 1000);
if (Number.isNaN(date.getTime())) {
return 'UNKNOWN';
}
return date.toISOString();
}
/**
* 构建远端只读探测脚本:计算目录总大小并按 mtime 升序列出全部文件
* 仅使用 du / find / sort 等只读命令,不包含任何删除操作;
* 目录不存在或无法计算大小时输出 NO_CLEANUP_NEEDED 并以 0 正常退出
*
* @return {string} 远端 bash 脚本内容
*/
function buildCleanProbeScript() {
const logsDir = quoteForBash(LOGS_DIR);
return [
'set -euo pipefail',
`LOGS_DIR=${logsDir}`,
'if [ ! -d "$LOGS_DIR" ]; then',
' echo "NO_CLEANUP_NEEDED: DIR_NOT_FOUND: $LOGS_DIR"',
' exit 0',
'fi',
'TOTAL_BYTES=$(du -sb "$LOGS_DIR" | awk \'{print $1}\')',
'if [ -z "${TOTAL_BYTES}" ]; then',
' echo "NO_CLEANUP_NEEDED: EMPTY_DIR_SIZE"',
' exit 0',
'fi',
'echo "PROBE_TOTAL_BYTES=${TOTAL_BYTES}"',
'find "$LOGS_DIR" -maxdepth 1 -type f -printf \'%T@\\t%s\\t%p\\n\' | sort -k1,1n',
].join('\n');
}
/**
* 解析远端只读探测脚本的标准输出
* 识别 NO_CLEANUP_NEEDED 提示与 PROBE_TOTAL_BYTES 总大小标记,
* 并把每行文件清单解析为 mtime / size / filePath 三元组
*
* @param {string} output - 远端探测脚本的标准输出
* @return {object} 解析结果:
* - noCleanup: {string|null} NO_CLEANUP_NEEDED 提示(无需清理时非空)
* - totalBytes: {number} 目录总字节数
* - files: {Array<{mtime:number,size:number,filePath:string}>} 文件清单
*/
function parseCleanProbe(output) {
const lines = String(output || '').split(/\r?\n/)
.map(line => line.trim())
.filter(Boolean);
const noCleanup = lines.find(line => line.startsWith('NO_CLEANUP_NEEDED'));
if (noCleanup) {
return { noCleanup, totalBytes: 0, files: [] };
}
const totalLine = lines.find(line => line.startsWith('PROBE_TOTAL_BYTES='));
if (!totalLine) {
throw new Error('远端探测输出缺少 PROBE_TOTAL_BYTES 标记,无法确定目录总大小。');
}
const totalBytes = Number(totalLine.slice('PROBE_TOTAL_BYTES='.length));
if (!Number.isFinite(totalBytes)) {
throw new Error(`远端探测输出的总大小非法: ${totalLine}`);
}
const files = [];
for (const line of lines) {
if (line.startsWith('PROBE_TOTAL_BYTES=')) {
continue;
}
const parts = line.split('\t');
if (parts.length < 3) {
continue;
}
const mtime = Number(parts[0]);
const size = Number(parts[1]);
const filePath = parts.slice(2).join('\t');
if (!Number.isFinite(mtime) || !Number.isFinite(size) || !filePath) {
continue;
}
files.push({ mtime, size, filePath });
}
return { noCleanup: null, totalBytes, files };
}
/**
* 从远端文件清单中筛选出允许删除的历史轮转日志文件
* 安全边界:仅保留 basename 严格匹配轮转命名模式的文件,
* 明确排除正在写入的活跃日志,并按 mtime 从旧到新排序
*
* @param {Array<{mtime:number,size:number,filePath:string}>} files - 远端文件清单
* @return {Array<{mtime:number,size:number,filePath:string}>} 候选删除文件(mtime 升序)
*/
function collectCleanCandidates(files) {
return files
.filter(item => {
const baseName = path.posix.basename(item.filePath);
if (baseName === ACTIVE_LOG_NAME) {
return false;
}
return ROTATED_LOG_PATTERN.test(baseName);
})
.sort((left, right) => left.mtime - right.mtime);
}
/**
* 按阈值 / 目标水位 / 最小保留数量计算删除计划
* 从最旧的候选文件开始逐个删除,直到预计总大小降到目标水位以下;
* 候选数量不足 MIN_KEEP_COUNT 时停止,避免误删近期日志
*
* @param {Array<{mtime:number,size:number,filePath:string}>} candidates - 候选删除文件(mtime 升序)
* @param {number} totalBytes - 当前目录总字节数
* @return {object} 删除计划:
* - deleteFiles: {Array} 计划删除的文件列表
* - keepFiles: {Array} 计划保留的文件列表
* - totalBytes: {number} 删除前总字节数
* - estimatedAfterBytes: {number} 删除后预计总字节数
* - limitedByMinKeep: {boolean} 是否因最小保留数量限制未能降到目标水位
*/
function buildCleanPlan(candidates, totalBytes) {
const pool = [ ...candidates ];
const deleteFiles = [];
let remaining = totalBytes;
while (pool.length > MIN_KEEP_COUNT && remaining > CLEAN_TARGET_BYTES) {
const oldest = pool.shift();
deleteFiles.push(oldest);
remaining -= oldest.size;
}
return {
deleteFiles,
keepFiles: pool,
totalBytes,
estimatedAfterBytes: remaining,
limitedByMinKeep: remaining > CLEAN_TARGET_BYTES && pool.length <= MIN_KEEP_COUNT,
};
}
/**
* 构建远端实际删除脚本(仅 --apply 模式使用)
* 删除清单通过 quoted here-doc 逐行传入,逐条校验路径必须严格位于
* ${LOGS_DIR}/ 下且 basename 匹配轮转命名模式,再用 `rm -f -- 完整路径` 删除;
* 不使用 rm -rf、不使用任何通配符;删除完成后重新计算并输出目录总大小
*
* @param {Array<string>} filePaths - 计划删除的完整路径列表
* @return {string} 远端 bash 脚本内容
*/
function buildCleanApplyScript(filePaths) {
const logsDir = quoteForBash(LOGS_DIR);
const lines = [
'set -euo pipefail',
`LOGS_DIR=${logsDir}`,
'if [ ! -d "$LOGS_DIR" ]; then',
' echo "NO_CLEANUP_NEEDED: DIR_NOT_FOUND: $LOGS_DIR"',
' exit 0',
'fi',
'while IFS= read -r FILE_PATH; do',
' [ -n "$FILE_PATH" ] || continue',
' case "$FILE_PATH" in',
' "$LOGS_DIR"/wms-web.log.[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;;',
' *)',
' echo "CLEAN_FAIL: 拒绝删除不匹配轮转命名模式的文件: $FILE_PATH" >&2',
' exit 1',
' ;;',
' esac',
' rm -f -- "$FILE_PATH"',
' echo "REMOVED $FILE_PATH"',
'done <<\'CLEAN_FILE_LIST\'',
...filePaths,
'CLEAN_FILE_LIST',
'AFTER_BYTES=$(du -sb "$LOGS_DIR" | awk \'{print $1}\')',
'echo "CLEAN_AFTER_BYTES=${AFTER_BYTES:-0}"',
];
return lines.join('\n');
}
/**
* 执行日志目录清理流程(dry-run 或 apply)
* 1. 通过 SSH 只读探测目录总大小与文件清单
* 2. 目录不存在或大小未超阈值时输出 NO_CLEANUP_NEEDED 并正常退出
* 3. dry-run 输出 WOULD_REMOVE 删除计划与删除前后总大小,不执行任何删除;
* apply 才真正执行删除并输出 REMOVED 清单与 CLEAN_AFTER_BYTES
*
* @param {object} config - 部署配置对象
* @param {object} options - 选项对象
* @param {boolean} options.apply - true 表示真正执行删除;false(默认)为 dry-run
* @return {void}
*/
function runClean(config, options = {}) {
const apply = Boolean(options.apply);
const modeText = apply ? 'apply' : 'dry-run';
console.log(`[CLEAN_ROOT_LOGS] env=${config.envName} host=${config.username}@${config.host}:${config.port} dir=${LOGS_DIR} mode=${modeText}`);
const probe = parseCleanProbe(runSsh(config, buildCleanProbeScript(), { stdio: 'pipe' }).trim());
if (probe.noCleanup) {
console.log(`NO_CLEANUP_NEEDED: ${probe.noCleanup}`);
console.log('CLEAN_SKIPPED');
return;
}
console.log(`TOTAL_BEFORE=${probe.totalBytes} bytes (${formatBytes(probe.totalBytes)}) threshold=${formatBytes(CLEAN_THRESHOLD_BYTES)} target=${formatBytes(CLEAN_TARGET_BYTES)}`);
if (probe.totalBytes <= CLEAN_THRESHOLD_BYTES) {
console.log(`NO_CLEANUP_NEEDED: 目录总大小未超过 ${formatBytes(CLEAN_THRESHOLD_BYTES)} 阈值,无需清理`);
console.log('CLEAN_SKIPPED');
return;
}
const candidates = collectCleanCandidates(probe.files);
if (candidates.length === 0) {
console.log('NO_CLEANUP_NEEDED: 未找到符合轮转命名模式的历史日志文件,不做任何删除');
console.log('CLEAN_SKIPPED');
return;
}
const plan = buildCleanPlan(candidates, probe.totalBytes);
if (plan.deleteFiles.length === 0) {
console.log(`NO_CLEANUP_NEEDED: 需保留最近 ${MIN_KEEP_COUNT} 个历史文件,无可安全删除的文件`);
console.log('CLEAN_SKIPPED');
return;
}
console.log(`CANDIDATES=${candidates.length} PLAN_DELETE=${plan.deleteFiles.length} KEEP_RECENT=${plan.keepFiles.length}`);
if (apply) {
console.log(runSsh(config, buildCleanApplyScript(plan.deleteFiles.map(file => file.filePath)), { stdio: 'pipe' }).trim());
} else {
for (const file of plan.deleteFiles) {
console.log(`WOULD_REMOVE ${file.filePath} ${file.size} bytes (${formatBytes(file.size)}) mtime=${formatMtime(file.mtime)}`);
}
console.log(`TOTAL_AFTER=${plan.estimatedAfterBytes} bytes (${formatBytes(plan.estimatedAfterBytes)}) (预计)`);
}
if (plan.limitedByMinKeep) {
console.log(`CLEAN_LIMITED: 受"保留最近 ${MIN_KEEP_COUNT} 个历史文件"限制,清理后目录可能仍高于目标水位 ${formatBytes(CLEAN_TARGET_BYTES)}`);
}
console.log(apply ? 'CLEAN_OK' : 'CLEAN_DRY_RUN_OK');
}
/**
* 解析命令行参数
* 用法:node script/check-root-logs.cjs [env] [--clean] [--apply]
* 不带 --clean 时保持原有只读检测行为;--clean 默认 dry-run 输出删除计划;
* --apply 必须与 --clean 一起使用才真正执行删除
*
* @param {Array<string>} argv - process.argv.slice(2)
* @return {object} 解析结果:{ envName: {string}, clean: {boolean}, apply: {boolean} }
*/
function parseArgs(argv) {
const result = { envName: DEFAULT_ENV, clean: false, apply: false };
for (const arg of argv) {
if (arg === '--clean') {
result.clean = true;
} else if (arg === '--apply') {
result.apply = true;
} else if (arg.startsWith('--')) {
throw new Error(`未知参数: ${arg}`);
} else {
result.envName = arg;
}
}
if (result.apply && !result.clean) {
throw new Error('--apply 必须与 --clean 一起使用;请先使用 --clean(默认 dry-run)预览删除计划。');
}
return result;
}
/**
* 脚本入口:加载部署配置并执行只读日志目录检测
* 无 --clean 时沿用原有只读检测流程;有 --clean 时执行日志目录清理(默认 dry-run,仅 --apply 真正删除);配置缺失或 SSH 失败时输出错误信息并以非零退出码结束
*
* @return {void}
*/
function main() {
const rawArgs = process.argv.slice(2);
const isCleanFlow = rawArgs.includes('--clean') || rawArgs.includes('--apply');
try {
ensureCommandAvailable('ssh', '请先安装 OpenSSH Client 并确保 ssh 在 PATH 中。');
const options = parseArgs(rawArgs);
const config = loadDeployConfig(options.envName);
if (options.clean) {
runClean(config, { apply: options.apply });
return;
}
console.log(`[CHECK_ROOT_LOGS] env=${options.envName} host=${config.username}@${config.host}:${config.port} dir=${LOGS_DIR}`);
console.log(checkRootLogs(config));
console.log('CHECK_ROOT_LOGS_OK');
} catch (error) {
const message = String(error && error.message ? error.message : error);
const failTag = isCleanFlow ? 'CLEAN_FAIL' : 'CHECK_ROOT_LOGS_FAIL';
console.error(`${failTag}: ${message}`);
process.exit(1);
}
}
main();

View File

@ -0,0 +1,60 @@
'use strict';
/**
* 本地自测桩:模拟远端只读探测输出,验证 check-root-logs.cjs 的
* --clean dry-run 删除计划逻辑(不连接任何真实服务器)。
*
* 用法:node script/tmp-test-clean-plan.cjs --clean
*/
const path = require('node:path');
const deployLibPath = require.resolve('./deploy-lib.cjs');
/**
* 构造模拟的远端探测输出:
* 总大小约 6.37 GiB(超过 4 GiB 阈值),包含活跃日志、7 个轮转日志、
* 其他命名模式的日志与 .gz 文件,用于验证排除规则与删除计划
*
* @return {string} 模拟的远端探测标准输出
*/
function buildFakeProbeOutput() {
const files = [
// 正在写入的活跃日志(无日期后缀),必须被排除
{ mtime: 1757780000, size: 1181116006, filePath: '/root/logs/wms/wms-web.log' },
// 历史轮转日志(mtime 升序,最旧的 4 个较大)
{ mtime: 1756150000, size: 1288490188, filePath: '/root/logs/wms/wms-web.log.2026-08-24' },
{ mtime: 1756236400, size: 1288490188, filePath: '/root/logs/wms/wms-web.log.2026-08-25' },
{ mtime: 1756322800, size: 1288490188, filePath: '/root/logs/wms/wms-web.log.2026-08-26' },
{ mtime: 1756409200, size: 1288490188, filePath: '/root/logs/wms/wms-web.log.2026-08-27' },
{ mtime: 1756495600, size: 214748365, filePath: '/root/logs/wms/wms-web.log.2026-08-28' },
{ mtime: 1756582000, size: 107374182, filePath: '/root/logs/wms/wms-web.log.2026-08-29' },
{ mtime: 1756668400, size: 107374182, filePath: '/root/logs/wms/wms-web.log.2026-08-30' },
// 非 wms-web 轮转命名模式的文件,必须被排除
{ mtime: 1756583000, size: 29360128, filePath: '/root/logs/wms/common-error.log.2026-08-29' },
{ mtime: 1756720000, size: 12582912, filePath: '/root/logs/wms/egg-schedule.log.2026-08-31' },
// .gz 后缀文件不符合严格命名模式,必须被排除
{ mtime: 1756730000, size: 31457280, filePath: '/root/logs/wms/wms-web.log.2026-08-31.gz' },
];
const totalBytes = files.reduce((sum, item) => sum + item.size, 0);
return [
`PROBE_TOTAL_BYTES=${totalBytes}`,
...files.map(item => `${item.mtime}\t${item.size}\t${item.filePath}`),
].join('\n');
}
// 将 deploy-lib.cjs 替换为本地桩,避免真实 SSH 连接
require.cache[deployLibPath] = {
id: deployLibPath,
filename: deployLibPath,
loaded: true,
exports: {
ensureCommandAvailable() {},
loadDeployConfig: () => ({ envName: 'production', username: 'root', host: '127.0.0.1', port: 22 }),
quoteForBash: value => `'${String(value)}'`,
runSsh: () => buildFakeProbeOutput(),
},
};
// 加载被测脚本(其 main() 会读取当前进程参数,即 --clean)
require(path.resolve(__dirname, 'check-root-logs.cjs'));