背景:接口流水原本逐条写 wms_system_log 单表(生产 687,030 行、.ibd 28.8 GB、DATA_FREE 仅 5 MB),
删行无法把空间还给操作系统;且每个 needLog 请求都要多一次 INSERT。改为经 Egg 自定义 logger 平铺落盘,
整表 DROP 回收磁盘(生产实测释放 28.8 GB,/www 可用 20G→49G);日志目录同时由 /root/logs/wms
迁到 /www/wwwlogs/wms,不再占用系统盘。
新增 app/lib/api_log_sink.js:
- 一行一条 `HH:mm:ss {JSON}`:前缀是本地时区时分秒,只给人眼定位(tail/grep 的时间锚点),与行内
created_at(UTC ISO 8601)是同一时刻;机器解析请用 created_at。
- 时间前缀刻意放在代码里而不是 config 的 formatter —— 生产 config 不参与 deploy、由人工维护,
放代码才能随发布自动生效(此前已因「代码上线、配套配置没上」出现过接口日志静默丢失)。
- 对象/数组一律 JSON.stringify,避免调用方漏 stringify 时落盘成 "[object Object]"。
- customLogger.apiLog 缺失时每个 worker 显式告警一次:egg 的 getLogger(name) 对未声明的 name 返回
null 而不抛错,不处理就会「既不落盘、也无任何报错」。
新增迁移 database/migrations/20260927090000-offline-wms-system-log.js:
- up 用 information_schema 探测后 DROP TABLE(幂等);该表对 wms_store 是出边外键,无需关外键检查。
- down 用原生 DDL 重建:queryInterface.createTable 在 MySQL 下默认字符集是 utf8mb3,且不会生成原始
DDL 里的 created_at ON UPDATE CURRENT_TIMESTAMP,只有原生 DDL 才能忠实还原。
- 已在开发库跑通 up → down → up:重建后 14 列、主键、store_id 索引、外键、utf8mb4 均与下线前一致。
下线代码:删除 app/model/system_log.js、app/service/systems/system_log.js(60 天保留期清理服务)、
app/schedule/system_log_cleanup.js(每日 03:00 清理任务);移除 config.systemLog(writeToDb 回滚开关)
与 config.systemLogCleanup;api_log_sink 不再保留任何入库分支。
文档与脚本:AI_METHOD_ROUTING.md 第 6 节清单移除已下线的 systems/system_log.js(文件数 220→219);
script/method-output-regression.cjs 的回放样本来源由 `SELECT ... FROM wms_system_log` 改为读
wms-api.log(含按天轮转文件),并改为先整体 parse、失败再截到第一个 `{` 重试,兼容带/不带时间前缀
两种行(在真实 EggLogger 上验证:FileBufferTransport 每 1000ms flush,需等过 1s 才能读到)。
注:按日切割与 31 天清理由 egg 内置 egg-logrotator 自动完成(walkLoggerFile 覆盖 customLogger),
无需额外配 cron。
473 lines
14 KiB
JavaScript
473 lines
14 KiB
JavaScript
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const ROOT = process.cwd();
|
||
const OUTPUT_DIR = path.join(ROOT, 'temp', 'method-output');
|
||
const LEGACY_RUN_DIR = path.join(ROOT, 'run');
|
||
const BASELINE_FILE = path.join(OUTPUT_DIR, 'method-output-baseline.json');
|
||
const BASELINE_SUMMARY = path.join(OUTPUT_DIR, 'method-output-baseline.md');
|
||
const COMPARE_FILE = path.join(OUTPUT_DIR, 'method-output-compare.json');
|
||
const LEGACY_BASELINE_FILE = path.join(LEGACY_RUN_DIR, 'method-output-baseline.json');
|
||
|
||
const MODE = (process.argv[2] || 'baseline').toLowerCase();
|
||
const LIMIT = Number(process.argv[3] || 8);
|
||
const TEST_USERNAME = process.env.WMS_TEST_USER || 'liufumingren';
|
||
const TEST_PASSWORD = process.env.WMS_TEST_PASS || '111111';
|
||
|
||
const READONLY_HINT = /(get|list|query|detail|info|stats|tree|search|load)/i;
|
||
const MUTATION_HINT = /(create|update|delete|remove|audit|confirm|complete|add|sync|synchronize|upload|assign|change|logout|login)/i;
|
||
const VOLATILE_KEYS = new Set([
|
||
'timestamp',
|
||
'token',
|
||
'last_login_at',
|
||
'updated_at',
|
||
'update_at',
|
||
'created_at',
|
||
'create_at',
|
||
'login_time',
|
||
'iat',
|
||
'exp',
|
||
]);
|
||
|
||
function ensureOutputDir() {
|
||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||
}
|
||
}
|
||
|
||
function resolveBaselineFile() {
|
||
if (fs.existsSync(BASELINE_FILE)) return BASELINE_FILE;
|
||
if (fs.existsSync(LEGACY_BASELINE_FILE)) return LEGACY_BASELINE_FILE;
|
||
return BASELINE_FILE;
|
||
}
|
||
|
||
function readLocalConfig() {
|
||
const localCfgFactory = require(path.join(ROOT, 'config', 'config.local.js'));
|
||
const cfg = localCfgFactory({ name: 'wms' });
|
||
return cfg;
|
||
}
|
||
|
||
function buildBaseUrl(cfg) {
|
||
const host = cfg.cluster?.listen?.hostname && cfg.cluster.listen.hostname !== '0.0.0.0'
|
||
? cfg.cluster.listen.hostname
|
||
: '127.0.0.1';
|
||
const port = cfg.cluster?.listen?.port || 6060;
|
||
return `http://${host}:${port}`;
|
||
}
|
||
|
||
function loadSystemMethodDocs() {
|
||
const file = path.join(ROOT, 'methods', 'system.json');
|
||
const raw = fs.readFileSync(file, 'utf8');
|
||
const json = JSON.parse(raw);
|
||
const map = new Map();
|
||
const groups = json?.methods?.groups || [];
|
||
for (const group of groups) {
|
||
for (const item of group.list || []) {
|
||
map.set(item.method, {
|
||
groupName: group.groupName,
|
||
module: group.module,
|
||
description: item.description,
|
||
isNeedAuth: !!item.isNeedAuth,
|
||
isNeedLogin: !!item.isNeedLogin,
|
||
params: (item.params || []).map(p => ({
|
||
name: p.name,
|
||
type: p.type,
|
||
required: !!p.required,
|
||
source: p.source,
|
||
description: p.description,
|
||
})),
|
||
});
|
||
}
|
||
}
|
||
return map;
|
||
}
|
||
|
||
function safeJsonParse(text, fallback = null) {
|
||
if (!text || typeof text !== 'string') return fallback;
|
||
try {
|
||
return JSON.parse(text);
|
||
} catch {
|
||
return fallback;
|
||
}
|
||
}
|
||
|
||
function normalizeValue(value) {
|
||
if (value === null || value === undefined) return value;
|
||
if (Array.isArray(value)) return value.map(item => normalizeValue(item));
|
||
if (typeof value !== 'object') return value;
|
||
|
||
const out = {};
|
||
for (const key of Object.keys(value)) {
|
||
if (VOLATILE_KEYS.has(key)) {
|
||
out[key] = '__VOLATILE__';
|
||
continue;
|
||
}
|
||
out[key] = normalizeValue(value[key]);
|
||
}
|
||
return out;
|
||
}
|
||
|
||
function stableStringify(obj) {
|
||
return JSON.stringify(obj);
|
||
}
|
||
|
||
function pickPlatform(headers, rowDevice) {
|
||
const fromHeaders = String(headers?.platform || '').trim().toLowerCase();
|
||
if (fromHeaders) return fromHeaders;
|
||
const fromDevice = String(rowDevice || '').trim().toLowerCase();
|
||
if (fromDevice === 'desktop') return 'web';
|
||
if (fromDevice) return fromDevice;
|
||
return 'web';
|
||
}
|
||
|
||
function pickProject(headers) {
|
||
const project = String(headers?.project || '').trim().toLowerCase();
|
||
if (!project) return 'wms';
|
||
if (project === 'all') return 'wms';
|
||
return project;
|
||
}
|
||
|
||
/** 接口流水日志目录:优先取环境变量,其次为本地开发目录 <repo>/logs(与 config.logPaths.base 一致) */
|
||
function resolveApiLogDir() {
|
||
if (process.env.WMS_LOG_DIR) return process.env.WMS_LOG_DIR;
|
||
return path.join(ROOT, 'logs');
|
||
}
|
||
|
||
/**
|
||
* 解析一行接口日志。
|
||
*
|
||
* 行格式为 `HH:mm:ss {JSON}`(行首时分秒前缀由 app/lib/api_log_sink.js 写入)。
|
||
* 为兼容不带前缀的历史行,先尝试整体 parse,失败再截到第一个 `{` 重试。
|
||
*
|
||
* @param {string} line 原始行。
|
||
* @return {Object|null} 解析出的记录;不是有效对象时返回 null。
|
||
*/
|
||
function parseApiLogLine(line) {
|
||
const trimmed = String(line || '').trim();
|
||
if (!trimmed) return null;
|
||
const braceIndex = trimmed.indexOf('{');
|
||
const candidates = braceIndex > 0 ? [ trimmed, trimmed.slice(braceIndex) ] : [ trimmed ];
|
||
for (const text of candidates) {
|
||
const parsed = safeJsonParse(text, null);
|
||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||
return parsed;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 读取 wms-api.log 及其按天轮转的历史文件,按“最近优先”返回记录。
|
||
*
|
||
* 背景:接口流水原写入 wms_system_log 表,该表已下线
|
||
* (见 database/migrations/20260927090000-offline-wms-system-log.js),
|
||
* 现由 Egg 自定义 logger 落盘为一行一条的平铺 JSON,字段与旧表一一对应,
|
||
* 因此回放样本改为从日志文件读取;返回结构保持与旧 SQL 结果同形
|
||
* (method / body / headers / auth / device / result / created_at,并补 id 作为来源标识)。
|
||
*
|
||
* @return {Array<Object>} 记录列表,最近优先;目录/文件缺失时返回空数组。
|
||
*/
|
||
function loadRecentApiLogRows() {
|
||
const dir = resolveApiLogDir();
|
||
if (!fs.existsSync(dir)) return [];
|
||
|
||
const files = fs.readdirSync(dir)
|
||
.filter(name => name === 'wms-api.log' || name.startsWith('wms-api.log.'))
|
||
.map(name => ({
|
||
full: path.join(dir, name),
|
||
mtime: fs.statSync(path.join(dir, name)).mtimeMs,
|
||
}))
|
||
.sort((a, b) => a.mtime - b.mtime);
|
||
|
||
const rows = [];
|
||
for (const file of files) {
|
||
const text = fs.readFileSync(file.full, 'utf8');
|
||
for (const line of text.split('\n')) {
|
||
const row = parseApiLogLine(line);
|
||
// 日志行没有自增 id,用 created_at 充当来源标识
|
||
if (row && row.method) {
|
||
rows.push(Object.assign({ id: row.created_at }, row));
|
||
}
|
||
}
|
||
}
|
||
|
||
// 日志为追加写,倒序即“最近优先”,与原 SQL 的 ORDER BY id DESC 语义一致
|
||
return rows.reverse();
|
||
}
|
||
|
||
function loadRecentApiLogCases(methodDocs, limit, username) {
|
||
const rows = loadRecentApiLogRows();
|
||
|
||
const preferred = [];
|
||
const fallback = [];
|
||
const seen = new Set();
|
||
|
||
for (const row of rows) {
|
||
if (!methodDocs.has(row.method)) continue;
|
||
if (row.method === 'user.login' || row.method === 'user.logout' || row.method === 'user.changeRole') continue;
|
||
|
||
const body = safeJsonParse(row.body, null);
|
||
if (!body || typeof body !== 'object') continue;
|
||
|
||
const signature = `${row.method}::${JSON.stringify(body)}`;
|
||
if (seen.has(signature)) continue;
|
||
seen.add(signature);
|
||
|
||
const headers = safeJsonParse(row.headers, {});
|
||
const auth = safeJsonParse(row.auth, {});
|
||
const fromUser = String(auth?.username || '').trim().toLowerCase() === String(username || '').trim().toLowerCase();
|
||
const methodLooksReadOnly = READONLY_HINT.test(row.method) && !MUTATION_HINT.test(row.method);
|
||
const historySuccess = typeof row.result === 'string' && row.result.includes('"status":200');
|
||
|
||
const item = {
|
||
source: 'wms-api.log',
|
||
sourceId: row.id,
|
||
sourceCreatedAt: row.created_at,
|
||
method: row.method,
|
||
body,
|
||
headers: {
|
||
platform: pickPlatform(headers, row.device),
|
||
project: pickProject(headers),
|
||
},
|
||
fromUser,
|
||
readOnlyHint: methodLooksReadOnly,
|
||
historySuccess,
|
||
};
|
||
|
||
if (fromUser && methodLooksReadOnly && historySuccess) preferred.push(item);
|
||
else if (fromUser && methodLooksReadOnly) preferred.push(item);
|
||
else if (methodLooksReadOnly) fallback.push(item);
|
||
|
||
if (preferred.length >= limit * 2) break;
|
||
}
|
||
|
||
const result = [];
|
||
const methodSeen = new Set();
|
||
|
||
for (const item of preferred) {
|
||
if (result.length >= limit) break;
|
||
if (methodSeen.has(item.method)) continue;
|
||
result.push(item);
|
||
methodSeen.add(item.method);
|
||
}
|
||
|
||
for (const item of preferred) {
|
||
if (result.length >= limit) break;
|
||
result.push(item);
|
||
}
|
||
|
||
for (const item of fallback) {
|
||
if (result.length >= limit) break;
|
||
if (methodSeen.has(item.method)) continue;
|
||
result.push(item);
|
||
methodSeen.add(item.method);
|
||
}
|
||
|
||
for (const item of fallback) {
|
||
if (result.length >= limit) break;
|
||
result.push(item);
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
async function callSystemMethod(baseUrl, method, body, headers, token) {
|
||
const ts = Math.floor(Date.now() / 1000);
|
||
const url = `${baseUrl}/system?method=${encodeURIComponent(method)}&v=1×tamp=${ts}`;
|
||
|
||
const reqHeaders = {
|
||
platform: headers.platform || 'web',
|
||
project: headers.project || 'wms',
|
||
'content-type': 'application/json',
|
||
};
|
||
if (token) {
|
||
reqHeaders.authorization = `Bearer ${token}`;
|
||
}
|
||
|
||
const resp = await fetch(url, {
|
||
method: 'POST',
|
||
headers: reqHeaders,
|
||
body: JSON.stringify(body || {}),
|
||
});
|
||
|
||
const text = await resp.text();
|
||
const data = safeJsonParse(text, { __raw_text__: text });
|
||
return {
|
||
httpStatus: resp.status,
|
||
response: data,
|
||
normalized: normalizeValue(data),
|
||
};
|
||
}
|
||
|
||
function extractToken(loginResponse) {
|
||
const direct = loginResponse?.response?.data?.token;
|
||
if (typeof direct === 'string' && direct.length > 10) return direct;
|
||
return '';
|
||
}
|
||
|
||
function renderBaselineMarkdown(baseline) {
|
||
let md = '';
|
||
md += '# method-output-baseline\n\n';
|
||
md += `GeneratedAt: ${baseline.generatedAt}\n`;
|
||
md += `BaseUrl: ${baseline.baseUrl}\n`;
|
||
md += `User: ${baseline.user}\n`;
|
||
md += `CaseCount: ${baseline.cases.length}\n\n`;
|
||
md += '| method | source | sourceId | status | group | note |\n';
|
||
md += '| --- | --- | --- | --- | --- | --- |\n';
|
||
for (const c of baseline.cases) {
|
||
md += `| ${c.method} | ${c.source} | ${c.sourceId} | ${c.httpStatus} | ${c.doc?.groupName || ''} | ${c.doc?.description || ''} |\n`;
|
||
}
|
||
md += '\n';
|
||
return md;
|
||
}
|
||
|
||
async function runBaseline() {
|
||
const cfg = readLocalConfig();
|
||
const baseUrl = buildBaseUrl(cfg);
|
||
const methodDocs = loadSystemMethodDocs();
|
||
|
||
const selected = loadRecentApiLogCases(methodDocs, LIMIT, TEST_USERNAME);
|
||
if (!selected.length) {
|
||
throw new Error(
|
||
`未从 ${path.join(resolveApiLogDir(), 'wms-api.log')} 找到可回放的只读 method 样本`
|
||
+ '(需先用真实账号跑一批只读接口,让日志里先有样本)'
|
||
);
|
||
}
|
||
|
||
const login = await callSystemMethod(
|
||
baseUrl,
|
||
'user.login',
|
||
{ username: TEST_USERNAME, password: TEST_PASSWORD },
|
||
{ platform: 'web', project: 'wms' },
|
||
''
|
||
);
|
||
const token = extractToken(login);
|
||
if (!token) {
|
||
throw new Error('登录成功但未拿到 token,请检查 user.login 返回结构');
|
||
}
|
||
|
||
const cases = [];
|
||
for (const item of selected) {
|
||
const call = await callSystemMethod(baseUrl, item.method, item.body, item.headers, token);
|
||
cases.push({
|
||
method: item.method,
|
||
source: item.source,
|
||
sourceId: item.sourceId,
|
||
sourceCreatedAt: item.sourceCreatedAt,
|
||
headers: item.headers,
|
||
requestBody: item.body,
|
||
doc: methodDocs.get(item.method) || null,
|
||
httpStatus: call.httpStatus,
|
||
response: call.response,
|
||
normalized: call.normalized,
|
||
});
|
||
}
|
||
|
||
const baseline = {
|
||
generatedAt: new Date().toISOString(),
|
||
baseUrl,
|
||
user: TEST_USERNAME,
|
||
note: 'Before-change baseline generated from recent wms-api.log samples and methods/system.json',
|
||
login: {
|
||
httpStatus: login.httpStatus,
|
||
normalized: login.normalized,
|
||
},
|
||
cases,
|
||
};
|
||
|
||
ensureOutputDir();
|
||
fs.writeFileSync(BASELINE_FILE, JSON.stringify(baseline, null, 2), 'utf8');
|
||
fs.writeFileSync(BASELINE_SUMMARY, renderBaselineMarkdown(baseline), 'utf8');
|
||
|
||
console.log(`BASELINE_OK cases=${cases.length}`);
|
||
console.log(`BASELINE_FILE ${BASELINE_FILE}`);
|
||
console.log(`BASELINE_SUMMARY ${BASELINE_SUMMARY}`);
|
||
}
|
||
|
||
async function runCompare() {
|
||
const baselineFile = resolveBaselineFile();
|
||
if (!fs.existsSync(baselineFile)) {
|
||
throw new Error(`baseline 文件不存在: ${BASELINE_FILE}`);
|
||
}
|
||
|
||
const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));
|
||
const cfg = readLocalConfig();
|
||
const baseUrl = buildBaseUrl(cfg);
|
||
|
||
const login = await callSystemMethod(
|
||
baseUrl,
|
||
'user.login',
|
||
{ username: TEST_USERNAME, password: TEST_PASSWORD },
|
||
{ platform: 'web', project: 'wms' },
|
||
''
|
||
);
|
||
const token = extractToken(login);
|
||
if (!token) {
|
||
throw new Error('compare 阶段登录后未拿到 token');
|
||
}
|
||
|
||
const compared = [];
|
||
let mismatchCount = 0;
|
||
|
||
for (const item of baseline.cases || []) {
|
||
const call = await callSystemMethod(
|
||
baseUrl,
|
||
item.method,
|
||
item.requestBody,
|
||
item.headers || { platform: 'web', project: 'wms' },
|
||
token
|
||
);
|
||
|
||
const beforeNorm = item.normalized;
|
||
const afterNorm = call.normalized;
|
||
const same = stableStringify(beforeNorm) === stableStringify(afterNorm);
|
||
if (!same) mismatchCount++;
|
||
|
||
compared.push({
|
||
method: item.method,
|
||
sourceId: item.sourceId,
|
||
httpStatusBefore: item.httpStatus,
|
||
httpStatusAfter: call.httpStatus,
|
||
outputEqual: same,
|
||
beforeNormalized: beforeNorm,
|
||
afterNormalized: afterNorm,
|
||
});
|
||
}
|
||
|
||
const report = {
|
||
comparedAt: new Date().toISOString(),
|
||
baseUrl,
|
||
user: TEST_USERNAME,
|
||
baselineFile,
|
||
total: compared.length,
|
||
mismatchCount,
|
||
pass: mismatchCount === 0,
|
||
items: compared,
|
||
};
|
||
|
||
ensureOutputDir();
|
||
fs.writeFileSync(COMPARE_FILE, JSON.stringify(report, null, 2), 'utf8');
|
||
|
||
console.log(`COMPARE_DONE total=${report.total} mismatch=${report.mismatchCount}`);
|
||
console.log(`COMPARE_FILE ${COMPARE_FILE}`);
|
||
if (report.mismatchCount > 0) {
|
||
process.exitCode = 1;
|
||
}
|
||
}
|
||
|
||
async function main() {
|
||
if (MODE !== 'baseline' && MODE !== 'compare') {
|
||
throw new Error('用法: node script/method-output-regression.cjs <baseline|compare> [limit]');
|
||
}
|
||
if (MODE === 'baseline') {
|
||
await runBaseline();
|
||
} else {
|
||
await runCompare();
|
||
}
|
||
}
|
||
|
||
main().catch(err => {
|
||
console.error(err);
|
||
process.exit(1);
|
||
});
|