WMS_MAIN/script/method-output-regression.cjs
liufumingren d129103d0b refactor(log): 接口流水改用 Egg 日志落盘,整表下线 wms_system_log
背景:接口流水原本逐条写 wms_system_log 单表(生产 687,030 行、.ibd 28.8 GB、DATA_FREE 仅 5 MB),
删行无法把空间还给操作系统;且每个 needLog 请求都要多一次 INSERT。改为经 Egg 自定义 logger 平铺落盘,
整表 DROP 回收磁盘(生产实测释放 28.8 GB,/www 可用 20G→49G);日志目录同时由 /root/logs/wms
迁到 /www/wwwlogs/wms,不再占用系统盘。

新增 app/lib/api_log_sink.js:
- 一行一条 `HH:mm:ss {JSON}`:前缀是本地时区时分秒,只给人眼定位(tail/grep 的时间锚点),与行内
  created_at(UTC ISO 8601)是同一时刻;机器解析请用 created_at。
- 时间前缀刻意放在代码里而不是 config 的 formatter —— 生产 config 不参与 deploy、由人工维护,
  放代码才能随发布自动生效(此前已因「代码上线、配套配置没上」出现过接口日志静默丢失)。
- 对象/数组一律 JSON.stringify,避免调用方漏 stringify 时落盘成 "[object Object]"。
- customLogger.apiLog 缺失时每个 worker 显式告警一次:egg 的 getLogger(name) 对未声明的 name 返回
  null 而不抛错,不处理就会「既不落盘、也无任何报错」。

新增迁移 database/migrations/20260927090000-offline-wms-system-log.js:
- up 用 information_schema 探测后 DROP TABLE(幂等);该表对 wms_store 是出边外键,无需关外键检查。
- down 用原生 DDL 重建:queryInterface.createTable 在 MySQL 下默认字符集是 utf8mb3,且不会生成原始
  DDL 里的 created_at ON UPDATE CURRENT_TIMESTAMP,只有原生 DDL 才能忠实还原。
- 已在开发库跑通 up → down → up:重建后 14 列、主键、store_id 索引、外键、utf8mb4 均与下线前一致。

下线代码:删除 app/model/system_log.js、app/service/systems/system_log.js(60 天保留期清理服务)、
app/schedule/system_log_cleanup.js(每日 03:00 清理任务);移除 config.systemLog(writeToDb 回滚开关)
与 config.systemLogCleanup;api_log_sink 不再保留任何入库分支。

文档与脚本:AI_METHOD_ROUTING.md 第 6 节清单移除已下线的 systems/system_log.js(文件数 220→219);
script/method-output-regression.cjs 的回放样本来源由 `SELECT ... FROM wms_system_log` 改为读
wms-api.log(含按天轮转文件),并改为先整体 parse、失败再截到第一个 `{` 重试,兼容带/不带时间前缀
两种行(在真实 EggLogger 上验证:FileBufferTransport 每 1000ms flush,需等过 1s 才能读到)。

注:按日切割与 31 天清理由 egg 内置 egg-logrotator 自动完成(walkLoggerFile 覆盖 customLogger),
无需额外配 cron。
2026-09-27 03:05:15 +08:00

473 lines
14 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

const fs = require('fs');
const path = require('path');
const ROOT = process.cwd();
const OUTPUT_DIR = path.join(ROOT, 'temp', 'method-output');
const LEGACY_RUN_DIR = path.join(ROOT, 'run');
const BASELINE_FILE = path.join(OUTPUT_DIR, 'method-output-baseline.json');
const BASELINE_SUMMARY = path.join(OUTPUT_DIR, 'method-output-baseline.md');
const COMPARE_FILE = path.join(OUTPUT_DIR, 'method-output-compare.json');
const LEGACY_BASELINE_FILE = path.join(LEGACY_RUN_DIR, 'method-output-baseline.json');
const MODE = (process.argv[2] || 'baseline').toLowerCase();
const LIMIT = Number(process.argv[3] || 8);
const TEST_USERNAME = process.env.WMS_TEST_USER || 'liufumingren';
const TEST_PASSWORD = process.env.WMS_TEST_PASS || '111111';
const READONLY_HINT = /(get|list|query|detail|info|stats|tree|search|load)/i;
const MUTATION_HINT = /(create|update|delete|remove|audit|confirm|complete|add|sync|synchronize|upload|assign|change|logout|login)/i;
const VOLATILE_KEYS = new Set([
'timestamp',
'token',
'last_login_at',
'updated_at',
'update_at',
'created_at',
'create_at',
'login_time',
'iat',
'exp',
]);
function ensureOutputDir() {
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
}
}
function resolveBaselineFile() {
if (fs.existsSync(BASELINE_FILE)) return BASELINE_FILE;
if (fs.existsSync(LEGACY_BASELINE_FILE)) return LEGACY_BASELINE_FILE;
return BASELINE_FILE;
}
function readLocalConfig() {
const localCfgFactory = require(path.join(ROOT, 'config', 'config.local.js'));
const cfg = localCfgFactory({ name: 'wms' });
return cfg;
}
function buildBaseUrl(cfg) {
const host = cfg.cluster?.listen?.hostname && cfg.cluster.listen.hostname !== '0.0.0.0'
? cfg.cluster.listen.hostname
: '127.0.0.1';
const port = cfg.cluster?.listen?.port || 6060;
return `http://${host}:${port}`;
}
function loadSystemMethodDocs() {
const file = path.join(ROOT, 'methods', 'system.json');
const raw = fs.readFileSync(file, 'utf8');
const json = JSON.parse(raw);
const map = new Map();
const groups = json?.methods?.groups || [];
for (const group of groups) {
for (const item of group.list || []) {
map.set(item.method, {
groupName: group.groupName,
module: group.module,
description: item.description,
isNeedAuth: !!item.isNeedAuth,
isNeedLogin: !!item.isNeedLogin,
params: (item.params || []).map(p => ({
name: p.name,
type: p.type,
required: !!p.required,
source: p.source,
description: p.description,
})),
});
}
}
return map;
}
function safeJsonParse(text, fallback = null) {
if (!text || typeof text !== 'string') return fallback;
try {
return JSON.parse(text);
} catch {
return fallback;
}
}
function normalizeValue(value) {
if (value === null || value === undefined) return value;
if (Array.isArray(value)) return value.map(item => normalizeValue(item));
if (typeof value !== 'object') return value;
const out = {};
for (const key of Object.keys(value)) {
if (VOLATILE_KEYS.has(key)) {
out[key] = '__VOLATILE__';
continue;
}
out[key] = normalizeValue(value[key]);
}
return out;
}
function stableStringify(obj) {
return JSON.stringify(obj);
}
function pickPlatform(headers, rowDevice) {
const fromHeaders = String(headers?.platform || '').trim().toLowerCase();
if (fromHeaders) return fromHeaders;
const fromDevice = String(rowDevice || '').trim().toLowerCase();
if (fromDevice === 'desktop') return 'web';
if (fromDevice) return fromDevice;
return 'web';
}
function pickProject(headers) {
const project = String(headers?.project || '').trim().toLowerCase();
if (!project) return 'wms';
if (project === 'all') return 'wms';
return project;
}
/** 接口流水日志目录:优先取环境变量,其次为本地开发目录 <repo>/logs(与 config.logPaths.base 一致) */
function resolveApiLogDir() {
if (process.env.WMS_LOG_DIR) return process.env.WMS_LOG_DIR;
return path.join(ROOT, 'logs');
}
/**
* 解析一行接口日志。
*
* 行格式为 `HH:mm:ss {JSON}`(行首时分秒前缀由 app/lib/api_log_sink.js 写入)。
* 为兼容不带前缀的历史行,先尝试整体 parse,失败再截到第一个 `{` 重试。
*
* @param {string} line 原始行。
* @return {Object|null} 解析出的记录;不是有效对象时返回 null。
*/
function parseApiLogLine(line) {
const trimmed = String(line || '').trim();
if (!trimmed) return null;
const braceIndex = trimmed.indexOf('{');
const candidates = braceIndex > 0 ? [ trimmed, trimmed.slice(braceIndex) ] : [ trimmed ];
for (const text of candidates) {
const parsed = safeJsonParse(text, null);
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
return parsed;
}
}
return null;
}
/**
* 读取 wms-api.log 及其按天轮转的历史文件,按“最近优先”返回记录。
*
* 背景:接口流水原写入 wms_system_log 表,该表已下线
* (见 database/migrations/20260927090000-offline-wms-system-log.js),
* 现由 Egg 自定义 logger 落盘为一行一条的平铺 JSON,字段与旧表一一对应,
* 因此回放样本改为从日志文件读取;返回结构保持与旧 SQL 结果同形
* (method / body / headers / auth / device / result / created_at,并补 id 作为来源标识)。
*
* @return {Array<Object>} 记录列表,最近优先;目录/文件缺失时返回空数组。
*/
function loadRecentApiLogRows() {
const dir = resolveApiLogDir();
if (!fs.existsSync(dir)) return [];
const files = fs.readdirSync(dir)
.filter(name => name === 'wms-api.log' || name.startsWith('wms-api.log.'))
.map(name => ({
full: path.join(dir, name),
mtime: fs.statSync(path.join(dir, name)).mtimeMs,
}))
.sort((a, b) => a.mtime - b.mtime);
const rows = [];
for (const file of files) {
const text = fs.readFileSync(file.full, 'utf8');
for (const line of text.split('\n')) {
const row = parseApiLogLine(line);
// 日志行没有自增 id,用 created_at 充当来源标识
if (row && row.method) {
rows.push(Object.assign({ id: row.created_at }, row));
}
}
}
// 日志为追加写,倒序即“最近优先”,与原 SQL 的 ORDER BY id DESC 语义一致
return rows.reverse();
}
function loadRecentApiLogCases(methodDocs, limit, username) {
const rows = loadRecentApiLogRows();
const preferred = [];
const fallback = [];
const seen = new Set();
for (const row of rows) {
if (!methodDocs.has(row.method)) continue;
if (row.method === 'user.login' || row.method === 'user.logout' || row.method === 'user.changeRole') continue;
const body = safeJsonParse(row.body, null);
if (!body || typeof body !== 'object') continue;
const signature = `${row.method}::${JSON.stringify(body)}`;
if (seen.has(signature)) continue;
seen.add(signature);
const headers = safeJsonParse(row.headers, {});
const auth = safeJsonParse(row.auth, {});
const fromUser = String(auth?.username || '').trim().toLowerCase() === String(username || '').trim().toLowerCase();
const methodLooksReadOnly = READONLY_HINT.test(row.method) && !MUTATION_HINT.test(row.method);
const historySuccess = typeof row.result === 'string' && row.result.includes('"status":200');
const item = {
source: 'wms-api.log',
sourceId: row.id,
sourceCreatedAt: row.created_at,
method: row.method,
body,
headers: {
platform: pickPlatform(headers, row.device),
project: pickProject(headers),
},
fromUser,
readOnlyHint: methodLooksReadOnly,
historySuccess,
};
if (fromUser && methodLooksReadOnly && historySuccess) preferred.push(item);
else if (fromUser && methodLooksReadOnly) preferred.push(item);
else if (methodLooksReadOnly) fallback.push(item);
if (preferred.length >= limit * 2) break;
}
const result = [];
const methodSeen = new Set();
for (const item of preferred) {
if (result.length >= limit) break;
if (methodSeen.has(item.method)) continue;
result.push(item);
methodSeen.add(item.method);
}
for (const item of preferred) {
if (result.length >= limit) break;
result.push(item);
}
for (const item of fallback) {
if (result.length >= limit) break;
if (methodSeen.has(item.method)) continue;
result.push(item);
methodSeen.add(item.method);
}
for (const item of fallback) {
if (result.length >= limit) break;
result.push(item);
}
return result;
}
async function callSystemMethod(baseUrl, method, body, headers, token) {
const ts = Math.floor(Date.now() / 1000);
const url = `${baseUrl}/system?method=${encodeURIComponent(method)}&v=1&timestamp=${ts}`;
const reqHeaders = {
platform: headers.platform || 'web',
project: headers.project || 'wms',
'content-type': 'application/json',
};
if (token) {
reqHeaders.authorization = `Bearer ${token}`;
}
const resp = await fetch(url, {
method: 'POST',
headers: reqHeaders,
body: JSON.stringify(body || {}),
});
const text = await resp.text();
const data = safeJsonParse(text, { __raw_text__: text });
return {
httpStatus: resp.status,
response: data,
normalized: normalizeValue(data),
};
}
function extractToken(loginResponse) {
const direct = loginResponse?.response?.data?.token;
if (typeof direct === 'string' && direct.length > 10) return direct;
return '';
}
function renderBaselineMarkdown(baseline) {
let md = '';
md += '# method-output-baseline\n\n';
md += `GeneratedAt: ${baseline.generatedAt}\n`;
md += `BaseUrl: ${baseline.baseUrl}\n`;
md += `User: ${baseline.user}\n`;
md += `CaseCount: ${baseline.cases.length}\n\n`;
md += '| method | source | sourceId | status | group | note |\n';
md += '| --- | --- | --- | --- | --- | --- |\n';
for (const c of baseline.cases) {
md += `| ${c.method} | ${c.source} | ${c.sourceId} | ${c.httpStatus} | ${c.doc?.groupName || ''} | ${c.doc?.description || ''} |\n`;
}
md += '\n';
return md;
}
async function runBaseline() {
const cfg = readLocalConfig();
const baseUrl = buildBaseUrl(cfg);
const methodDocs = loadSystemMethodDocs();
const selected = loadRecentApiLogCases(methodDocs, LIMIT, TEST_USERNAME);
if (!selected.length) {
throw new Error(
`未从 ${path.join(resolveApiLogDir(), 'wms-api.log')} 找到可回放的只读 method 样本`
+ '(需先用真实账号跑一批只读接口,让日志里先有样本)'
);
}
const login = await callSystemMethod(
baseUrl,
'user.login',
{ username: TEST_USERNAME, password: TEST_PASSWORD },
{ platform: 'web', project: 'wms' },
''
);
const token = extractToken(login);
if (!token) {
throw new Error('登录成功但未拿到 token,请检查 user.login 返回结构');
}
const cases = [];
for (const item of selected) {
const call = await callSystemMethod(baseUrl, item.method, item.body, item.headers, token);
cases.push({
method: item.method,
source: item.source,
sourceId: item.sourceId,
sourceCreatedAt: item.sourceCreatedAt,
headers: item.headers,
requestBody: item.body,
doc: methodDocs.get(item.method) || null,
httpStatus: call.httpStatus,
response: call.response,
normalized: call.normalized,
});
}
const baseline = {
generatedAt: new Date().toISOString(),
baseUrl,
user: TEST_USERNAME,
note: 'Before-change baseline generated from recent wms-api.log samples and methods/system.json',
login: {
httpStatus: login.httpStatus,
normalized: login.normalized,
},
cases,
};
ensureOutputDir();
fs.writeFileSync(BASELINE_FILE, JSON.stringify(baseline, null, 2), 'utf8');
fs.writeFileSync(BASELINE_SUMMARY, renderBaselineMarkdown(baseline), 'utf8');
console.log(`BASELINE_OK cases=${cases.length}`);
console.log(`BASELINE_FILE ${BASELINE_FILE}`);
console.log(`BASELINE_SUMMARY ${BASELINE_SUMMARY}`);
}
async function runCompare() {
const baselineFile = resolveBaselineFile();
if (!fs.existsSync(baselineFile)) {
throw new Error(`baseline 文件不存在: ${BASELINE_FILE}`);
}
const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));
const cfg = readLocalConfig();
const baseUrl = buildBaseUrl(cfg);
const login = await callSystemMethod(
baseUrl,
'user.login',
{ username: TEST_USERNAME, password: TEST_PASSWORD },
{ platform: 'web', project: 'wms' },
''
);
const token = extractToken(login);
if (!token) {
throw new Error('compare 阶段登录后未拿到 token');
}
const compared = [];
let mismatchCount = 0;
for (const item of baseline.cases || []) {
const call = await callSystemMethod(
baseUrl,
item.method,
item.requestBody,
item.headers || { platform: 'web', project: 'wms' },
token
);
const beforeNorm = item.normalized;
const afterNorm = call.normalized;
const same = stableStringify(beforeNorm) === stableStringify(afterNorm);
if (!same) mismatchCount++;
compared.push({
method: item.method,
sourceId: item.sourceId,
httpStatusBefore: item.httpStatus,
httpStatusAfter: call.httpStatus,
outputEqual: same,
beforeNormalized: beforeNorm,
afterNormalized: afterNorm,
});
}
const report = {
comparedAt: new Date().toISOString(),
baseUrl,
user: TEST_USERNAME,
baselineFile,
total: compared.length,
mismatchCount,
pass: mismatchCount === 0,
items: compared,
};
ensureOutputDir();
fs.writeFileSync(COMPARE_FILE, JSON.stringify(report, null, 2), 'utf8');
console.log(`COMPARE_DONE total=${report.total} mismatch=${report.mismatchCount}`);
console.log(`COMPARE_FILE ${COMPARE_FILE}`);
if (report.mismatchCount > 0) {
process.exitCode = 1;
}
}
async function main() {
if (MODE !== 'baseline' && MODE !== 'compare') {
throw new Error('用法: node script/method-output-regression.cjs <baseline|compare> [limit]');
}
if (MODE === 'baseline') {
await runBaseline();
} else {
await runCompare();
}
}
main().catch(err => {
console.error(err);
process.exit(1);
});