2026-04-21 18:35:09 +08:00

1197 lines
39 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { mkdir, readdir, readFile, writeFile, stat } from "node:fs/promises";
import path from "node:path";
import { crawlApi } from "../apidata/crawler.mjs";
const APIDATA_DIR = path.join("workflows", "api-doc-system", "apidata");
const APIDATA_CONFIG_FILE = path.join(APIDATA_DIR, "config.json");
const APIDATA_DATA_FILE = path.join(APIDATA_DIR, "data.json");
const RESPONSE_MARKDOWN_FILE = path.join("workflows", "api-doc-agent", "config", "响应.md");
const MODULE_DATA_ROOT = path.join("workflows", "api-doc-system", "data", "module");
const MODULE_LIST_FILE = path.join("workflows", "api-doc-system", "data", "module-list", "module-list.json");
const CRAWLER_CONFIG = {
baseUrl: "http://192.168.11.56:6060/system",
version: "1.0",
/** 普通抓取:单接口 fetch 上限 */
timeoutMs: 20000,
/** --quick 全量顺序请求时服务端易排队,单接口默认放宽(可被 config.requestTimeoutMs 覆盖) */
quickModeTimeoutMs: 60000,
/** 单请求失败(fetch failed / timeout / 5xx)自动重试次数(不含首次) */
defaultMaxRetries: 2,
quickMaxRetries: 3,
/** 连续请求之间的间隔(ms),防止服务端雪崩 */
defaultDelayMs: 0,
quickDelayMs: 120,
/** 重试退避起始时间(ms),指数退避上限 */
retryBackoffMsBase: 500,
retryBackoffMsMax: 5000,
};
/** @param {number} ms */
function clampRequestTimeoutMs(ms) {
const n = Math.floor(ms);
if (!Number.isFinite(n) || n < 1000) {
return CRAWLER_CONFIG.timeoutMs;
}
return Math.min(n, 900000);
}
/**
* apidata/config.json 根级可选 requestTimeoutMs(毫秒)覆盖默认;否则 quick 用 quickModeTimeoutMs。
* CLI 的 --timeout=<ms> 优先级最高。
* @param {{ rawConfig: object, quickCheck: boolean, cliTimeoutMs?: number | null }} args
*/
function resolveCrawlTimeoutMs({ rawConfig, quickCheck, cliTimeoutMs = null }) {
if (Number.isFinite(cliTimeoutMs) && cliTimeoutMs > 0) {
return clampRequestTimeoutMs(cliTimeoutMs);
}
const fromFile = Number(rawConfig?.requestTimeoutMs);
if (Number.isFinite(fromFile) && fromFile > 0) {
return clampRequestTimeoutMs(fromFile);
}
return quickCheck ? CRAWLER_CONFIG.quickModeTimeoutMs : CRAWLER_CONFIG.timeoutMs;
}
/** @param {number} n */
function clampNonNegativeInt(n) {
const v = Math.floor(Number(n));
if (!Number.isFinite(v) || v < 0) {
return 0;
}
return v;
}
/**
* @param {{ rawConfig: object, quickCheck: boolean, cliDelayMs?: number | null }} args
*/
function resolveRequestDelayMs({ rawConfig, quickCheck, cliDelayMs = null }) {
if (Number.isFinite(cliDelayMs) && cliDelayMs >= 0) {
return clampNonNegativeInt(cliDelayMs);
}
const v = Number(rawConfig?.requestDelayMs);
if (Number.isFinite(v) && v >= 0) {
return clampNonNegativeInt(v);
}
return quickCheck ? CRAWLER_CONFIG.quickDelayMs : CRAWLER_CONFIG.defaultDelayMs;
}
/**
* @param {{ rawConfig: object, quickCheck: boolean, cliMaxRetries?: number | null }} args
*/
function resolveMaxRetries({ rawConfig, quickCheck, cliMaxRetries = null }) {
if (Number.isFinite(cliMaxRetries) && cliMaxRetries >= 0) {
return clampNonNegativeInt(cliMaxRetries);
}
const v = Number(rawConfig?.maxRetries);
if (Number.isFinite(v) && v >= 0) {
return clampNonNegativeInt(v);
}
return quickCheck ? CRAWLER_CONFIG.quickMaxRetries : CRAWLER_CONFIG.defaultMaxRetries;
}
/**
* 连续超时熔断阈值,默认 5(表示连续 5 个接口超时即中止抓取)。
* 可通过 apidata/config.json 根级 maxConsecutiveTimeouts 或 CLI --timeout-streak 覆盖。
* @param {{ rawConfig: object, cliTimeoutStreak?: number | null }} args
*/
function resolveMaxConsecutiveTimeouts({ rawConfig, cliTimeoutStreak = null }) {
const clampMinOne = (n) => Math.max(1, clampNonNegativeInt(n));
if (Number.isFinite(cliTimeoutStreak) && cliTimeoutStreak > 0) {
return clampMinOne(cliTimeoutStreak);
}
const v = Number(rawConfig?.maxConsecutiveTimeouts);
if (Number.isFinite(v) && v > 0) {
return clampMinOne(v);
}
return 5;
}
function sleepMs(ms) {
if (!Number.isFinite(ms) || ms <= 0) {
return Promise.resolve();
}
return new Promise((r) => setTimeout(r, ms));
}
/** 瞬时网络错误/超时/5xx 视为可重试 */
function isRetryableCrawlError(err) {
const name = String(err?.name ?? "");
const msg = String(err?.message ?? err ?? "");
if (name === "TimeoutError" || /timeout/i.test(msg)) {
return true;
}
if (/fetch failed|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|EPIPE|socket hang up/i.test(msg)) {
return true;
}
return false;
}
function isRetryableStatus(status) {
return status === 0 || status === 408 || status === 425 || status === 429 || (status >= 500 && status <= 599);
}
/**
* 仅把“请求超时”归入连续超时计数,普通网络错误不计入。
* @param {{ status: number, error?: string }} result
*/
function isTimeoutFailureResult(result) {
const errMsg = String(result?.error ?? "");
if (/timeout|timed out|请求超时/i.test(errMsg)) {
return true;
}
return Number(result?.status) === 408;
}
function backoffDelayMs(attempt) {
const base = CRAWLER_CONFIG.retryBackoffMsBase;
const max = CRAWLER_CONFIG.retryBackoffMsMax;
const exp = Math.min(max, base * 2 ** attempt);
const jitter = Math.floor(Math.random() * Math.min(300, exp));
return exp + jitter;
}
/**
* 带重试的 crawlApi 封装:捕获网络异常与可重试状态码
* @returns {Promise<{ ok: boolean, status: number, response: unknown, error?: string, attempts: number, elapsedMs: number }>}
*/
async function crawlApiWithRetry({ method, body, token, timeoutMs, maxRetries }) {
const started = Date.now();
let lastError = null;
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
try {
const r = await crawlApi({
baseUrl: CRAWLER_CONFIG.baseUrl,
version: CRAWLER_CONFIG.version,
method,
body,
timeoutMs,
token,
});
if (!r.ok && isRetryableStatus(r.status) && attempt < maxRetries) {
await sleepMs(backoffDelayMs(attempt));
continue;
}
return {
ok: r.ok,
status: r.status,
response: r.response,
attempts: attempt + 1,
elapsedMs: Date.now() - started,
};
} catch (err) {
lastError = err;
if (attempt < maxRetries && isRetryableCrawlError(err)) {
await sleepMs(backoffDelayMs(attempt));
continue;
}
return {
ok: false,
status: 0,
response: null,
error: String(err?.message ?? err),
attempts: attempt + 1,
elapsedMs: Date.now() - started,
};
}
}
return {
ok: false,
status: 0,
response: null,
error: String(lastError?.message ?? lastError ?? "unknown"),
attempts: maxRetries + 1,
elapsedMs: Date.now() - started,
};
}
const LOGIN_METHOD = "user.login";
/** apidata/config.json 未配置 successMessages 时使用;可在 config 中写 successMessages 数组覆盖 */
const DEFAULT_SUCCESS_MESSAGES = ["ok", "成功", "登录成功",];
function normalizeSuccessMessageKey(value) {
return String(value ?? "")
.trim()
.toLowerCase();
}
function resolveSuccessMessages(rawConfig) {
const arr = rawConfig?.successMessages;
if (Array.isArray(arr) && arr.length > 0) {
const cleaned = arr.map((x) => String(x).trim()).filter(Boolean);
if (cleaned.length > 0) {
return cleaned;
}
}
return [...DEFAULT_SUCCESS_MESSAGES];
}
/** message 与成功名单某项在 trim + 小写比较下一致则视为成功(英文 ok 与 OK 等价) */
function isResponseMessageOk(message, successMessages) {
const key = normalizeSuccessMessageKey(message);
if (!key) {
return false;
}
const allow = new Set(successMessages.map((x) => normalizeSuccessMessageKey(x)).filter(Boolean));
return allow.has(key);
}
/**
* 成功响应中 data 含 list 且列表无元素([] 或 null)时视为「空数据」。
* 无 list 字段的接口(如登录)不归入此类。
* @param {unknown} data
*/
function isDataWithEmptyList(data) {
if (!data || typeof data !== "object" || Array.isArray(data)) {
return false;
}
const obj = /** @type {Record<string, unknown>} */ (data);
if (!Object.prototype.hasOwnProperty.call(obj, "list")) {
return false;
}
const list = obj.list;
if (list == null) {
return true;
}
return Array.isArray(list) && list.length === 0;
}
/**
* @param {Array<{ 接口: string, message: string, code?: unknown, data: unknown }>} rows
* @param {string[]} successMessages
*/
function buildDataOverview(rows, successMessages) {
const total = rows.length;
let success = 0;
/** @type {{ 接口: string, message: string, code: unknown }[]} */
const 失败接口 = [];
/** @type {{ 接口: string, message: string, code: unknown }[]} */
const 空数据接口 = [];
for (const row of rows) {
if (isResponseMessageOk(row.message, successMessages)) {
success += 1;
if (isDataWithEmptyList(row.data)) {
空数据接口.push({
接口: row.接口,
message: String(row.message ?? ""),
code: row.code ?? null,
});
}
} else {
失败接口.push({
接口: row.接口,
message: String(row.message ?? ""),
code: row.code ?? null,
});
}
}
return {
当前请求数: total,
成功数: success,
全部请求数: total,
成功数与全部请求数: `${success}/${total}`,
成功判定名单: [...successMessages],
失败接口,
空数据接口,
};
}
function ensurePlainObject(value, fallbackValue = {}) {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return fallbackValue;
}
return value;
}
async function readJson(filePath) {
const content = await readFile(filePath, "utf8");
return JSON.parse(content);
}
async function writeJson(filePath, data) {
await mkdir(path.dirname(filePath), { recursive: true });
await writeFile(filePath, `${JSON.stringify(data, null, 2)}\n`, "utf8");
}
function normalizeConfigRequest(item) {
const method = String(item?.method ?? "").trim();
if (!method) {
return null;
}
return {
method,
body: ensurePlainObject(item?.body, {}),
needAuth: item?.needAuth ?? method !== LOGIN_METHOD,
};
}
function normalizeConfig(rawConfig) {
const rawRequests = Array.isArray(rawConfig) ? rawConfig : rawConfig?.requests;
const requests = Array.isArray(rawRequests) ? rawRequests.map(normalizeConfigRequest).filter(Boolean) : [];
if (requests.length === 0) {
throw new Error("apidata/config.json 缺少有效接口配置,请至少提供一个 method");
}
return {
requests,
};
}
function printHelp() {
console.log(`
apidata — 按 config.json 请求远端并写入 apidata/data.json;默认同时写入 api-doc-agent/config/响应.md(仅含各接口 method 与 data)。
快速检查(--quick)不写入 响应.md,且 data.json 仅含「数据一览」、不含「详情」。
成功判定: 响应 message 与「成功判定名单」某项在 trim 且英文忽略大小写后一致即成功(默认含 ok、成功、登录成功)。
可在 apidata/config.json 根级设置 successMessages 数组覆盖默认名单。
可在同文件根级设置 requestTimeoutMs(数字,毫秒)覆盖单次 fetch 超时;未设置时 --quick 默认单接口 ${CRAWLER_CONFIG.quickModeTimeoutMs}ms,普通模式 ${CRAWLER_CONFIG.timeoutMs}ms。
可在同文件根级设置 maxConsecutiveTimeouts(数字,默认 5)控制连续超时熔断阈值。
失败接口: 默认在每条失败记录上附加「模块目录」(来自 data/module-list/module-list.json)、
「api封装」(在 src/api 中匹配到的 export)及「引用页面」(扫描 src/views、src/components、src/layout 下 .vue/.ts/.tsx 的引用路径,相对仓库根)。
空数据接口: message 判定成功且响应 data 中存在 list 字段且为 [] 或 null 的接口,写入「数据一览.空数据接口」,默认附加信息与失败接口相同。
快速检查(--quick)下:失败/空数据条目仅含 接口、message、文件(api 封装所在文件路径,去重)、引用页面;不含 模块目录、code、api 封装明细等。
用法:
npm run apiconfig:apidata
npm run apiconfig:apidata -- [选项] [模块名 ...]
选项:
-c, -r, --clear 清空除 ${LOGIN_METHOD} 外的请求项(无模块名时仅保留登录)
-n, --no-clear 指定模块时仍保留 config 中其它模块的请求(默认:指定模块时会先清空)
-q, --quick 快速检查:自动包含 data/module 下全部模块的接口(等同清空后写入全量),
data.json 仅含「数据一览」、不含「详情」;失败/空数据条目仅含 接口、message、文件、引用页面;不写入 响应.md
--timeout=<ms> 覆盖单次 fetch 超时;优先级高于 config.requestTimeoutMs
--delay=<ms> 请求间隔;优先级高于 config.requestDelayMs(--quick 默认 ${CRAWLER_CONFIG.quickDelayMs}ms)
--retries=<n> 单请求失败重试次数(不含首次);优先级高于 config.maxRetries(--quick 默认 ${CRAWLER_CONFIG.quickMaxRetries})
--timeout-streak=<n> 连续超时熔断阈值(默认 5);达到阈值后终止后续请求
-h, --help 显示本说明
模块名:
与 data/module/<folder> 一致,例如 order.index、order.revieworder(大小写不敏感)
指定一个或多个模块时,默认会先「清空除登录外的条目」,再写入这些模块在 module.json 中的全部 method。
示例:
npm run apiconfig:apidata
npm run apiconfig:apidata -- order.index order.revieworder
npm run apiconfig:apidata -- -c
npm run apiconfig:apidata -- --no-clear order.index
npm run apiconfig:apidata -- --quick
npm run apiconfig:apidata:quick
`);
}
/**
* @param {string[]} argv process.argv.slice(2)
*/
export function parseApidataCli(argv) {
const flags = { clear: false, noClear: false, help: false, quickCheck: false };
/** @type {{ delayMs: number | null, maxRetries: number | null, timeoutMs: number | null, timeoutStreak: number | null }} */
const tune = { delayMs: null, maxRetries: null, timeoutMs: null, timeoutStreak: null };
const positionals = [];
/** @param {string} raw */
const parseIntOpt = (raw) => {
const v = Number(raw);
if (!Number.isFinite(v) || v < 0) {
throw new Error(`无效数值: ${raw}`);
}
return Math.floor(v);
};
for (const arg of argv) {
if (arg === "--") {
continue;
}
if (arg === "-h" || arg === "--help") {
flags.help = true;
continue;
}
if (arg === "-c" || arg === "-r" || arg === "--clear") {
flags.clear = true;
continue;
}
if (arg === "-n" || arg === "--no-clear") {
flags.noClear = true;
continue;
}
if (arg === "-q" || arg === "--quick") {
flags.quickCheck = true;
continue;
}
if (arg.startsWith("--delay=")) {
tune.delayMs = parseIntOpt(arg.slice("--delay=".length));
continue;
}
if (arg.startsWith("--retries=")) {
tune.maxRetries = parseIntOpt(arg.slice("--retries=".length));
continue;
}
if (arg.startsWith("--timeout=")) {
tune.timeoutMs = parseIntOpt(arg.slice("--timeout=".length));
continue;
}
if (arg.startsWith("--timeout-streak=")) {
tune.timeoutStreak = parseIntOpt(arg.slice("--timeout-streak=".length));
continue;
}
if (arg.startsWith("-")) {
throw new Error(`未知参数: ${arg}(使用 --help 查看说明)`);
}
positionals.push(arg.trim());
}
const modules = positionals.filter(Boolean);
let clearOld = false;
if (flags.help) {
return { flags, modules, clearOld, help: true, quickCheck: false, tune };
}
if (flags.quickCheck && flags.noClear) {
throw new Error("快速检查模式(--quick)不能与 --no-clear 同时使用");
}
if (flags.quickCheck && modules.length > 0) {
throw new Error("快速检查模式会自动包含 data/module 下全部模块,请勿再指定模块名");
}
if (flags.noClear && flags.clear) {
throw new Error("不能同时使用 --clear 与 --no-clear");
}
if (modules.length > 0) {
clearOld = !flags.noClear;
} else {
clearOld = flags.clear;
}
return { flags, modules, clearOld, help: false, quickCheck: flags.quickCheck, tune };
}
async function listModuleFolders() {
const entries = await readdir(MODULE_DATA_ROOT, { withFileTypes: true });
return entries.filter((e) => e.isDirectory()).map((e) => e.name);
}
async function resolveModuleFolder(userInput) {
const name = String(userInput ?? "").trim();
if (!name) {
return null;
}
const folders = await listModuleFolders();
const exact = folders.find((f) => f === name);
if (exact) {
return exact;
}
const lower = name.toLowerCase();
const ci = folders.find((f) => f.toLowerCase() === lower);
if (ci) {
return ci;
}
return null;
}
async function loadMethodsFromModuleFolder(folder) {
const filePath = path.join(MODULE_DATA_ROOT, folder, "module.json");
const raw = await readJson(filePath);
const list = Array.isArray(raw?.interfaces) ? raw.interfaces : [];
const methods = [];
for (const item of list) {
const m = String(item?.method ?? "").trim();
if (m) {
methods.push(m);
}
}
return methods;
}
function requestEntry(method, body = {}) {
return {
method,
body: ensurePlainObject(body, {}),
};
}
/**
* @param {{ existingRaw: object, modules: string[], clearOld: boolean }} opts
*/
async function buildAndWriteConfig(opts) {
const { existingRaw, modules, clearOld } = opts;
const existing = normalizeConfig(existingRaw);
const loginReq = existing.requests.find((r) => r.method === LOGIN_METHOD);
const methodToRequest = new Map();
for (const r of existing.requests) {
methodToRequest.set(r.method, { method: r.method, body: { ...r.body }, needAuth: r.needAuth });
}
const unknownModules = [];
const moduleMethodsOrdered = [];
for (const mod of modules) {
const folder = await resolveModuleFolder(mod);
if (!folder) {
unknownModules.push(mod);
continue;
}
const methods = await loadMethodsFromModuleFolder(folder);
for (const m of methods) {
moduleMethodsOrdered.push(m);
}
}
const seenMethod = new Set();
const uniqueModuleMethods = [];
for (const m of moduleMethodsOrdered) {
if (seenMethod.has(m)) {
continue;
}
seenMethod.add(m);
uniqueModuleMethods.push(m);
}
if (unknownModules.length) {
const available = (await listModuleFolders()).sort().join(", ");
throw new Error(
`未找到模块: ${unknownModules.join(", ")}\n` + `data/module 下可用 folder: ${available}`,
);
}
const moduleMethodSet = new Set(uniqueModuleMethods);
let nextRequests = [];
if (clearOld) {
if (loginReq) {
nextRequests.push(requestEntry(LOGIN_METHOD, loginReq.body));
} else {
throw new Error(`清空模式下需要在 config.json 中保留 ${LOGIN_METHOD} 及 body,以便后续接口鉴权`);
}
for (const m of uniqueModuleMethods) {
if (m === LOGIN_METHOD) {
continue;
}
nextRequests.push(requestEntry(m, {}));
}
} else if (modules.length > 0) {
const merged = [];
for (const r of existing.requests) {
if (r.method === LOGIN_METHOD) {
merged.push({ method: r.method, body: { ...r.body } });
continue;
}
if (moduleMethodSet.has(r.method)) {
continue;
}
merged.push({ method: r.method, body: { ...r.body } });
}
for (const m of uniqueModuleMethods) {
if (m === LOGIN_METHOD) {
continue;
}
const prev = methodToRequest.get(m);
merged.push(requestEntry(m, prev?.body ?? {}));
}
nextRequests = merged;
const hasLogin = nextRequests.some((r) => r.method === LOGIN_METHOD);
if (!hasLogin) {
throw new Error(`config 中须包含 ${LOGIN_METHOD},或先使用清空模式生成配置`);
}
} else {
nextRequests = existing.requests.map((r) => ({ method: r.method, body: { ...r.body } }));
}
nextRequests.sort((a, b) => {
if (a.method === LOGIN_METHOD) {
return -1;
}
if (b.method === LOGIN_METHOD) {
return 1;
}
return 0;
});
const out = {
requests: nextRequests.map((r) => ({
method: r.method,
body: r.body ?? {},
})),
};
const prevSm = existingRaw?.successMessages;
if (Array.isArray(prevSm) && prevSm.length > 0) {
const cleaned = prevSm.map((x) => String(x).trim()).filter(Boolean);
if (cleaned.length > 0) {
out.successMessages = cleaned;
}
}
const prevRt = Number(existingRaw?.requestTimeoutMs);
if (Number.isFinite(prevRt) && prevRt > 0) {
out.requestTimeoutMs = clampRequestTimeoutMs(prevRt);
}
const prevTimeoutStreak = Number(existingRaw?.maxConsecutiveTimeouts);
if (Number.isFinite(prevTimeoutStreak) && prevTimeoutStreak > 0) {
out.maxConsecutiveTimeouts = Math.max(1, clampNonNegativeInt(prevTimeoutStreak));
}
await writeJson(APIDATA_CONFIG_FILE, out);
console.log(`已更新 ${APIDATA_CONFIG_FILE}(共 ${out.requests.length} 条请求)`);
return out;
}
/**
* 响应.md 仅含「接口 method + 响应 data」,不含数据一览、message 等。
* @param {Array<{ 接口: string, message?: string, code?: unknown, data: unknown }>} rows
*/
function buildMethodDataOnlyForMarkdown(rows) {
return rows.map((row) => ({
接口: row.接口,
data: row.data ?? null,
}));
}
function createResponseMarkdown(rows) {
const slim = buildMethodDataOnlyForMarkdown(rows);
return ["```json", JSON.stringify(slim, null, 2), "```", ""].join("\n");
}
function escapeRegex(s) {
return String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function toPosix(p) {
return p.split(path.sep).join("/");
}
/** @returns {Promise<string[]>} */
async function walkFilesRecursive(rootDir, { extensions } = {}) {
const out = [];
let st;
try {
st = await stat(rootDir);
} catch {
return out;
}
if (!st.isDirectory()) {
return out;
}
const entries = await readdir(rootDir, { withFileTypes: true });
for (const e of entries) {
const full = path.join(rootDir, e.name);
if (e.isDirectory()) {
out.push(...(await walkFilesRecursive(full, { extensions })));
} else if (e.isFile()) {
if (extensions && extensions.length > 0) {
const ext = path.extname(e.name).toLowerCase();
if (!extensions.includes(ext)) {
continue;
}
}
out.push(full);
}
}
return out;
}
/**
* 从 module-list.json 构建 method -> folder(module.json 所在目录名,对应 src/api/<folder>)
* @returns {Promise<Map<string, string>>}
*/
async function loadMethodToFolderFromModuleList() {
const map = new Map();
try {
const raw = await readJson(MODULE_LIST_FILE);
const modules = Array.isArray(raw?.modules) ? raw.modules : [];
for (const mod of modules) {
const folder = String(mod?.folder ?? "").trim();
const interfaces = Array.isArray(mod?.interfaces) ? mod.interfaces : [];
for (const it of interfaces) {
const m = String(it?.method ?? "").trim();
if (m && folder && !map.has(m)) {
map.set(m, folder);
}
}
}
} catch {
// 无 module-list 时仅全量扫描 src/api
}
return map;
}
/**
* @param {string} apiFileAbs
* @param {string} repoRoot
*/
function apiFileToImportPath(apiFileAbs, repoRoot) {
const rel = toPosix(path.relative(repoRoot, apiFileAbs));
const withoutExt = rel.replace(/\.ts$/i, "");
if (withoutExt.startsWith("src/")) {
return "@" + withoutExt.slice("src".length);
}
return "@" + withoutExt;
}
/**
* 解析单文件:method 字符串对应的封装(export function / export const fn = ( / 聚合对象上的属性)
* @returns {{ kind: 'function', exportName: string } | { kind: 'objectProperty', objectName: string, propName: string }}[]
*/
function extractWrappersFromApiSource(content, methodStr) {
const needleSingle = `method: '${methodStr}'`;
const needleDouble = `method: "${methodStr}"`;
if (!content.includes(needleSingle) && !content.includes(needleDouble)) {
return [];
}
const lines = content.split(/\r?\n/);
let currentExport = null;
/** @type {Map<string, Set<string>>} method -> function export names */
const methodToFnExports = new Map();
const exportFnRe = /export\s+function\s+(\w+)\s*\(/;
const exportConstFnRe = /export\s+const\s+(\w+)\s*=\s*(?:async\s*)?\(/;
const methodLineRe = /method:\s*['"]([^'"]+)['"]/;
function addFnExport(meth, name) {
if (!meth || !name) {
return;
}
if (!methodToFnExports.has(meth)) {
methodToFnExports.set(meth, new Set());
}
methodToFnExports.get(meth).add(name);
}
for (const line of lines) {
let m = line.match(exportFnRe);
if (m) {
currentExport = m[1];
} else {
m = line.match(exportConstFnRe);
if (m) {
currentExport = m[1];
}
}
const mm = line.match(methodLineRe);
if (mm && currentExport) {
addFnExport(mm[1], currentExport);
}
}
const fnNames = methodToFnExports.get(methodStr);
if (!fnNames || fnNames.size === 0) {
return [];
}
/** @type {any[]} */
const wrappers = [];
for (const fnName of fnNames) {
const aliases = findObjectPropertyAliases(content, fnName);
// 若已发现聚合对象对外导出,优先展示该调用形态,避免同一接口重复显示“函数 + 对象属性”两条封装。
if (aliases.length > 0) {
for (const { objectName, propName } of aliases) {
wrappers.push({ kind: "objectProperty", objectName, propName });
}
continue;
}
wrappers.push({ kind: "function", exportName: fnName });
}
return wrappers;
}
/**
* @param {string} content
* @param {string} fnName
* @returns {{ objectName: string, propName: string }[]}
*/
function findObjectPropertyAliases(content, fnName) {
const re = new RegExp(`\\b(\\w+)\\s*:\\s*${escapeRegex(fnName)}\\b`, "g");
const results = [];
let m;
while ((m = re.exec(content)) !== null) {
const propName = m[1];
const before = content.slice(0, m.index);
const objs = [...before.matchAll(/export\s+const\s+(\w+)\s*=\s*\{/g)];
if (objs.length === 0) {
continue;
}
const objectName = objs[objs.length - 1][1];
results.push({ objectName, propName });
}
return results;
}
/**
* @param {string} importPath
*/
function importsFromPath(content, importPath) {
return new RegExp(`from\\s*['"]${escapeRegex(importPath)}['"]`, "i").test(content);
}
/**
* import { a, b } from 'path'(支持多行)
*/
function namedImportDeclaresSymbol(content, importPath, symbol) {
if (!importsFromPath(content, importPath)) {
return false;
}
const ep = escapeRegex(importPath);
const blocks = [...content.matchAll(new RegExp(`import\\s*\\{([\\s\\S]*?)\\}\\s*from\\s*['"]${ep}['"]`, "gi"))];
for (const b of blocks) {
if (new RegExp(`\\b${escapeRegex(symbol)}\\b`).test(b[1])) {
return true;
}
}
return false;
}
function namespaceAlias(content, importPath) {
const ep = escapeRegex(importPath);
const m = content.match(new RegExp(`import\\s*\\*\\s*as\\s+(\\w+)\\s+from\\s*['"]${ep}['"]`, "i"));
return m ? m[1] : null;
}
function viewUsesFunctionExport(content, importPath, exportName) {
const alias = namespaceAlias(content, importPath);
if (alias && new RegExp(`\\b${escapeRegex(alias)}\\.${escapeRegex(exportName)}\\s*\\(`).test(content)) {
return true;
}
if (namedImportDeclaresSymbol(content, importPath, exportName)) {
if (new RegExp(`(?:^|[^.\\w])${escapeRegex(exportName)}\\s*\\(`).test(content)) {
return true;
}
}
return false;
}
function viewUsesObjectProperty(content, importPath, objectName, propName) {
if (!namedImportDeclaresSymbol(content, importPath, objectName)) {
return false;
}
return new RegExp(
`\\b${escapeRegex(objectName)}\\.${escapeRegex(propName)}\\s*\\(`,
).test(content);
}
/**
* 扫描整个 src/api(module-list 的 folder 仅写入失败项的「模块目录」,不排除其它目录下的同名 method 封装)
* @param {string} repoRoot
* @param {string} methodStr
*/
async function collectWrappersForMethod(repoRoot, methodStr) {
const srcApi = path.join(repoRoot, "src", "api");
const tsFiles = await walkFilesRecursive(srcApi, { extensions: [".ts"] });
const seen = new Set();
/** @type {{ relFile: string, importPath: string, kind: string, exportName?: string, objectName?: string, propName?: string }[]} */
const out = [];
for (const abs of tsFiles) {
let text;
try {
text = await readFile(abs, "utf8");
} catch {
continue;
}
const wrappers = extractWrappersFromApiSource(text, methodStr);
if (wrappers.length === 0) {
continue;
}
const relFile = toPosix(path.relative(repoRoot, abs));
const importPath = apiFileToImportPath(abs, repoRoot);
for (const w of wrappers) {
const key =
w.kind === "function"
? `${relFile}|fn|${w.exportName}`
: `${relFile}|obj|${w.objectName}.${w.propName}`;
if (seen.has(key)) {
continue;
}
seen.add(key);
if (w.kind === "function") {
out.push({ relFile, importPath, kind: "function", exportName: w.exportName });
} else {
out.push({
relFile,
importPath,
kind: "objectProperty",
objectName: w.objectName,
propName: w.propName,
});
}
}
}
return out;
}
/** 失败接口「引用页面」扫描目录(相对 src/) */
const APIDATA_USAGE_SCAN_SEGMENTS = ["views", "components", "layout"];
/**
* @param {string} repoRoot
* @param {Awaited<ReturnType<typeof collectWrappersForMethod>>} wrappers
*/
async function findViewsUsingWrappers(repoRoot, wrappers) {
if (wrappers.length === 0) {
return [];
}
const candidates = [];
for (const seg of APIDATA_USAGE_SCAN_SEGMENTS) {
const root = path.join(repoRoot, "src", seg);
const files = await walkFilesRecursive(root, { extensions: [".vue", ".ts", ".tsx"] });
candidates.push(...files);
}
const hits = new Set();
for (const abs of candidates) {
let content;
try {
content = await readFile(abs, "utf8");
} catch {
continue;
}
for (const w of wrappers) {
let ok = false;
if (w.kind === "function") {
ok = viewUsesFunctionExport(content, w.importPath, w.exportName);
} else if (w.kind === "objectProperty") {
ok = viewUsesObjectProperty(content, w.importPath, w.objectName, w.propName);
}
if (ok) {
hits.add(toPosix(path.relative(repoRoot, abs)));
break;
}
}
}
return [...hits].sort();
}
/**
* 为「失败接口」补充 api 封装与引用页面路径(views / components / layout)
* @param {Array<{ 接口: string, message: string, code: unknown }>} failedList
*/
async function enrichFailedInterfacesWithUsage(failedList) {
const repoRoot = process.cwd();
const methodToFolder = await loadMethodToFolderFromModuleList();
const enriched = [];
for (const item of failedList) {
const methodStr = String(item.接口 ?? "").trim();
const folder = methodToFolder.get(methodStr) ?? null;
const wrappers = await collectWrappersForMethod(repoRoot, methodStr);
const 引用页面 = await findViewsUsingWrappers(repoRoot, wrappers);
const api封装 = wrappers.map((w) => {
if (w.kind === "function") {
return { 文件: w.relFile, 方法名: w.exportName };
}
return {
文件: w.relFile,
方法名: `${w.objectName}.${w.propName}`,
说明: "聚合对象方法",
};
});
enriched.push({
...item,
模块目录: folder,
api封装,
引用页面,
});
}
return enriched;
}
/**
* 快速检查:失败/空数据条目仅保留 接口、message、文件(api 封装去重后的相对路径)、引用页面
* @param {Awaited<ReturnType<typeof enrichFailedInterfacesWithUsage>>[number]} enriched
*/
function slimUsageRowForQuickCheck(enriched) {
const 文件 = [];
const seen = new Set();
for (const w of enriched.api封装 ?? []) {
const f = w?.文件;
if (typeof f === "string" && f && !seen.has(f)) {
seen.add(f);
文件.push(f);
}
}
文件.sort();
return {
接口: enriched.接口,
message: enriched.message,
文件,
引用页面: Array.isArray(enriched.引用页面) ? [...enriched.引用页面] : [],
};
}
/**
* @param {{ quickCheck?: boolean, tune?: { delayMs?: number | null, maxRetries?: number | null, timeoutMs?: number | null } }} [opts]
*/
async function runCrawlFromConfigFile(opts = {}) {
const { quickCheck = false, tune = {} } = opts;
const rawConfig = await readJson(APIDATA_CONFIG_FILE);
const successMessages = resolveSuccessMessages(rawConfig);
const config = normalizeConfig(rawConfig);
const crawlTimeoutMs = resolveCrawlTimeoutMs({
rawConfig,
quickCheck,
cliTimeoutMs: tune.timeoutMs ?? null,
});
const requestDelayMs = resolveRequestDelayMs({
rawConfig,
quickCheck,
cliDelayMs: tune.delayMs ?? null,
});
const maxRetries = resolveMaxRetries({
rawConfig,
quickCheck,
cliMaxRetries: tune.maxRetries ?? null,
});
const maxConsecutiveTimeouts = resolveMaxConsecutiveTimeouts({
rawConfig,
cliTimeoutStreak: tune.timeoutStreak ?? null,
});
console.log(
`apidata 抓取参数: 超时=${crawlTimeoutMs}ms 间隔=${requestDelayMs}ms 重试=${maxRetries} 连续超时熔断=${maxConsecutiveTimeouts}${quickCheck ? "(快速检查)" : ""}`,
);
const rows = [];
let authToken = null;
const total = config.requests.length;
let idx = 0;
const networkErrors = [];
let consecutiveTimeoutCount = 0;
for (const req of config.requests) {
idx += 1;
const isLogin = req.method === LOGIN_METHOD;
const result = await crawlApiWithRetry({
method: req.method,
body: req.body,
timeoutMs: crawlTimeoutMs,
token: req.needAuth ? authToken : null,
maxRetries: isLogin ? Math.max(maxRetries, 1) : maxRetries,
});
const res = result.response;
const isObj = res && typeof res === "object" && !Array.isArray(res);
if (isObj && /** @type {{ data?: { token?: unknown } }} */ (res).data?.token) {
authToken = /** @type {{ data: { token: string } }} */ (res).data.token;
}
if (result.error) {
rows.push({
接口: req.method,
message: `NETWORK: ${result.error}`,
code: null,
data: null,
});
networkErrors.push({ method: req.method, error: result.error, attempts: result.attempts });
console.log(
` [${idx}/${total}] ${req.method} — 网络失败(重试${result.attempts - 1}次): ${result.error}`,
);
if (isLogin) {
throw new Error(`登录接口 ${LOGIN_METHOD} 多次失败,无法继续鉴权抓取:${result.error}`);
}
} else {
rows.push({
接口: req.method,
message: isObj ? String(/** @type {{ message?: unknown }} */ (res).message ?? "") : "",
code: isObj && "code" in res ? /** @type {{ code?: unknown }} */ (res).code : null,
data: isObj ? /** @type {{ data?: unknown }} */ (res).data ?? null : null,
});
if (quickCheck || result.attempts > 1) {
console.log(
` [${idx}/${total}] ${req.method} — ${result.elapsedMs}ms${result.attempts > 1 ? `(重试${result.attempts - 1}次)` : ""}`,
);
}
}
if (isTimeoutFailureResult(result)) {
consecutiveTimeoutCount += 1;
if (consecutiveTimeoutCount >= maxConsecutiveTimeouts) {
throw new Error(
`检测到连续 ${consecutiveTimeoutCount} 个接口请求超时(阈值=${maxConsecutiveTimeouts}),疑似后端异常,已终止后续请求`,
);
}
} else {
consecutiveTimeoutCount = 0;
}
if (requestDelayMs > 0 && idx < total) {
await sleepMs(requestDelayMs);
}
}
const 数据一览 = buildDataOverview(rows, successMessages);
const failedEnriched = await enrichFailedInterfacesWithUsage(数据一览.失败接口);
const emptyEnriched = await enrichFailedInterfacesWithUsage(数据一览.空数据接口);
if (quickCheck) {
数据一览.失败接口 = failedEnriched.map(slimUsageRowForQuickCheck);
数据一览.空数据接口 = emptyEnriched.map(slimUsageRowForQuickCheck);
} else {
数据一览.失败接口 = failedEnriched;
数据一览.空数据接口 = emptyEnriched;
}
const payload = quickCheck ? { 数据一览 } : { 数据一览, 详情: rows };
await writeJson(APIDATA_DATA_FILE, payload);
console.log(`已写入接口响应数据: ${APIDATA_DATA_FILE}`);
if (quickCheck) {
console.log("快速检查模式:已跳过写入 响应.md");
} else {
await mkdir(path.dirname(RESPONSE_MARKDOWN_FILE), { recursive: true });
await writeFile(RESPONSE_MARKDOWN_FILE, createResponseMarkdown(rows), "utf8");
console.log(`已更新响应文档: ${RESPONSE_MARKDOWN_FILE}`);
}
console.log(
`数据一览: 当前请求数=${数据一览.当前请求数} 成功数/全部请求数=${数据一览.成功数与全部请求数} 失败数=${数据一览.失败接口.length} 空数据数=${数据一览.空数据接口.length} 网络错误=${networkErrors.length}`,
);
if (数据一览.空数据接口.length) {
for (const item of 数据一览.空数据接口) {
const codePart =
!quickCheck && "code" in item ? ` — code=${JSON.stringify(/** @type {{ code?: unknown }} */ (item).code)}` : "";
console.log(` 空数据: ${item.接口}${codePart} — ${item.message}`);
}
}
if (数据一览.失败接口.length) {
for (const item of 数据一览.失败接口) {
const codePart =
!quickCheck && "code" in item ? ` — code=${JSON.stringify(/** @type {{ code?: unknown }} */ (item).code)}` : "";
console.log(` 失败: ${item.接口}${codePart} — ${item.message}`);
}
}
}
export async function runApidata() {
const cli = parseApidataCli(process.argv.slice(2));
if (cli.help) {
printHelp();
return;
}
const existingRaw = await readJson(APIDATA_CONFIG_FILE);
if (cli.quickCheck) {
const all = (await listModuleFolders()).sort();
if (all.length === 0) {
throw new Error("快速检查:data/module 下无模块目录");
}
console.log(`apidata 快速检查:将清空并请求全部模块接口(${all.length} 个模块目录)`);
await buildAndWriteConfig({
existingRaw,
modules: all,
clearOld: true,
});
} else if (cli.modules.length > 0 || cli.clearOld) {
console.log(
`apidata: clearOld=${cli.clearOld}, modules=[${cli.modules.join(", ")}]`,
);
await buildAndWriteConfig({
existingRaw,
modules: cli.modules,
clearOld: cli.clearOld,
});
}
await runCrawlFromConfigFile({ quickCheck: cli.quickCheck, tune: cli.tune });
}