2026-04-15 21:01:12 +08:00

1138 lines
37 KiB
JavaScript

import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { API_DOC_SYSTEM_CONFIG } from "./config.mjs";
const MODULE_LIFECYCLE_FILE = "_module.lifecycle.json";
const MODULE_DELETED_MARK_PREFIX = "_deleted.";
const ENTITY_MAP = {
"&": "&",
"&lt;": "<",
"&gt;": ">",
"&quot;": '"',
"&#39;": "'",
"&nbsp;": " ",
};
function decodeHtmlEntities(value) {
const withNamed = Object.entries(ENTITY_MAP).reduce(
(acc, [entity, replacement]) => acc.split(entity).join(replacement),
value,
);
return withNamed.replace(/&#(\d+);/g, (_, num) => String.fromCharCode(Number(num)));
}
function normalizeText(value) {
return decodeHtmlEntities(value.replace(/<[^>]*>/g, " ")).replace(/\s+/g, " ").trim();
}
function parseAttributes(tag) {
const attrs = {};
const attrRegex = /([:@a-zA-Z0-9_-]+)\s*=\s*"([^"]*)"/g;
for (const match of tag.matchAll(attrRegex)) {
attrs[match[1]] = decodeHtmlEntities(match[2]);
}
return attrs;
}
function extractSelectOptions(html, headerName) {
const regex = new RegExp(
`<select[^>]*data-header-name="${headerName}"[^>]*>([\\s\\S]*?)<\\/select>`,
"i",
);
const selectMatch = html.match(regex);
if (!selectMatch) {
return [];
}
const options = [];
const optionRegex = /<option\s+value="([^"]*)">([\s\S]*?)<\/option>/g;
for (const optionMatch of selectMatch[1].matchAll(optionRegex)) {
options.push({
value: decodeHtmlEntities(optionMatch[1]).trim(),
label: normalizeText(optionMatch[2]),
});
}
return options;
}
function findRequiredFilterValue(html, headerName, expectedValue) {
const options = extractSelectOptions(html, headerName);
return options.find((option) => option.value === expectedValue) ?? null;
}
function extractSectionTableRows(cardHtml, sectionName) {
const escapedName = sectionName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const sectionRegex = new RegExp(
`<details>\\s*<summary>\\s*${escapedName}\\s*\\((\\d+)\\)<\\/summary>[\\s\\S]*?<tbody>([\\s\\S]*?)<\\/tbody>[\\s\\S]*?<\\/details>`,
"i",
);
const match = cardHtml.match(sectionRegex);
if (!match) {
return [];
}
const tbodyHtml = match[2];
if (/暂无数据/.test(tbodyHtml)) {
return [];
}
const rows = [];
const rowRegex = /<tr>([\s\S]*?)<\/tr>/g;
for (const rowMatch of tbodyHtml.matchAll(rowRegex)) {
const cellValues = [...rowMatch[1].matchAll(/<td[^>]*>([\s\S]*?)<\/td>/g)].map((cellMatch) =>
normalizeText(cellMatch[1]),
);
if (cellValues.length < 4) {
continue;
}
rows.push({
name: cellValues[0],
type: cellValues[1],
required: cellValues[2] === "是",
description: cellValues[3],
});
}
return rows;
}
function extractBadgeValue(cardHtml, badgeName) {
const badgeRegex = new RegExp(`<span class="badge">${badgeName}:\\s*([^<]+)<\\/span>`, "i");
const match = cardHtml.match(badgeRegex);
return match ? normalizeText(match[1]) : "";
}
function parseMethodCard(cardHtml) {
const openTagMatch = cardHtml.match(/^<section class="method-card"[^>]*>/i);
if (!openTagMatch) {
return null;
}
const attrs = parseAttributes(openTagMatch[0]);
const methodName = normalizeText(cardHtml.match(/<h3>([\s\S]*?)<\/h3>/i)?.[1] ?? "");
if (!methodName) {
return null;
}
const description = normalizeText(cardHtml.match(/<p class="method-desc">([\s\S]*?)<\/p>/i)?.[1] ?? "");
const endpoint = attrs["data-endpoint"] ?? extractBadgeValue(cardHtml, "入口");
const authRequired = extractBadgeValue(cardHtml, "鉴权") === "是";
const loginRequired =
attrs["data-login-required"] === "true" || extractBadgeValue(cardHtml, "登录") === "是";
return {
method: methodName,
title: description || `${methodName} 接口`,
description,
endpoint,
authRequired,
loginRequired,
headers: extractSectionTableRows(cardHtml, "Headers"),
params: extractSectionTableRows(cardHtml, "Params"),
response: extractSectionTableRows(cardHtml, "Response"),
};
}
function parseMethodCards(blockHtml) {
const starts = [...blockHtml.matchAll(/<section class="method-card"[^>]*>/g)];
const cards = [];
for (let i = 0; i < starts.length; i += 1) {
const start = starts[i].index;
const end =
i + 1 < starts.length ? starts[i + 1].index : blockHtml.lastIndexOf("</section>") + "</section>".length;
const cardHtml = blockHtml.slice(start, end);
const parsed = parseMethodCard(cardHtml);
if (parsed) {
cards.push(parsed);
}
}
return cards;
}
function sanitizeFolderName(name) {
const normalized = name.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
return normalized.replace(/^-+|-+$/g, "") || "module";
}
function parseGroupBlock(blockHtml) {
const openTagMatch = blockHtml.match(/^<section class="group-block"[^>]*>/i);
if (!openTagMatch) {
return null;
}
const attrs = parseAttributes(openTagMatch[0]);
const headingMatch = blockHtml.match(/<h2>([\s\S]*?)<small>([\s\S]*?)<\/small><\/h2>/i);
const moduleName = headingMatch ? normalizeText(headingMatch[1]) : "";
const moduleKey = headingMatch ? normalizeText(headingMatch[2]) : attrs["data-group-key"] ?? "";
const methods = parseMethodCards(blockHtml);
if (!moduleName || methods.length === 0) {
return null;
}
return {
moduleName,
moduleKey,
baseFolderName: sanitizeFolderName(moduleKey || moduleName),
methods,
};
}
function parseGroupsFromHtml(html) {
const starts = [...html.matchAll(/<section class="group-block"[^>]*>/g)];
const parsedGroups = [];
const usedFolders = new Map();
for (let i = 0; i < starts.length; i += 1) {
const start = starts[i].index;
const end = i + 1 < starts.length ? starts[i + 1].index : html.lastIndexOf("</section>") + "</section>".length;
const blockHtml = html.slice(start, end);
const parsed = parseGroupBlock(blockHtml);
if (!parsed) {
continue;
}
const currentCount = usedFolders.get(parsed.baseFolderName) ?? 0;
usedFolders.set(parsed.baseFolderName, currentCount + 1);
const folderName =
currentCount === 0 ? parsed.baseFolderName : `${parsed.baseFolderName}-${String(currentCount + 1)}`;
parsedGroups.push({
moduleName: parsed.moduleName,
moduleKey: parsed.moduleKey,
folderName,
methods: parsed.methods,
});
}
return parsedGroups;
}
async function writeJson(filePath, data) {
await mkdir(path.dirname(filePath), { recursive: true });
await writeFile(filePath, `${JSON.stringify(data, null, 2)}\n`, "utf8");
}
async function readJsonIfExists(filePath, fallbackValue = null) {
try {
const content = await readFile(filePath, "utf8");
return JSON.parse(content);
} catch (error) {
if (error?.code === "ENOENT") {
return fallbackValue;
}
throw error;
}
}
function isSameJson(a, b) {
return JSON.stringify(a ?? null) === JSON.stringify(b ?? null);
}
function getDateStamp(date = new Date()) {
const year = String(date.getFullYear());
const month = String(date.getMonth() + 1).padStart(2, "0");
const day = String(date.getDate()).padStart(2, "0");
return `${year}${month}${day}`;
}
function getDateTimeStamp(date = new Date()) {
const dateStamp = getDateStamp(date);
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, "0");
return `${dateStamp}${hours}${minutes}${seconds}`;
}
function formatLocalDateTime(input) {
const date = input instanceof Date ? input : new Date(input);
const year = String(date.getFullYear());
const month = String(date.getMonth() + 1).padStart(2, "0");
const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, "0");
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
}
function formatStampToLocalDateTime(stamp) {
if (!/^\d{14}$/.test(stamp)) {
return null;
}
const year = Number(stamp.slice(0, 4));
const month = Number(stamp.slice(4, 6));
const day = Number(stamp.slice(6, 8));
const hour = Number(stamp.slice(8, 10));
const minute = Number(stamp.slice(10, 12));
const second = Number(stamp.slice(12, 14));
return formatLocalDateTime(new Date(year, month - 1, day, hour, minute, second));
}
async function pruneFilesByPattern(targetDir, keepCount, pattern) {
await mkdir(targetDir, { recursive: true });
const files = await readdir(targetDir);
const matched = files.filter((fileName) => pattern.test(fileName));
if (matched.length <= keepCount) {
return;
}
const fileStats = await Promise.all(
matched.map(async (fileName) => {
const filePath = path.join(targetDir, fileName);
const fileInfo = await stat(filePath);
return {
filePath,
mtimeMs: fileInfo.mtimeMs,
};
}),
);
fileStats.sort((a, b) => b.mtimeMs - a.mtimeMs);
const expired = fileStats.slice(keepCount);
await Promise.all(expired.map((item) => rm(item.filePath)));
}
async function pruneDailyFiles(targetDir, keepDays) {
await pruneFilesByPattern(targetDir, keepDays, /^\d{8}\.json$/);
}
async function listModuleFolders(moduleRootDir) {
await mkdir(moduleRootDir, { recursive: true });
const entries = await readdir(moduleRootDir, { withFileTypes: true });
return entries
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort((a, b) => a.localeCompare(b, "zh-CN"));
}
async function upsertModuleLifecycle({ folderPath, moduleMeta, nowIso }) {
const lifecycleFilePath = path.join(folderPath, MODULE_LIFECYCLE_FILE);
const current = await readJsonIfExists(lifecycleFilePath, null);
if (!current) {
await writeJson(lifecycleFilePath, {
module: moduleMeta,
status: "active",
createdAt: nowIso,
updatedAt: nowIso,
deletedAt: null,
restoredAt: null,
history: [{ event: "created", at: nowIso, note: "Module folder initialized" }],
});
return;
}
const next = {
...current,
module: moduleMeta,
updatedAt: nowIso,
};
if (current.status === "deleted") {
next.status = "active";
next.deletedAt = null;
next.restoredAt = nowIso;
next.history = Array.isArray(current.history)
? [...current.history, { event: "restored", at: nowIso, note: "Module returned in latest doc" }]
: [{ event: "restored", at: nowIso, note: "Module returned in latest doc" }];
}
await writeJson(lifecycleFilePath, next);
}
async function markModuleFolderDeleted({ folderPath, folderName, nowIso, todayStamp }) {
const lifecycleFilePath = path.join(folderPath, MODULE_LIFECYCLE_FILE);
const lifecycle = await readJsonIfExists(lifecycleFilePath, null);
if (lifecycle?.status === "deleted") {
return false;
}
const markerFilePath = path.join(folderPath, `${MODULE_DELETED_MARK_PREFIX}${todayStamp}.json`);
await writeJson(markerFilePath, {
folder: folderName,
deletedAt: nowIso,
reason: "Missing from latest API doc parsing result",
rollbackHint: "Folder and recent snapshots are kept for rollback.",
});
const moduleMeta = lifecycle?.module ?? {
name: folderName,
key: folderName,
folder: folderName,
};
const currentHistory = Array.isArray(lifecycle?.history) ? lifecycle.history : [];
const createdAt = lifecycle?.createdAt ?? nowIso;
await writeJson(lifecycleFilePath, {
module: moduleMeta,
status: "deleted",
createdAt,
updatedAt: nowIso,
deletedAt: nowIso,
restoredAt: lifecycle?.restoredAt ?? null,
history: [...currentHistory, { event: "deleted", at: nowIso, note: "Module missing in latest doc" }],
});
return true;
}
async function getLatestDailyFileInDir(targetDir) {
try {
const files = await readdir(targetDir);
const dailyFiles = files.filter((fileName) => /^\d{8}\.json$/.test(fileName)).sort().reverse();
const latest = dailyFiles[0];
return latest ? path.join(targetDir, latest) : null;
} catch (error) {
if (error?.code === "ENOENT") {
return null;
}
throw error;
}
}
async function listRecentSnapshots(targetDir, limit = API_DOC_SYSTEM_CONFIG.retentionDays) {
await mkdir(targetDir, { recursive: true });
const files = await readdir(targetDir);
const dailyFiles = files.filter((fileName) => /^\d{8}\.json$/.test(fileName)).sort().reverse().slice(0, limit);
const snapshots = [];
for (const fileName of dailyFiles) {
const filePath = path.join(targetDir, fileName);
const parsed = await readJsonIfExists(filePath, null);
if (!parsed) {
continue;
}
snapshots.push({
stamp: fileName.replace(".json", ""),
filePath,
generatedAt: parsed.generatedAt ?? null,
moduleCount: parsed.moduleCount ?? null,
totalInterfaceCount: parsed.totalInterfaceCount ?? null,
});
}
return snapshots;
}
async function readLatestModuleSnapshots(moduleDir) {
const snapshots = new Map();
const folders = await listModuleFolders(moduleDir);
for (const folderName of folders) {
const folderPath = path.join(moduleDir, folderName);
const lifecycle = await readJsonIfExists(path.join(folderPath, MODULE_LIFECYCLE_FILE), null);
const latestFilePath = await getLatestDailyFileInDir(folderPath);
if (!latestFilePath) {
continue;
}
const parsed = await readJsonIfExists(latestFilePath, null);
if (!parsed) {
continue;
}
snapshots.set(folderName, {
filePath: latestFilePath,
data: parsed,
lifecycleStatus: lifecycle?.status ?? "active",
});
}
return snapshots;
}
async function readPreviousStatusesFromModules(moduleDir) {
const statusMap = new Map();
const folders = await listModuleFolders(moduleDir);
for (const folderName of folders) {
const folderPath = path.join(moduleDir, folderName);
const latestFilePath = await getLatestDailyFileInDir(folderPath);
if (!latestFilePath) {
continue;
}
const parsed = await readJsonIfExists(latestFilePath, null);
const interfaces = Array.isArray(parsed?.interfaces) ? parsed.interfaces : [];
for (const item of interfaces) {
if (!item?.method) {
continue;
}
statusMap.set(item.method, {
isUsed: Boolean(item?.status?.isUsed),
isPlaceholder: Boolean(item?.status?.isPlaceholder),
});
}
}
return statusMap;
}
async function fetchDocHtml() {
try {
const response = await fetch(API_DOC_SYSTEM_CONFIG.docUrl, {
method: "GET",
signal: AbortSignal.timeout(API_DOC_SYSTEM_CONFIG.requestTimeoutMs),
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
return {
source: "remote",
html: await response.text(),
};
} catch (error) {
const fallbackContent = await readFile(API_DOC_SYSTEM_CONFIG.file.localHtmlSnapshot, "utf8");
console.warn(`抓取远端文档失败,已使用本地快照继续执行:${error.message}`);
return {
source: "local-fallback",
html: fallbackContent,
};
}
}
function assertFilterOptionsAvailable(html) {
const platform = findRequiredFilterValue(html, "platform", API_DOC_SYSTEM_CONFIG.filter.platform.value);
const project = findRequiredFilterValue(html, "project", API_DOC_SYSTEM_CONFIG.filter.project.value);
if (!platform) {
throw new Error(`文档中未找到 platform=${API_DOC_SYSTEM_CONFIG.filter.platform.value} 的选项。`);
}
if (!project) {
throw new Error(`文档中未找到 project=${API_DOC_SYSTEM_CONFIG.filter.project.value} 的选项。`);
}
return {
platform,
project,
};
}
function toModuleComparable(moduleLike) {
const interfaces = Array.isArray(moduleLike.interfaces) ? moduleLike.interfaces : [];
const methodMap = new Map();
for (const item of interfaces) {
if (!item?.method) {
continue;
}
methodMap.set(item.method, {
method: item.method,
params: item.params ?? [],
response: item.response ?? [],
});
}
return {
moduleName: moduleLike.moduleName ?? moduleLike?.module?.name ?? "",
moduleKey: moduleLike.moduleKey ?? moduleLike?.module?.key ?? "",
folder: moduleLike.folder ?? moduleLike?.module?.folder ?? "",
methodMap,
methodSet: new Set(methodMap.keys()),
};
}
function getSetDiff(sourceSet, targetSet) {
return [...sourceSet].filter((item) => !targetSet.has(item));
}
function getSetIntersection(sourceSet, targetSet) {
return [...sourceSet].filter((item) => targetSet.has(item));
}
function buildMethodOwnershipMap(moduleMap) {
const ownership = new Map();
for (const [folder, moduleInfo] of moduleMap.entries()) {
for (const method of moduleInfo.methodSet) {
ownership.set(method, folder);
}
}
return ownership;
}
function getModuleBasics(moduleInfo) {
return {
moduleName: moduleInfo.moduleName,
moduleKey: moduleInfo.moduleKey,
folder: moduleInfo.folder,
};
}
function detectModuleAndInterfaceChanges(previousModuleMap, currentModuleMap) {
const previousFolders = new Set(previousModuleMap.keys());
const currentFolders = new Set(currentModuleMap.keys());
const addedFolders = getSetDiff(currentFolders, previousFolders);
const deletedFolders = getSetDiff(previousFolders, currentFolders);
const commonFolders = getSetIntersection(currentFolders, previousFolders);
const moduleChanges = {
added: [],
deleted: [],
updated: [],
split: [],
merged: [],
replaced: [],
};
const interfaceChanges = {
added: [],
deleted: [],
paramsUpdated: [],
responseUpdated: [],
};
const perModuleDiff = new Map();
for (const folder of addedFolders) {
const moduleInfo = currentModuleMap.get(folder);
const methods = [...moduleInfo.methodSet].sort((a, b) => a.localeCompare(b, "zh-CN"));
moduleChanges.added.push({
...getModuleBasics(moduleInfo),
methods,
interfaceCount: methods.length,
});
perModuleDiff.set(folder, {
type: "added",
interfaceChanges: {
addedMethods: methods,
deletedMethods: [],
paramsUpdatedMethods: [],
responseUpdatedMethods: [],
},
});
for (const method of methods) {
interfaceChanges.added.push({
...getModuleBasics(moduleInfo),
method,
});
}
}
for (const folder of deletedFolders) {
const moduleInfo = previousModuleMap.get(folder);
const methods = [...moduleInfo.methodSet].sort((a, b) => a.localeCompare(b, "zh-CN"));
moduleChanges.deleted.push({
...getModuleBasics(moduleInfo),
methods,
interfaceCount: methods.length,
});
for (const method of methods) {
interfaceChanges.deleted.push({
...getModuleBasics(moduleInfo),
method,
});
}
}
for (const folder of commonFolders) {
const previousModule = previousModuleMap.get(folder);
const currentModule = currentModuleMap.get(folder);
const addedMethods = getSetDiff(currentModule.methodSet, previousModule.methodSet).sort((a, b) =>
a.localeCompare(b, "zh-CN"),
);
const deletedMethods = getSetDiff(previousModule.methodSet, currentModule.methodSet).sort((a, b) =>
a.localeCompare(b, "zh-CN"),
);
const sameMethods = getSetIntersection(currentModule.methodSet, previousModule.methodSet);
const paramsUpdatedMethods = [];
const responseUpdatedMethods = [];
for (const method of sameMethods) {
const oldMethod = previousModule.methodMap.get(method);
const newMethod = currentModule.methodMap.get(method);
if (!isSameJson(oldMethod?.params, newMethod?.params)) {
paramsUpdatedMethods.push(method);
}
if (!isSameJson(oldMethod?.response, newMethod?.response)) {
responseUpdatedMethods.push(method);
}
}
paramsUpdatedMethods.sort((a, b) => a.localeCompare(b, "zh-CN"));
responseUpdatedMethods.sort((a, b) => a.localeCompare(b, "zh-CN"));
if (
addedMethods.length > 0 ||
deletedMethods.length > 0 ||
paramsUpdatedMethods.length > 0 ||
responseUpdatedMethods.length > 0 ||
previousModule.moduleName !== currentModule.moduleName ||
previousModule.moduleKey !== currentModule.moduleKey
) {
moduleChanges.updated.push({
...getModuleBasics(currentModule),
previousModuleName: previousModule.moduleName,
previousModuleKey: previousModule.moduleKey,
interfaceChanges: {
addedMethods,
deletedMethods,
paramsUpdatedMethods,
responseUpdatedMethods,
},
});
}
perModuleDiff.set(folder, {
type: "updated",
interfaceChanges: {
addedMethods,
deletedMethods,
paramsUpdatedMethods,
responseUpdatedMethods,
},
});
for (const method of addedMethods) {
interfaceChanges.added.push({
...getModuleBasics(currentModule),
method,
});
}
for (const method of deletedMethods) {
interfaceChanges.deleted.push({
...getModuleBasics(currentModule),
method,
});
}
for (const method of paramsUpdatedMethods) {
interfaceChanges.paramsUpdated.push({
...getModuleBasics(currentModule),
method,
});
}
for (const method of responseUpdatedMethods) {
interfaceChanges.responseUpdated.push({
...getModuleBasics(currentModule),
method,
});
}
}
for (const oldFolder of deletedFolders) {
const oldModule = previousModuleMap.get(oldFolder);
const overlaps = [];
for (const [newFolder, newModule] of currentModuleMap.entries()) {
const sharedMethods = getSetIntersection(oldModule.methodSet, newModule.methodSet);
if (sharedMethods.length === 0) {
continue;
}
overlaps.push({
...getModuleBasics(newModule),
sharedMethods: sharedMethods.sort((a, b) => a.localeCompare(b, "zh-CN")),
});
}
if (overlaps.length >= 2) {
moduleChanges.split.push({
from: getModuleBasics(oldModule),
to: overlaps,
});
continue;
}
if (overlaps.length === 1 && addedFolders.includes(overlaps[0].folder)) {
moduleChanges.replaced.push({
from: getModuleBasics(oldModule),
to: {
moduleName: overlaps[0].moduleName,
moduleKey: overlaps[0].moduleKey,
folder: overlaps[0].folder,
},
sharedMethods: overlaps[0].sharedMethods,
});
}
}
for (const newFolder of addedFolders) {
const newModule = currentModuleMap.get(newFolder);
const overlaps = [];
for (const [oldFolder, oldModule] of previousModuleMap.entries()) {
const sharedMethods = getSetIntersection(oldModule.methodSet, newModule.methodSet);
if (sharedMethods.length === 0) {
continue;
}
overlaps.push({
...getModuleBasics(oldModule),
sharedMethods: sharedMethods.sort((a, b) => a.localeCompare(b, "zh-CN")),
});
}
if (overlaps.length >= 2) {
moduleChanges.merged.push({
to: getModuleBasics(newModule),
from: overlaps,
});
}
}
const prevOwnership = buildMethodOwnershipMap(previousModuleMap);
const currOwnership = buildMethodOwnershipMap(currentModuleMap);
return {
moduleChanges,
interfaceChanges,
perModuleDiff,
summary: {
moduleAdded: moduleChanges.added.length,
moduleDeleted: moduleChanges.deleted.length,
moduleUpdated: moduleChanges.updated.length,
moduleSplit: moduleChanges.split.length,
moduleMerged: moduleChanges.merged.length,
moduleReplaced: moduleChanges.replaced.length,
interfaceAdded: interfaceChanges.added.length,
interfaceDeleted: interfaceChanges.deleted.length,
interfaceParamsUpdated: interfaceChanges.paramsUpdated.length,
interfaceResponseUpdated: interfaceChanges.responseUpdated.length,
movedAcrossModules: [...currOwnership.keys()].filter((method) => prevOwnership.get(method) !== currOwnership.get(method))
.length,
},
};
}
function buildPendingLegacy(moduleChanges, interfaceChanges) {
const grouped = new Map();
const pushMethod = (moduleLike, method) => {
const key = moduleLike.folder;
const current = grouped.get(key) ?? {
moduleName: moduleLike.moduleName,
moduleKey: moduleLike.moduleKey,
methods: [],
};
current.methods.push(method);
grouped.set(key, current);
};
for (const item of interfaceChanges.added) {
pushMethod(item, item.method);
}
for (const item of interfaceChanges.deleted) {
pushMethod(item, item.method);
}
for (const item of interfaceChanges.paramsUpdated) {
pushMethod(item, item.method);
}
for (const item of interfaceChanges.responseUpdated) {
pushMethod(item, item.method);
}
for (const item of moduleChanges.added) {
for (const method of item.methods) {
pushMethod(item, method);
}
}
return [...grouped.values()]
.map((item) => ({
moduleName: item.moduleName,
moduleKey: item.moduleKey,
methods: [...new Set(item.methods)].sort((a, b) => a.localeCompare(b, "zh-CN")),
}))
.sort((a, b) => a.moduleName.localeCompare(b.moduleName, "zh-CN"));
}
async function run() {
const pendingUpdates = await readJsonIfExists(API_DOC_SYSTEM_CONFIG.file.pendingUpdates, null);
const hasLegacyPending = Array.isArray(pendingUpdates) && pendingUpdates.length > 0;
const hasRichPending = Boolean(
pendingUpdates &&
!Array.isArray(pendingUpdates) &&
(pendingUpdates?.summary?.totalChangedItems ?? 0) > 0,
);
if (hasLegacyPending || hasRichPending) {
console.log("还有未更新接口等待处理!请处理待更新接口再执行此程序。");
process.exitCode = 1;
return;
}
const { source, html } = await fetchDocHtml();
const filterSnapshot = assertFilterOptionsAvailable(html);
const groups = parseGroupsFromHtml(html);
if (groups.length === 0) {
throw new Error("未解析出任何模块,请检查文档结构是否变更。");
}
const now = new Date();
const nowIso = now.toISOString();
const nowLocal = formatLocalDateTime(now);
const todayStamp = getDateStamp(now);
const nowStamp = getDateTimeStamp(now);
const previousModuleSnapshots = await readLatestModuleSnapshots(API_DOC_SYSTEM_CONFIG.file.moduleDir);
const previousStatuses = await readPreviousStatusesFromModules(API_DOC_SYSTEM_CONFIG.file.moduleDir);
const moduleListItems = [];
const allInterfaces = [];
const activeFolderNames = new Set();
const currentModuleComparableMap = new Map();
const previousActiveComparableMap = new Map();
for (const [folderName, snapshot] of previousModuleSnapshots.entries()) {
if (snapshot.lifecycleStatus === "deleted") {
continue;
}
const moduleComparable = toModuleComparable({
moduleName: snapshot?.data?.module?.name,
moduleKey: snapshot?.data?.module?.key,
folder: snapshot?.data?.module?.folder ?? folderName,
interfaces: snapshot?.data?.interfaces ?? [],
});
previousActiveComparableMap.set(folderName, moduleComparable);
}
await mkdir(API_DOC_SYSTEM_CONFIG.file.moduleDir, { recursive: true });
for (const group of groups) {
const moduleFolderPath = path.join(API_DOC_SYSTEM_CONFIG.file.moduleDir, group.folderName);
activeFolderNames.add(group.folderName);
await upsertModuleLifecycle({
folderPath: moduleFolderPath,
moduleMeta: {
name: group.moduleName,
key: group.moduleKey,
folder: group.folderName,
},
nowIso,
});
const previousModule = previousModuleSnapshots.get(group.folderName)?.data;
const previousInterfaces = Array.isArray(previousModule?.interfaces) ? previousModule.interfaces : [];
const previousInterfaceMap = new Map(previousInterfaces.map((item) => [item.method, item]));
const moduleInterfaces = group.methods.map((methodInfo) => {
const status = previousStatuses.get(methodInfo.method) ?? {
isUsed: false,
isPlaceholder: false,
};
return {
method: methodInfo.method,
title: methodInfo.title,
description: methodInfo.description,
endpoint: methodInfo.endpoint,
authRequired: methodInfo.authRequired,
loginRequired: methodInfo.loginRequired,
status,
headers: methodInfo.headers,
params: methodInfo.params,
response: methodInfo.response,
};
});
const moduleComparable = toModuleComparable({
moduleName: group.moduleName,
moduleKey: group.moduleKey,
folder: group.folderName,
interfaces: moduleInterfaces,
});
currentModuleComparableMap.set(group.folderName, moduleComparable);
const currentMethodSet = new Set(moduleInterfaces.map((item) => item.method));
const previousMethodSet = new Set(previousInterfaces.map((item) => item.method));
const addedMethods = getSetDiff(currentMethodSet, previousMethodSet).sort((a, b) => a.localeCompare(b, "zh-CN"));
const deletedMethods = getSetDiff(previousMethodSet, currentMethodSet).sort((a, b) => a.localeCompare(b, "zh-CN"));
const sharedMethods = getSetIntersection(currentMethodSet, previousMethodSet);
const paramsUpdatedMethods = [];
const responseUpdatedMethods = [];
for (const method of sharedMethods) {
const oldItem = previousInterfaceMap.get(method);
const newItem = moduleInterfaces.find((entry) => entry.method === method);
if (!isSameJson(oldItem?.params, newItem?.params)) {
paramsUpdatedMethods.push(method);
}
if (!isSameJson(oldItem?.response, newItem?.response)) {
responseUpdatedMethods.push(method);
}
}
paramsUpdatedMethods.sort((a, b) => a.localeCompare(b, "zh-CN"));
responseUpdatedMethods.sort((a, b) => a.localeCompare(b, "zh-CN"));
const moduleFilePath = path.join(API_DOC_SYSTEM_CONFIG.file.moduleDir, group.folderName, `${todayStamp}.json`);
const recentModuleVersions = await listRecentSnapshots(moduleFolderPath);
const modulePayload = {
generatedAt: nowIso,
generatedAtLocal: nowLocal,
source,
filter: filterSnapshot,
versionWindow: {
keepLatest: API_DOC_SYSTEM_CONFIG.retentionDays,
recentVersions: recentModuleVersions,
},
module: {
name: group.moduleName,
key: group.moduleKey,
folder: group.folderName,
interfaceCount: group.methods.length,
},
changes: {
fromPreviousSnapshot: {
moduleAdded: !previousModule,
moduleRenamed: previousModule
? previousModule?.module?.name !== group.moduleName || previousModule?.module?.key !== group.moduleKey
: false,
interfaceChanges: {
addedMethods,
deletedMethods,
paramsUpdatedMethods,
responseUpdatedMethods,
},
},
},
interfaces: moduleInterfaces,
};
await writeJson(moduleFilePath, modulePayload);
await pruneDailyFiles(path.dirname(moduleFilePath), API_DOC_SYSTEM_CONFIG.retentionDays);
moduleListItems.push({
moduleName: group.moduleName,
moduleKey: group.moduleKey,
folder: group.folderName,
interfaceCount: moduleInterfaces.length,
interfaces: moduleInterfaces.map((item) => ({
method: item.method,
isUsed: item.status.isUsed,
isPlaceholder: item.status.isPlaceholder,
})),
});
for (const methodInfo of moduleInterfaces) {
allInterfaces.push({
moduleName: group.moduleName,
moduleKey: group.moduleKey,
moduleFolder: group.folderName,
method: methodInfo.method,
title: methodInfo.title,
description: methodInfo.description,
endpoint: methodInfo.endpoint,
authRequired: methodInfo.authRequired,
loginRequired: methodInfo.loginRequired,
status: methodInfo.status,
headers: methodInfo.headers,
params: methodInfo.params,
response: methodInfo.response,
});
}
}
const existingFolders = await listModuleFolders(API_DOC_SYSTEM_CONFIG.file.moduleDir);
let deletedModuleCount = 0;
for (const folderName of existingFolders) {
if (activeFolderNames.has(folderName)) {
continue;
}
const folderPath = path.join(API_DOC_SYSTEM_CONFIG.file.moduleDir, folderName);
const marked = await markModuleFolderDeleted({
folderPath,
folderName,
nowIso,
todayStamp,
});
await pruneDailyFiles(folderPath, API_DOC_SYSTEM_CONFIG.retentionDays);
if (marked) {
deletedModuleCount += 1;
}
}
const changeBundle = detectModuleAndInterfaceChanges(previousActiveComparableMap, currentModuleComparableMap);
const pendingLegacy = buildPendingLegacy(changeBundle.moduleChanges, changeBundle.interfaceChanges);
const totalChangedItems =
changeBundle.summary.moduleAdded +
changeBundle.summary.moduleDeleted +
changeBundle.summary.moduleUpdated +
changeBundle.summary.moduleSplit +
changeBundle.summary.moduleMerged +
changeBundle.summary.moduleReplaced +
changeBundle.summary.interfaceAdded +
changeBundle.summary.interfaceDeleted +
changeBundle.summary.interfaceParamsUpdated +
changeBundle.summary.interfaceResponseUpdated;
const moduleListFilePath = path.join(API_DOC_SYSTEM_CONFIG.file.moduleListDir, `${todayStamp}.json`);
const moduleListRecentVersions = await listRecentSnapshots(API_DOC_SYSTEM_CONFIG.file.moduleListDir);
await writeJson(moduleListFilePath, {
generatedAt: nowIso,
generatedAtLocal: nowLocal,
source,
filter: filterSnapshot,
versionWindow: {
keepLatest: API_DOC_SYSTEM_CONFIG.retentionDays,
recentVersions: moduleListRecentVersions,
},
moduleCount: moduleListItems.length,
totalInterfaceCount: allInterfaces.length,
changes: {
summary: changeBundle.summary,
moduleUpdates: changeBundle.moduleChanges,
interfaceUpdates: changeBundle.interfaceChanges,
},
modules: moduleListItems.sort((a, b) => a.moduleName.localeCompare(b.moduleName, "zh-CN")),
});
await pruneDailyFiles(API_DOC_SYSTEM_CONFIG.file.moduleListDir, API_DOC_SYSTEM_CONFIG.retentionDays);
const detailFilePath = path.join(API_DOC_SYSTEM_CONFIG.file.interfaceDetailDir, `${todayStamp}.json`);
const interfaceDetailRecentVersions = await listRecentSnapshots(API_DOC_SYSTEM_CONFIG.file.interfaceDetailDir);
await writeJson(detailFilePath, {
generatedAt: nowIso,
generatedAtLocal: nowLocal,
source,
filter: filterSnapshot,
versionWindow: {
keepLatest: API_DOC_SYSTEM_CONFIG.retentionDays,
recentVersions: interfaceDetailRecentVersions,
},
totalInterfaceCount: allInterfaces.length,
changes: {
summary: changeBundle.summary,
moduleUpdates: changeBundle.moduleChanges,
interfaceUpdates: changeBundle.interfaceChanges,
},
interfaces: allInterfaces.sort((a, b) => a.method.localeCompare(b.method, "zh-CN")),
});
await pruneDailyFiles(API_DOC_SYSTEM_CONFIG.file.interfaceDetailDir, API_DOC_SYSTEM_CONFIG.retentionDays);
const pendingHistoryFilePath = path.join(API_DOC_SYSTEM_CONFIG.file.pendingHistoryDir, `${nowStamp}.json`);
const pendingPayload = {
generatedAt: nowIso,
generatedAtLocal: nowLocal,
source,
filter: filterSnapshot,
summary: {
...changeBundle.summary,
totalChangedItems,
},
moduleUpdates: changeBundle.moduleChanges,
interfaceUpdates: changeBundle.interfaceChanges,
legacy: pendingLegacy,
};
await writeJson(API_DOC_SYSTEM_CONFIG.file.pendingUpdates, pendingPayload);
await writeJson(pendingHistoryFilePath, pendingPayload);
await pruneFilesByPattern(
API_DOC_SYSTEM_CONFIG.file.pendingHistoryDir,
API_DOC_SYSTEM_CONFIG.retentionDays,
/^\d{14}\.json$/,
);
const pendingHistoryRecentVersions = await readdir(API_DOC_SYSTEM_CONFIG.file.pendingHistoryDir)
.then((files) => files.filter((fileName) => /^\d{14}\.json$/.test(fileName)).sort().reverse().slice(0, 5))
.catch(() => []);
await writeJson(API_DOC_SYSTEM_CONFIG.file.pendingUpdates, {
...pendingPayload,
versionWindow: {
keepLatest: API_DOC_SYSTEM_CONFIG.retentionDays,
recentVersions: pendingHistoryRecentVersions.map((fileName) => ({
stamp: fileName.replace(".json", ""),
displayTime: formatStampToLocalDateTime(fileName.replace(".json", "")),
filePath: path.join(API_DOC_SYSTEM_CONFIG.file.pendingHistoryDir, fileName),
})),
},
});
console.log(`执行时间: ${nowLocal}`);
console.log(`API 文档已清洗完成,模块数: ${moduleListItems.length},接口数: ${allInterfaces.length}`);
console.log(`本次标记删除模块文件夹: ${deletedModuleCount}`);
console.log(
`模块变化统计 -> 新增:${changeBundle.summary.moduleAdded} 删除:${changeBundle.summary.moduleDeleted} 更新:${changeBundle.summary.moduleUpdated} 拆分:${changeBundle.summary.moduleSplit} 合并:${changeBundle.summary.moduleMerged} 替换:${changeBundle.summary.moduleReplaced}`,
);
console.log(
`接口变化统计 -> 新增:${changeBundle.summary.interfaceAdded} 删除:${changeBundle.summary.interfaceDeleted} Params更新:${changeBundle.summary.interfaceParamsUpdated} Response更新:${changeBundle.summary.interfaceResponseUpdated}`,
);
console.log(`模块列表文件: ${moduleListFilePath}`);
console.log(`接口明细文件: ${detailFilePath}`);
console.log(`待更新接口文件: ${API_DOC_SYSTEM_CONFIG.file.pendingUpdates}`);
}
run().catch((error) => {
console.error("执行 API 文档抓取与清洗失败:", error);
process.exitCode = 1;
});