WMS_MAIN/app/lib/file_writer.js
2026-04-27 16:09:26 +08:00

1575 lines
49 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

'use strict';
const fs = require('node:fs');
const fsp = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024;
const DEFAULT_MAX_FILE_LINES = 2000;
const DEFAULT_MAX_CONSUME_RECORDS_PER_WINDOW = 1000;
const DEFAULT_WRITE_WINDOW_SECONDS = 1;
const DEFAULT_MAX_WRITE_RECORDS_PER_WINDOW = 2000;
const DEFAULT_RETENTION_DAYS = 7;
const DEFAULT_CLEANUP_INTERVAL_MS = 60 * 60 * 1000;
const DEFAULT_STATE_MAP_MAX_ENTRIES = 256;
const DEFAULT_WRITE_WINDOW_BUCKET_COUNT = 8;
/**
* 本地文件队列写入器。
* 负责把业务数据按 JSON Lines 追加到本地文件,并保证单条写入串行、fsync 落盘和自动轮转。
*/
class LocalFileWriter {
/**
* 创建本地文件写入器实例。
* @param {Egg.Application} app Egg 应用实例。
*/
constructor(app) {
this.app = app;
this.config = app.config.fileWriter || {};
this.baseDir = path.resolve(this.config.baseDir || path.join(app.baseDir, 'runtime', 'file-writer'));
this.maxFileSize = this.normalizeSize(this.config.maxFileSize, DEFAULT_MAX_FILE_SIZE);
this.maxFileLines = this.normalizePositiveInteger(this.config.maxFileLines, DEFAULT_MAX_FILE_LINES);
this.maxConsumeRecordsPerWindow = this.normalizePositiveInteger(
this.config.maxConsumeRecordsPerWindow
?? this.config.maxConsumeRecordsPerRun
?? app.config.maxConsumeRecordsPerWindow
?? app.config.maxConsumeRecordsPerRun,
DEFAULT_MAX_CONSUME_RECORDS_PER_WINDOW
);
this.maxConsumeRecordsPerRun = this.maxConsumeRecordsPerWindow;
this.writeWindowSeconds = this.normalizeInteger(
this.config.writeWindowSeconds
?? app.config.writeWindowSeconds,
DEFAULT_WRITE_WINDOW_SECONDS
);
this.maxWriteRecordsPerWindow = this.normalizeInteger(
this.config.maxWriteRecordsPerWindow
?? app.config.maxWriteRecordsPerWindow,
DEFAULT_MAX_WRITE_RECORDS_PER_WINDOW
);
this.writeWindowStartedAt = 0;
this.writeWindowRecordCount = 0;
this.writeWindowBucketCount = this.normalizePositiveInteger(
this.config.writeWindowBucketCount
?? app.config.writeWindowBucketCount,
DEFAULT_WRITE_WINDOW_BUCKET_COUNT
);
this.preferredBucket = this.hashStringToIndex(this.workerId + '-' + this.pid, this.writeWindowBucketCount);
this.retentionDays = this.normalizeInteger(this.config.retentionDays, DEFAULT_RETENTION_DAYS);
this.cleanupIntervalMs = this.normalizeInteger(this.config.cleanupIntervalMs, DEFAULT_CLEANUP_INTERVAL_MS);
this.machineId = this.sanitizeSegment(this.config.machineId || os.hostname() || 'machine');
this.workerId = this.sanitizeSegment(this.config.workerId || process.env.EGG_WORKER_ID || 'worker');
this.pid = process.pid;
this.processTag = `${this.machineId}-${this.workerId}-p${this.pid}`;
this.sequenceWidth = this.normalizeInteger(this.config.sequenceWidth, 4);
this.stateMapMaxEntries = this.normalizePositiveInteger(this.config.stateMapMaxEntries, DEFAULT_STATE_MAP_MAX_ENTRIES);
this.fileExtension = this.normalizeExtension(this.config.fileExtension, '.log');
this.progressExtension = this.normalizeExtension(this.config.progressExtension, '.progress');
this.doneExtension = this.normalizeExtension(this.config.doneExtension, '.done');
this.stateMap = new Map();
this.queue = Promise.resolve();
this.lastCleanupAt = 0;
}
/**
* 将写入任务串行排队执行。
* @param {Function} task 需要排队执行的异步任务。
* @return {Promise<*>} 任务执行结果。
*/
enqueue(task) {
const currentTask = this.queue.then(() => task());
this.queue = currentTask.then(() => undefined, () => undefined);
return currentTask;
}
/**
* 等待当前队列里的所有写入完成。
* @return {Promise<void>} 等待完成的 Promise。
*/
async flush() {
await this.queue;
}
/**
* 把业务数据写入本地 JSON Lines 文件。
* @param {string} businessName 业务名称。
* @param {*} record 需要写入的数据。
* @param {Object} [options] 写入选项。
* @param {Date|string|number} [options.date] 指定归档日期,默认使用当前时间。
* @return {Promise<Object>} 写入结果。
*/
async writeRecord(businessName, record, options = {}) {
return this.enqueue(async () => {
const windowCheck = await this.atomicCheckAndIncrementWriteWindow();
this.writeWindowStartedAt = windowCheck.counter.startedAt;
this.writeWindowRecordCount = windowCheck.counter.recordCount;
if (!windowCheck.allowed) {
const writeWindowMs = this.writeWindowSeconds * 1000;
const resumeAt = new Date(windowCheck.counter.startedAt + writeWindowMs).toISOString();
return {
throttled: true,
businessName,
resumeAt,
writeWindowSeconds: this.writeWindowSeconds,
writeWindowRecordCount: windowCheck.counter.recordCount,
maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow,
};
}
const partitionState = await this.getPartitionState(businessName, options.date);
const jsonLine = this.stringifyRecord(record);
const payload = `${jsonLine}\n`;
const payloadSize = Buffer.byteLength(payload);
if (
partitionState.currentLineCount >= this.maxFileLines
|| (partitionState.currentSize > 0 && partitionState.currentSize + payloadSize > this.maxFileSize)
) {
partitionState.sequence += 1;
partitionState.currentFilePath = this.buildDataFilePath(partitionState, partitionState.sequence);
partitionState.currentSize = 0;
partitionState.currentLineCount = 0;
}
await this.ensureParentDirectory(partitionState.partitionDir);
const handle = await fsp.open(partitionState.currentFilePath, 'a');
try {
await handle.write(payload);
await handle.sync();
} finally {
await handle.close();
}
partitionState.currentSize += payloadSize;
partitionState.currentLineCount += 1;
await this.maybeCleanup();
return {
businessName: partitionState.businessName,
dateKey: partitionState.dateKey,
filePath: partitionState.currentFilePath,
size: partitionState.currentSize,
lineCount: partitionState.currentLineCount,
bytesWritten: payloadSize,
sequence: partitionState.sequence,
machineId: this.machineId,
workerId: this.workerId,
pid: this.pid,
throttled: false,
writeWindowRecordCount: windowCheck.counter.recordCount,
maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow,
};
});
}
/**
* 写入方法别名,便于业务侧直接调用。
* @param {string} businessName 业务名称。
* @param {*} record 需要写入的数据。
* @param {Object} [options] 写入选项。
* @return {Promise<Object>} 写入结果。
*/
async write(businessName, record, options = {}) {
return this.writeRecord(businessName, record, options);
}
/**
* 获取某个业务在指定日期下的目录信息。
* @param {string} businessName 业务名称。
* @param {Date|string|number} [date] 指定日期。
* @return {Object} 目录信息。
*/
getPartitionInfo(businessName, date = new Date()) {
const businessSegment = this.sanitizeSegment(businessName || 'default');
const dateParts = this.getDateParts(date);
const partitionDir = path.join(this.baseDir, businessSegment, dateParts.year, dateParts.month, dateParts.day);
return {
businessName,
businessSegment,
dateKey: dateParts.dateKey,
partitionDir,
filePrefix: `${businessSegment}-${this.processTag}`,
processTag: this.processTag,
};
}
/**
* 获取消费进度文件路径。
* @param {string} businessName 业务名称。
* @param {Date|string|number|null} [date] 指定日期;传空时使用业务级共享进度文件。
* @param {string} [consumerName] 消费者名称。
* @return {string} 进度文件路径。
*/
getProgressFilePath(businessName, date = new Date(), consumerName = 'default') {
const businessSegment = this.sanitizeSegment(businessName || 'default');
const safeConsumerName = this.sanitizeSegment(consumerName || 'default');
if (date == null) {
return path.join(this.baseDir, businessSegment, `.${safeConsumerName}${this.progressExtension}`);
}
const partitionInfo = this.getPartitionInfo(businessName, date);
return path.join(partitionInfo.partitionDir, `.${safeConsumerName}${this.progressExtension}`);
}
/**
* 获取某个数据文件的消费完成标记路径。
* @param {string} filePath 数据文件路径。
* @return {string} 完成标记路径。
*/
getDoneFilePath(filePath) {
const resolvedFilePath = path.resolve(filePath);
return `${resolvedFilePath}${this.doneExtension}`;
}
/**
* 列出某个业务日期分区下的所有日志文件。
* @param {string} businessName 业务名称。
* @param {Date|string|number|null} [date] 指定日期;传空时递归列出业务目录下所有日志文件。
* @return {Promise<Array<string>>} 排序后的文件路径列表。
*/
async listDataFiles(businessName, date = new Date()) {
const businessSegment = this.sanitizeSegment(businessName || 'default');
const businessDir = path.join(this.baseDir, businessSegment);
const targetDir = date == null ? businessDir : this.getPartitionInfo(businessName, date).partitionDir;
const files = await this.collectDataFiles(targetDir);
return files
.sort((left, right) => {
return this.compareDataFileSortKey(left.sortKey, right.sortKey);
})
.map(entry => entry.filePath);
}
/**
* 读取单个日志文件中的 JSON Lines 数据。
* @param {string} filePath 日志文件路径。
* @param {Object} [options] 读取选项。
* @param {number} [options.startOffset=0] 起始字节偏移量。
* @param {number} [options.maxRecords=0] 最大读取条数,0 或不传不限制。
* @param {boolean} [options.parseJson=true] 是否自动解析 JSON。
* @return {Promise<Object>} 读取结果。
*/
async readRecords(filePath, options = {}) {
const resolvedFilePath = path.resolve(filePath);
const startOffset = this.normalizeInteger(options.startOffset, 0);
const maxRecords = this.normalizeInteger(options.maxRecords, 0);
const parseJson = options.parseJson !== false;
const records = [];
let nextOffset = startOffset;
let lineNumber = 0;
try {
for await (const lineEntry of this.iterateJsonLines(resolvedFilePath, startOffset)) {
lineNumber = lineEntry.lineNumber;
nextOffset = lineEntry.nextOffset;
if (!lineEntry.line.trim()) {
continue;
}
if (!parseJson) {
records.push(lineEntry.line);
if (maxRecords > 0 && records.length >= maxRecords) {
break;
}
continue;
}
try {
records.push(JSON.parse(lineEntry.line));
if (maxRecords > 0 && records.length >= maxRecords) {
break;
}
} catch (parseError) {
parseError.message = `解析文件 ${resolvedFilePath} 第 ${lineNumber} 行失败: ${parseError.message}`;
throw parseError;
}
}
} catch (error) {
if (error.code === 'ENOENT') {
return {
filePath: resolvedFilePath,
records: [],
nextOffset: startOffset,
lineNumber: 0,
};
}
throw error;
}
return {
filePath: resolvedFilePath,
records,
nextOffset,
lineNumber,
};
}
/**
* 消费某个文件,并在每条记录处理完成后更新进度文件。
* @param {string} filePath 日志文件路径。
* @param {Object} options 消费选项。
* @param {string} options.progressFilePath 进度文件路径。
* @param {Function} options.onRecord 每条记录的处理函数。
* @param {number} [options.startOffset=0] 起始偏移量。
* @param {boolean} [options.parseJson=true] 是否自动解析 JSON。
* @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。
* @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。
* @param {Object|null} [options.progress] 已加载的进度数据,供内部复用。
* @return {Promise<Object>} 消费结果,包含 filePath、processedCount、nextOffset、limitReached、maxConsumeRecordsPerWindow。
*/
async consumeFile(filePath, options = {}) {
const progressFilePath = options.progressFilePath;
const onRecord = options.onRecord;
if (!progressFilePath) {
throw new Error('progressFilePath 不能为空');
}
if (typeof onRecord !== 'function') {
throw new Error('onRecord 必须是函数');
}
const progress = options.progress || await this.loadProgress(progressFilePath);
const consumeLimit = this.normalizePositiveInteger(
options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun,
this.maxConsumeRecordsPerWindow
);
const resolvedFilePath = path.resolve(filePath);
const startOffset = this.resolveConsumeStartOffset(progress, resolvedFilePath, options);
const parseJson = options.parseJson !== false;
let offset = startOffset;
let processedCount = 0;
let lineNumber = 0;
let limitReached = false;
try {
for await (const lineEntry of this.iterateJsonLines(resolvedFilePath, startOffset)) {
if (consumeLimit > 0 && processedCount >= consumeLimit) {
limitReached = true;
break;
}
lineNumber = lineEntry.lineNumber;
offset = lineEntry.nextOffset;
if (!lineEntry.line.trim()) {
continue;
}
let record = lineEntry.line;
if (parseJson) {
try {
record = JSON.parse(lineEntry.line);
} catch (parseError) {
parseError.message = `解析文件 ${resolvedFilePath} 第 ${lineNumber} 行失败: ${parseError.message}`;
throw parseError;
}
}
await onRecord(record, {
filePath: resolvedFilePath,
offset,
index: processedCount,
});
processedCount += 1;
const progressData = {
filePath: resolvedFilePath,
offset,
processedCount,
updatedAt: new Date().toISOString(),
};
await this.saveProgress(progressFilePath, progressData);
if (consumeLimit > 0 && processedCount >= consumeLimit) {
limitReached = true;
break;
}
}
} catch (error) {
if (error.code === 'ENOENT') {
return {
filePath: resolvedFilePath,
processedCount: 0,
nextOffset: startOffset,
limitReached: false,
maxConsumeRecordsPerRun: consumeLimit,
maxConsumeRecordsPerWindow: consumeLimit,
};
}
throw error;
}
if (limitReached) {
return {
filePath: resolvedFilePath,
processedCount,
nextOffset: offset,
limitReached: true,
maxConsumeRecordsPerRun: consumeLimit,
maxConsumeRecordsPerWindow: consumeLimit,
};
}
const completionData = {
filePath: resolvedFilePath,
offset,
processedCount,
completedAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
};
await this.saveProgress(progressFilePath, completionData);
await this.saveCompletionMarker(resolvedFilePath, completionData);
return {
filePath: resolvedFilePath,
processedCount,
nextOffset: offset,
limitReached: false,
maxConsumeRecordsPerRun: consumeLimit,
maxConsumeRecordsPerWindow: consumeLimit,
};
}
/**
* 按业务目录消费所有日志文件,支持断点续传。
* @param {Object} options 消费选项。
* @param {string} options.businessName 业务名称。
* @param {string} options.progressFilePath 进度文件路径。
* @param {Function} options.onRecord 每条记录的处理函数。
* @param {boolean} [options.parseJson=true] 是否自动解析 JSON。
* @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。
* @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。
* @return {Promise<Object>} 消费结果,包含 businessName、dataFiles、processedCount、progress、limitReached、maxConsumeRecordsPerWindow。
*/
async consumeFiles(options = {}) {
const businessName = options.businessName;
const progressFilePath = options.progressFilePath;
const onRecord = options.onRecord;
if (!businessName) {
throw new Error('businessName 不能为空');
}
if (!progressFilePath) {
throw new Error('progressFilePath 不能为空');
}
if (typeof onRecord !== 'function') {
throw new Error('onRecord 必须是函数');
}
const progress = await this.loadProgress(progressFilePath);
const consumeLimit = this.normalizePositiveInteger(
options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun,
this.maxConsumeRecordsPerWindow
);
const dataFiles = await this.listDataFiles(businessName, null);
if (dataFiles.length === 0) {
return {
businessName,
dataFiles,
processedCount: 0,
progress,
limitReached: false,
maxConsumeRecordsPerRun: consumeLimit,
maxConsumeRecordsPerWindow: consumeLimit,
};
}
let startIndex = 0;
let startOffset = 0;
let limitReached = false;
if (progress && progress.filePath) {
const matchedIndex = dataFiles.findIndex(filePath => path.resolve(filePath) === path.resolve(progress.filePath));
if (matchedIndex >= 0) {
startIndex = matchedIndex;
startOffset = this.normalizeInteger(progress.offset, 0);
} else {
const progressSortKey = this.getDataFileSortKey(progress.filePath);
const fallbackIndex = dataFiles.findIndex(filePath => this.compareDataFileSortKey(
this.getDataFileSortKey(filePath),
progressSortKey
) > 0);
if (fallbackIndex >= 0) {
startIndex = fallbackIndex;
startOffset = 0;
} else {
startIndex = dataFiles.length;
}
}
}
let processedCount = 0;
let currentProgress = progress;
let totalConsumedCount = 0;
for (let index = startIndex; index < dataFiles.length; index += 1) {
if (consumeLimit > 0 && totalConsumedCount >= consumeLimit) {
limitReached = true;
break;
}
const filePath = dataFiles[index];
const completion = await this.loadCompletionMarker(filePath);
if (completion && path.resolve(completion.filePath || filePath) === path.resolve(filePath)) {
currentProgress = {
...completion,
filePath: completion.filePath || filePath,
updatedAt: new Date().toISOString(),
};
await this.saveProgress(progressFilePath, currentProgress);
continue;
}
const remainingLimit = consumeLimit > 0 ? consumeLimit - totalConsumedCount : consumeLimit;
const fileResult = await this.consumeFile(filePath, {
progressFilePath,
onRecord,
startOffset: index === startIndex ? startOffset : 0,
parseJson: options.parseJson,
maxConsumeRecordsPerWindow: remainingLimit,
progress,
});
processedCount += fileResult.processedCount;
totalConsumedCount += fileResult.processedCount;
currentProgress = {
filePath,
offset: fileResult.nextOffset,
processedCount,
updatedAt: new Date().toISOString(),
};
await this.saveProgress(progressFilePath, currentProgress);
if (fileResult.limitReached) {
limitReached = true;
break;
}
}
return {
businessName,
dataFiles,
processedCount,
progress: currentProgress,
limitReached,
maxConsumeRecordsPerRun: consumeLimit,
maxConsumeRecordsPerWindow: consumeLimit,
};
}
/**
* 递归收集目录下的所有数据文件,并记录用于排序的分区日期与序号。
* @param {string} dirPath 目录路径。
* @param {Array<Object>} [result=[]] 累积结果。
* @return {Promise<Array<Object>>} 收集结果。
*/
async collectDataFiles(dirPath, result = []) {
let entries;
try {
entries = await fsp.readdir(dirPath, { withFileTypes: true });
} catch (error) {
if (error.code === 'ENOENT') {
return result;
}
throw error;
}
for (const entry of entries) {
const entryPath = path.join(dirPath, entry.name);
if (entry.isDirectory()) {
await this.collectDataFiles(entryPath, result);
continue;
}
if (!entry.isFile() || !entry.name.endsWith(this.fileExtension)) {
continue;
}
result.push({
filePath: entryPath,
sortKey: this.getDataFileSortKey(entryPath),
});
}
return result;
}
/**
* 解析数据文件的消费顺序键,按分区日期和文件序号排序。
* @param {string} filePath 数据文件路径。
* @return {Object} 排序键。
*/
getDataFileSortKey(filePath) {
const resolvedFilePath = path.resolve(filePath);
const relativeParts = path.relative(this.baseDir, resolvedFilePath).split(path.sep).filter(Boolean);
const fileName = path.basename(resolvedFilePath);
const sequenceMatch = fileName.match(/-(\d+)\.[^.]+$/);
let partitionDateKey = '';
if (relativeParts.length >= 5) {
const year = relativeParts[1];
const month = relativeParts[2];
const day = relativeParts[3];
if (/^\d+$/.test(year) && /^\d+$/.test(month) && /^\d+$/.test(day)) {
partitionDateKey = `${year.padStart(4, '0')}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`;
}
}
return {
partitionDateKey,
sequence: sequenceMatch ? Number.parseInt(sequenceMatch[1], 10) : Number.MAX_SAFE_INTEGER,
filePath: resolvedFilePath,
};
}
/**
* 比较两个数据文件的排序键。
* @param {Object} left 左侧排序键。
* @param {Object} right 右侧排序键。
* @return {number} 排序结果。
*/
compareDataFileSortKey(left, right) {
if (left.partitionDateKey !== right.partitionDateKey) {
return left.partitionDateKey.localeCompare(right.partitionDateKey, 'en');
}
if (left.sequence !== right.sequence) {
return left.sequence - right.sequence;
}
return left.filePath.localeCompare(right.filePath, 'en', { numeric: true });
}
/**
* 读取进度文件。
* @param {string} progressFilePath 进度文件路径。
* @return {Promise<Object|null>} 进度内容。
*/
async loadProgress(progressFilePath) {
const resolvedPath = path.resolve(progressFilePath);
try {
const content = await fsp.readFile(resolvedPath, 'utf8');
if (!content.trim()) {
return null;
}
return this.normalizeProgressData(JSON.parse(content), resolvedPath);
} catch (error) {
if (error.code === 'ENOENT') {
return null;
}
if (error instanceof SyntaxError) {
this.app.logger.warn('[fileWriter.loadProgress] 忽略损坏的进度文件', resolvedPath, error.message);
return null;
}
throw error;
}
}
/**
* 保存进度文件。
* @param {string} progressFilePath 进度文件路径。
* @param {Object} progressData 进度数据。
* @return {Promise<Object>} 保存后的进度数据。
*/
async saveProgress(progressFilePath, progressData) {
const resolvedPath = path.resolve(progressFilePath);
const payload = `${JSON.stringify({
...progressData,
updatedAt: progressData.updatedAt || new Date().toISOString(),
}, null, 2)}\n`;
await this.writeAtomicTextFile(resolvedPath, payload);
return {
...progressData,
updatedAt: progressData.updatedAt || new Date().toISOString(),
};
}
/**
* 保存消费完成标记。
* @param {string} filePath 数据文件路径。
* @param {Object} completionData 完成数据。
* @return {Promise<Object>} 保存后的完成数据。
*/
async saveCompletionMarker(filePath, completionData) {
const resolvedFilePath = path.resolve(filePath);
const doneFilePath = this.getDoneFilePath(resolvedFilePath);
const payload = `${JSON.stringify({
...completionData,
filePath: resolvedFilePath,
completedAt: completionData.completedAt || new Date().toISOString(),
}, null, 2)}\n`;
await this.writeAtomicTextFile(doneFilePath, payload);
return {
...completionData,
filePath: resolvedFilePath,
completedAt: completionData.completedAt || new Date().toISOString(),
};
}
/**
* 读取消费完成标记。
* @param {string} filePath 数据文件路径。
* @return {Promise<Object|null>} 完成标记内容。
*/
async loadCompletionMarker(filePath) {
const resolvedFilePath = path.resolve(filePath);
const doneFilePath = this.getDoneFilePath(resolvedFilePath);
try {
const content = await fsp.readFile(doneFilePath, 'utf8');
if (!content.trim()) {
return null;
}
return this.normalizeCompletionData(JSON.parse(content), resolvedFilePath);
} catch (error) {
if (error.code === 'ENOENT') {
return null;
}
if (error instanceof SyntaxError) {
this.app.logger.warn('[fileWriter.loadCompletionMarker] 忽略损坏的完成标记', doneFilePath, error.message);
return null;
}
throw error;
}
}
/**
* 清理超过保留期的日志文件与进度文件。
* @return {Promise<Object>} 清理结果。
*/
async cleanupExpiredFiles() {
if (!this.retentionDays || this.retentionDays <= 0) {
return {
removedFiles: 0,
removedDirectories: 0,
};
}
const cutoffTime = Date.now() - (this.retentionDays * 24 * 60 * 60 * 1000);
const result = {
removedFiles: 0,
removedDirectories: 0,
};
await this.walkAndCleanup(this.baseDir, cutoffTime, result);
return result;
}
/**
* 规范化日期并返回目录分段。
* @param {Date|string|number} date 日期。
* @return {Object} 日期分段。
*/
getDateParts(date = new Date()) {
const targetDate = date instanceof Date ? date : new Date(date);
if (Number.isNaN(targetDate.getTime())) {
throw new Error(`无效的日期参数: ${date}`);
}
const year = String(targetDate.getFullYear());
const month = String(targetDate.getMonth() + 1).padStart(2, '0');
const day = String(targetDate.getDate()).padStart(2, '0');
return {
year,
month,
day,
dateKey: `${year}-${month}-${day}`,
};
}
/**
* 获取写入分区状态。
* @param {string} businessName 业务名称。
* @param {Date|string|number} [date] 指定日期。
* @return {Promise<Object>} 分区状态。
*/
async getPartitionState(businessName, date = new Date()) {
const partitionInfo = this.getPartitionInfo(businessName, date);
const stateKey = `${partitionInfo.businessSegment}:${partitionInfo.dateKey}`;
if (this.stateMap.has(stateKey)) {
const existingState = this.stateMap.get(stateKey);
this.stateMap.delete(stateKey);
this.stateMap.set(stateKey, existingState);
return existingState;
}
await this.ensureParentDirectory(partitionInfo.partitionDir);
const state = {
businessName,
businessSegment: partitionInfo.businessSegment,
dateKey: partitionInfo.dateKey,
partitionDir: partitionInfo.partitionDir,
filePrefix: partitionInfo.filePrefix,
sequence: 1,
currentSize: 0,
currentLineCount: 0,
currentFilePath: this.buildDataFilePath({
partitionDir: partitionInfo.partitionDir,
filePrefix: partitionInfo.filePrefix,
}, 1),
};
const latestDataFileState = await this.loadLatestDataFileState(partitionInfo.partitionDir, partitionInfo.filePrefix);
if (latestDataFileState) {
state.sequence = latestDataFileState.sequence;
state.currentSize = latestDataFileState.currentSize;
state.currentLineCount = latestDataFileState.currentLineCount;
state.currentFilePath = latestDataFileState.currentFilePath;
}
this.evictOldestStateIfNeeded();
this.stateMap.set(stateKey, state);
return state;
}
/**
* 读取分区中最新数据文件的大小与行数,用于恢复写入状态。
* @param {string} partitionDir 分区目录。
* @param {string} filePrefix 文件前缀。
* @return {Promise<Object|null>} 最新文件状态。
*/
async loadLatestDataFileState(partitionDir, filePrefix) {
const dataFiles = await this.collectDataFiles(partitionDir);
if (dataFiles.length === 0) {
return null;
}
dataFiles.sort((left, right) => this.compareDataFileSortKey(left.sortKey, right.sortKey));
const latestFile = dataFiles[dataFiles.length - 1];
const stat = await fsp.stat(latestFile.filePath);
const currentLineCount = await this.countFileLines(latestFile.filePath);
return {
filePrefix,
sequence: latestFile.sortKey.sequence > 0 ? latestFile.sortKey.sequence : 1,
currentFilePath: latestFile.filePath,
currentSize: stat.size,
currentLineCount,
};
}
/**
* 统计文件中的行数。
* @param {string} filePath 文件路径。
* @return {Promise<number>} 行数。
*/
async countFileLines(filePath) {
let lineCount = 0;
let lastByte = null;
const stream = fs.createReadStream(filePath);
try {
for await (const chunk of stream) {
lastByte = chunk.length > 0 ? chunk[chunk.length - 1] : lastByte;
for (let i = 0; i < chunk.length; i += 1) {
if (chunk[i] === 0x0A) {
lineCount += 1;
}
}
}
} finally {
stream.destroy();
}
if (lineCount === 0) {
return 0;
}
if (lastByte !== 0x0A) {
lineCount += 1;
}
return lineCount;
}
/**
* 逐行读取 JSON Lines 文件,并返回每行的精确字节偏移。
* @param {string} filePath 文件路径。
* @param {number} [startOffset=0] 起始字节偏移量。
* @return {AsyncGenerator<Object>} 逐行迭代器。
*/
async* iterateJsonLines(filePath, startOffset = 0) {
const stream = fs.createReadStream(filePath, {
start: startOffset,
});
let buffer = Buffer.alloc(0);
let offset = startOffset;
let lineNumber = 0;
try {
for await (const chunk of stream) {
buffer = Buffer.concat([ buffer, chunk ]);
while (buffer.length > 0) {
const lineFeedIndex = buffer.indexOf(0x0A);
if (lineFeedIndex < 0) {
break;
}
const lineEndIndex = lineFeedIndex > 0 && buffer[lineFeedIndex - 1] === 0x0D
? lineFeedIndex - 1
: lineFeedIndex;
const consumedBytes = lineFeedIndex + 1;
const lineBuffer = buffer.slice(0, lineEndIndex);
lineNumber += 1;
offset += consumedBytes;
buffer = buffer.slice(consumedBytes);
yield {
lineNumber,
line: lineBuffer.toString('utf8'),
nextOffset: offset,
};
}
}
if (buffer.length > 0) {
lineNumber += 1;
offset += buffer.length;
yield {
lineNumber,
line: buffer.toString('utf8'),
nextOffset: offset,
};
}
} finally {
stream.destroy();
}
}
/**
* 归一化进度文件内容,兼容损坏或旧字段。
* @param {*} progressData 原始进度内容。
* @param {string} [fallbackFilePath] 兜底文件路径。
* @return {Object|null} 归一化后的进度内容。
*/
normalizeProgressData(progressData, fallbackFilePath = null) {
if (!progressData || typeof progressData !== 'object') {
return null;
}
const filePath = typeof progressData.filePath === 'string' && progressData.filePath.trim()
? path.resolve(progressData.filePath)
: (fallbackFilePath || null);
const offset = this.normalizeInteger(progressData.offset ?? progressData.nextOffset, 0);
const processedCount = this.normalizeInteger(progressData.processedCount ?? progressData.processed ?? progressData.count, 0);
return {
...progressData,
filePath,
offset,
processedCount,
updatedAt: typeof progressData.updatedAt === 'string' && progressData.updatedAt.trim()
? progressData.updatedAt
: undefined,
completedAt: typeof progressData.completedAt === 'string' && progressData.completedAt.trim()
? progressData.completedAt
: undefined,
};
}
/**
* 归一化完成标记内容,兼容损坏或旧字段。
* @param {*} completionData 原始完成标记内容。
* @param {string} fallbackFilePath 兜底文件路径。
* @return {Object|null} 归一化后的完成标记内容。
*/
normalizeCompletionData(completionData, fallbackFilePath) {
if (!completionData || typeof completionData !== 'object') {
return null;
}
const filePath = typeof completionData.filePath === 'string' && completionData.filePath.trim()
? path.resolve(completionData.filePath)
: fallbackFilePath;
if (!filePath) {
return null;
}
const completedAtMs = this.resolveTimestamp(completionData.completedAt ?? completionData.updatedAt);
return {
...completionData,
filePath,
completedAt: this.formatTimestamp(completedAtMs),
updatedAt: typeof completionData.updatedAt === 'string' && completionData.updatedAt.trim()
? completionData.updatedAt
: undefined,
};
}
/**
* 回收最旧的分区状态,避免 stateMap 无界增长。
* @return {void}
*/
evictOldestStateIfNeeded() {
if (this.stateMap.size < this.stateMapMaxEntries) {
return;
}
const oldestKey = this.stateMap.keys().next().value;
if (oldestKey !== undefined) {
this.stateMap.delete(oldestKey);
}
}
/**
* 构建日志文件路径。
* @param {Object} partitionState 分区状态。
* @param {number} sequence 序号。
* @return {string} 文件路径。
*/
buildDataFilePath(partitionState, sequence) {
const sequenceText = String(sequence).padStart(this.sequenceWidth, '0');
const fileName = `${partitionState.filePrefix}-${sequenceText}${this.fileExtension}`;
return path.join(partitionState.partitionDir, fileName);
}
/**
* 把要写入的记录转换为 JSON 字符串。
* @param {*} record 记录内容。
* @return {string} JSON 字符串。
*/
stringifyRecord(record) {
const json = JSON.stringify(record, this.jsonReplacer);
if (json === undefined) {
throw new Error('写入内容无法被序列化为 JSON');
}
return json;
}
/**
* JSON 序列化时处理 BigInt。
* @param {string} key 属性名。
* @param {*} value 属性值。
* @return {*} 序列化后的值。
*/
jsonReplacer(key, value) {
if (typeof value === 'bigint') {
return value.toString();
}
return value;
}
/**
* 规范化目录片段,避免路径穿越。
* @param {*} value 待处理值。
* @return {string} 安全片段。
*/
sanitizeSegment(value) {
const normalized = String(value || '').trim();
if (!normalized) {
return 'default';
}
const safeValue = Array.from(normalized).map(character => {
const code = character.charCodeAt(0);
if (code < 32 || '<>:"/\\|?*'.includes(character)) {
return '_';
}
return character;
}).join('');
return safeValue
.replace(/\s+/g, '_')
.replace(/\.+/g, '.')
.replace(/^\.+|\.+$/g, '')
.replace(/_+/g, '_')
.slice(0, 80) || 'default';
}
/**
* 规范化扩展名。
* @param {*} value 扩展名。
* @param {string} fallback 回退扩展名。
* @return {string} 规范化后的扩展名。
*/
normalizeExtension(value, fallback) {
const raw = String(value || fallback || '').trim();
if (!raw) {
return fallback;
}
return raw.startsWith('.') ? raw : `.${raw}`;
}
/**
* 规范化数字。
* @param {*} value 数值或字符串。
* @param {number} fallback 回退值。
* @return {number} 规范化后的整数。
*/
normalizeInteger(value, fallback) {
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
}
/**
* 规范化大小配置。
* @param {*} value 大小配置。
* @param {number} fallback 回退值。
* @return {number} 字节数。
*/
normalizeSize(value, fallback) {
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
return Math.floor(value);
}
if (typeof value === 'string') {
const trimmed = value.trim();
const match = trimmed.match(/^(\d+(?:\.\d+)?)([kKmMgGtT]?)$/);
if (match) {
const number = Number.parseFloat(match[1]);
const unit = match[2].toLowerCase();
const unitMap = {
'': 1,
k: 1024,
m: 1024 * 1024,
g: 1024 * 1024 * 1024,
t: 1024 * 1024 * 1024 * 1024,
};
return Math.floor(number * (unitMap[unit] || 1));
}
}
return fallback;
}
/**
* 规范化正整数。
* @param {*} value 数值或字符串。
* @param {number} fallback 回退值。
* @return {number} 规范化后的正整数。
*/
normalizePositiveInteger(value, fallback) {
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
/**
* 解析时间值并返回毫秒时间戳。
* @param {Date|string|number} [value] 时间值。
* @return {number} 毫秒时间戳。
*/
resolveTimestamp(value) {
if (value == null) {
return 0;
}
if (value instanceof Date) {
return value.getTime();
}
if (typeof value === 'number' && Number.isFinite(value)) {
return value;
}
if (typeof value === 'string') {
const trimmed = value.trim();
if (!trimmed) {
return 0;
}
if (/^\d+$/.test(trimmed)) {
return Number.parseInt(trimmed, 10);
}
const parsed = Date.parse(trimmed);
if (!Number.isNaN(parsed)) {
return parsed;
}
}
return 0;
}
/**
* 把毫秒时间戳转换为 ISO 字符串。
* @param {number} timestamp 毫秒时间戳。
* @return {string|null} ISO 字符串。
*/
formatTimestamp(timestamp) {
if (!Number.isFinite(timestamp) || timestamp <= 0) {
return null;
}
return new Date(timestamp).toISOString();
}
/**
* 获取本次消费限额。
* @param {Object} [options] 消费选项。
* @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。
* @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。
* @return {number} 当前消费限额。
*/
getConsumeLimit(options = {}) {
return this.normalizePositiveInteger(
options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun,
this.maxConsumeRecordsPerWindow
);
}
/**
* 解析消费起始偏移量。
* @param {Object|null} progress 进度数据。
* @param {string} filePath 当前文件路径。
* @param {Object} [options] 消费选项。
* @param {number} [options.startOffset] 显式起始偏移量。
* @return {number} 起始偏移量。
*/
resolveConsumeStartOffset(progress, filePath, options = {}) {
if (options.startOffset != null) {
return this.normalizeInteger(options.startOffset, 0);
}
if (progress && progress.filePath && path.resolve(progress.filePath) === path.resolve(filePath)) {
return this.normalizeInteger(progress.offset, 0);
}
return 0;
}
/**
* 获取某个桶的共享写入窗口计数器文件路径。
* @param {number} bucketIndex 桶索引。
* @return {string} 计数器文件路径。
*/
getWriteWindowCounterPath(bucketIndex = 0) {
return path.join(this.baseDir, `.write-window-counter-${bucketIndex}.json`);
}
/**
* 多桶分片原子限流。
* 将总限额均分到 N 个桶,从首选桶开始尝试,竞争失败或桶满时自动漂移到下一个桶。
* @return {Promise<Object>} { allowed: boolean, counter: { startedAt, recordCount }, source: 'file' }。
*/
async atomicCheckAndIncrementWriteWindow() {
if (this.maxWriteRecordsPerWindow <= 0) {
return { allowed: true, counter: { startedAt: 0, recordCount: 0 } };
}
const bucketCount = this.writeWindowBucketCount;
const perBucketLimit = Math.ceil(this.maxWriteRecordsPerWindow / bucketCount);
const writeWindowMs = this.writeWindowSeconds * 1000;
const now = Date.now();
await this.ensureParentDirectory(this.baseDir);
let lastCounter = null;
for (let i = 0; i < bucketCount; i++) {
const bucketIndex = (this.preferredBucket + i) % bucketCount;
const counterPath = this.getWriteWindowCounterPath(bucketIndex);
const lockPath = `${counterPath}.lock`;
let lockHandle = null;
try {
// eslint-disable-next-line no-bitwise
lockHandle = await fsp.open(lockPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY);
} catch (error) {
if (error.code === 'EEXIST' || error.code === 'EPERM') {
continue;
}
throw error;
}
try {
const counter = { startedAt: 0, recordCount: 0 };
try {
const content = await fsp.readFile(counterPath, 'utf8');
if (content.trim()) {
const data = JSON.parse(content);
counter.startedAt = typeof data.startedAt === 'number' && Number.isFinite(data.startedAt) ? data.startedAt : 0;
counter.recordCount = typeof data.recordCount === 'number' && Number.isFinite(data.recordCount) ? data.recordCount : 0;
}
} catch (error) {
if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) {
throw error;
}
}
if (counter.startedAt > 0 && now - counter.startedAt >= writeWindowMs) {
counter.startedAt = now;
counter.recordCount = 0;
}
if (counter.startedAt === 0) {
counter.startedAt = now;
}
lastCounter = counter;
if (counter.recordCount >= perBucketLimit) {
continue;
}
counter.recordCount += 1;
await fsp.writeFile(counterPath, `${JSON.stringify(counter)}\n`, 'utf8');
return { allowed: true, counter, source: 'file' };
} finally {
await lockHandle.close();
await fsp.rm(lockPath, { force: true });
}
}
return { allowed: false, counter: lastCounter || { startedAt: now, recordCount: 0 }, source: 'file' };
}
/**
* 获取当前写入窗口状态(汇总所有桶)。
* @return {Promise<Object>} 写入窗口状态。
*/
async getWriteWindowState() {
const writeWindowMs = this.writeWindowSeconds * 1000;
const now = Date.now();
let totalRecordCount = 0;
let earliestStartedAt = 0;
for (let i = 0; i < this.writeWindowBucketCount; i++) {
try {
const content = await fsp.readFile(this.getWriteWindowCounterPath(i), 'utf8');
if (content.trim()) {
const data = JSON.parse(content);
const startedAt = typeof data.startedAt === 'number' && Number.isFinite(data.startedAt) ? data.startedAt : 0;
const recordCount = typeof data.recordCount === 'number' && Number.isFinite(data.recordCount) ? data.recordCount : 0;
if (startedAt > 0 && now - startedAt < writeWindowMs) {
totalRecordCount += recordCount;
if (earliestStartedAt === 0 || startedAt < earliestStartedAt) {
earliestStartedAt = startedAt;
}
}
}
} catch { /* ignore missing/corrupt counter files */ }
}
const windowExpired = earliestStartedAt > 0 && now - earliestStartedAt >= writeWindowMs;
const resumeAt = earliestStartedAt > 0
? new Date(earliestStartedAt + writeWindowMs).toISOString()
: null;
return {
writeWindowSeconds: this.writeWindowSeconds,
maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow,
writeWindowBucketCount: this.writeWindowBucketCount,
writeWindowRecordCount: totalRecordCount,
writeWindowStartedAt: earliestStartedAt > 0
? new Date(earliestStartedAt).toISOString()
: null,
windowExpired,
resumeAt: windowExpired ? null : resumeAt,
source: 'file',
};
}
/**
* 将字符串映射到 [0, max) 范围内的索引。
* @param {string} str 输入字符串。
* @param {number} max 范围上限。
* @return {number} 索引。
*/
hashStringToIndex(str, max) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
// eslint-disable-next-line no-bitwise
hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0;
}
return ((hash % max) + max) % max;
}
/**
* 确保目录存在。
* @param {string} dirPath 目录路径。
* @return {Promise<void>} Promise。
*/
async ensureParentDirectory(dirPath) {
await fsp.mkdir(dirPath, { recursive: true });
}
/**
* 判断是否需要执行清理。
* @return {Promise<void>} Promise。
*/
async maybeCleanup() {
if (!this.cleanupIntervalMs || this.cleanupIntervalMs <= 0) {
return;
}
const now = Date.now();
if (now - this.lastCleanupAt < this.cleanupIntervalMs) {
return;
}
this.lastCleanupAt = now;
try {
await this.cleanupExpiredFiles();
} catch (error) {
this.app.logger.warn('[fileWriter.cleanup] 清理本地文件失败', error.message);
}
}
/**
* 递归清理目录中的过期文件。
* @param {string} dirPath 当前目录。
* @param {number} cutoffTime 过期时间戳。
* @param {Object} result 清理统计。
* @return {Promise<void>} Promise。
*/
async walkAndCleanup(dirPath, cutoffTime, result) {
let entries;
try {
entries = await fsp.readdir(dirPath, { withFileTypes: true });
} catch (error) {
if (error.code === 'ENOENT') {
return;
}
throw error;
}
for (const entry of entries) {
const entryPath = path.join(dirPath, entry.name);
if (entry.isDirectory()) {
await this.walkAndCleanup(entryPath, cutoffTime, result);
await this.removeEmptyDirectory(entryPath, result);
continue;
}
if (!entry.isFile()) {
continue;
}
if (!entry.name.endsWith(this.fileExtension)) {
// 【风险点】清理只覆盖数据文件,.progress 文件不会被清理,长期会残留陈旧进度文件。
continue;
}
const completion = await this.loadCompletionMarker(entryPath);
if (!completion || path.resolve(completion.filePath || entryPath) !== path.resolve(entryPath)) {
continue;
}
const stat = await fsp.stat(entryPath).catch(error => {
if (error.code === 'ENOENT') {
return null;
}
throw error;
});
if (!stat || stat.mtimeMs >= cutoffTime) {
continue;
}
await fsp.rm(entryPath, { force: true });
await fsp.rm(this.getDoneFilePath(entryPath), { force: true });
result.removedFiles += 1;
const siblingEntries = await fsp.readdir(dirPath, { withFileTypes: true });
for (const siblingEntry of siblingEntries) {
if (!siblingEntry.isFile() || !siblingEntry.name.endsWith(this.progressExtension)) {
continue;
}
const progressPath = path.join(dirPath, siblingEntry.name);
const progressData = await this.loadProgress(progressPath);
if (!progressData || !progressData.completedAt) {
continue;
}
if (path.resolve(progressData.filePath || '') !== path.resolve(entryPath)) {
continue;
}
await fsp.rm(progressPath, { force: true });
}
}
}
/**
* 原子写入文本文件。
* @param {string} targetPath 目标文件路径。
* @param {string} content 文件内容。
* @return {Promise<void>} Promise。
*/
async writeAtomicTextFile(targetPath, content) {
const directory = path.dirname(targetPath);
const tempPath = `${targetPath}.${this.pid}.${Date.now()}.tmp`;
await this.ensureParentDirectory(directory);
const handle = await fsp.open(tempPath, 'w');
try {
await handle.write(content);
await handle.sync();
} finally {
await handle.close();
}
await fsp.rename(tempPath, targetPath);
}
/**
* 删除空目录。
* @param {string} dirPath 目录路径。
* @param {Object} result 清理统计。
* @return {Promise<void>} Promise。
*/
async removeEmptyDirectory(dirPath, result) {
try {
const entries = await fsp.readdir(dirPath);
if (entries.length > 0) {
return;
}
await fsp.rmdir(dirPath);
result.removedDirectories += 1;
} catch (error) {
if (error.code === 'ENOENT' || error.code === 'ENOTEMPTY') {
return;
}
throw error;
}
}
}
module.exports = LocalFileWriter;