1575 lines
49 KiB
JavaScript
1575 lines
49 KiB
JavaScript
'use strict';
|
||
|
||
const fs = require('node:fs');
|
||
const fsp = require('node:fs/promises');
|
||
const os = require('node:os');
|
||
const path = require('node:path');
|
||
|
||
const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024;
|
||
const DEFAULT_MAX_FILE_LINES = 2000;
|
||
const DEFAULT_MAX_CONSUME_RECORDS_PER_WINDOW = 1000;
|
||
const DEFAULT_WRITE_WINDOW_SECONDS = 1;
|
||
const DEFAULT_MAX_WRITE_RECORDS_PER_WINDOW = 2000;
|
||
const DEFAULT_RETENTION_DAYS = 7;
|
||
const DEFAULT_CLEANUP_INTERVAL_MS = 60 * 60 * 1000;
|
||
const DEFAULT_STATE_MAP_MAX_ENTRIES = 256;
|
||
const DEFAULT_WRITE_WINDOW_BUCKET_COUNT = 8;
|
||
|
||
/**
|
||
* 本地文件队列写入器。
|
||
* 负责把业务数据按 JSON Lines 追加到本地文件,并保证单条写入串行、fsync 落盘和自动轮转。
|
||
*/
|
||
class LocalFileWriter {
|
||
/**
|
||
* 创建本地文件写入器实例。
|
||
* @param {Egg.Application} app Egg 应用实例。
|
||
*/
|
||
constructor(app) {
|
||
this.app = app;
|
||
this.config = app.config.fileWriter || {};
|
||
this.baseDir = path.resolve(this.config.baseDir || path.join(app.baseDir, 'runtime', 'file-writer'));
|
||
this.maxFileSize = this.normalizeSize(this.config.maxFileSize, DEFAULT_MAX_FILE_SIZE);
|
||
this.maxFileLines = this.normalizePositiveInteger(this.config.maxFileLines, DEFAULT_MAX_FILE_LINES);
|
||
this.maxConsumeRecordsPerWindow = this.normalizePositiveInteger(
|
||
this.config.maxConsumeRecordsPerWindow
|
||
?? this.config.maxConsumeRecordsPerRun
|
||
?? app.config.maxConsumeRecordsPerWindow
|
||
?? app.config.maxConsumeRecordsPerRun,
|
||
DEFAULT_MAX_CONSUME_RECORDS_PER_WINDOW
|
||
);
|
||
this.maxConsumeRecordsPerRun = this.maxConsumeRecordsPerWindow;
|
||
this.writeWindowSeconds = this.normalizeInteger(
|
||
this.config.writeWindowSeconds
|
||
?? app.config.writeWindowSeconds,
|
||
DEFAULT_WRITE_WINDOW_SECONDS
|
||
);
|
||
this.maxWriteRecordsPerWindow = this.normalizeInteger(
|
||
this.config.maxWriteRecordsPerWindow
|
||
?? app.config.maxWriteRecordsPerWindow,
|
||
DEFAULT_MAX_WRITE_RECORDS_PER_WINDOW
|
||
);
|
||
this.writeWindowStartedAt = 0;
|
||
this.writeWindowRecordCount = 0;
|
||
this.writeWindowBucketCount = this.normalizePositiveInteger(
|
||
this.config.writeWindowBucketCount
|
||
?? app.config.writeWindowBucketCount,
|
||
DEFAULT_WRITE_WINDOW_BUCKET_COUNT
|
||
);
|
||
this.preferredBucket = this.hashStringToIndex(this.workerId + '-' + this.pid, this.writeWindowBucketCount);
|
||
this.retentionDays = this.normalizeInteger(this.config.retentionDays, DEFAULT_RETENTION_DAYS);
|
||
this.cleanupIntervalMs = this.normalizeInteger(this.config.cleanupIntervalMs, DEFAULT_CLEANUP_INTERVAL_MS);
|
||
this.machineId = this.sanitizeSegment(this.config.machineId || os.hostname() || 'machine');
|
||
this.workerId = this.sanitizeSegment(this.config.workerId || process.env.EGG_WORKER_ID || 'worker');
|
||
this.pid = process.pid;
|
||
this.processTag = `${this.machineId}-${this.workerId}-p${this.pid}`;
|
||
this.sequenceWidth = this.normalizeInteger(this.config.sequenceWidth, 4);
|
||
this.stateMapMaxEntries = this.normalizePositiveInteger(this.config.stateMapMaxEntries, DEFAULT_STATE_MAP_MAX_ENTRIES);
|
||
this.fileExtension = this.normalizeExtension(this.config.fileExtension, '.log');
|
||
this.progressExtension = this.normalizeExtension(this.config.progressExtension, '.progress');
|
||
this.doneExtension = this.normalizeExtension(this.config.doneExtension, '.done');
|
||
this.stateMap = new Map();
|
||
this.queue = Promise.resolve();
|
||
this.lastCleanupAt = 0;
|
||
}
|
||
|
||
/**
|
||
* 将写入任务串行排队执行。
|
||
* @param {Function} task 需要排队执行的异步任务。
|
||
* @return {Promise<*>} 任务执行结果。
|
||
*/
|
||
enqueue(task) {
|
||
const currentTask = this.queue.then(() => task());
|
||
this.queue = currentTask.then(() => undefined, () => undefined);
|
||
return currentTask;
|
||
}
|
||
|
||
/**
|
||
* 等待当前队列里的所有写入完成。
|
||
* @return {Promise<void>} 等待完成的 Promise。
|
||
*/
|
||
async flush() {
|
||
await this.queue;
|
||
}
|
||
|
||
/**
|
||
* 把业务数据写入本地 JSON Lines 文件。
|
||
* @param {string} businessName 业务名称。
|
||
* @param {*} record 需要写入的数据。
|
||
* @param {Object} [options] 写入选项。
|
||
* @param {Date|string|number} [options.date] 指定归档日期,默认使用当前时间。
|
||
* @return {Promise<Object>} 写入结果。
|
||
*/
|
||
async writeRecord(businessName, record, options = {}) {
|
||
return this.enqueue(async () => {
|
||
const windowCheck = await this.atomicCheckAndIncrementWriteWindow();
|
||
this.writeWindowStartedAt = windowCheck.counter.startedAt;
|
||
this.writeWindowRecordCount = windowCheck.counter.recordCount;
|
||
|
||
if (!windowCheck.allowed) {
|
||
const writeWindowMs = this.writeWindowSeconds * 1000;
|
||
const resumeAt = new Date(windowCheck.counter.startedAt + writeWindowMs).toISOString();
|
||
return {
|
||
throttled: true,
|
||
businessName,
|
||
resumeAt,
|
||
writeWindowSeconds: this.writeWindowSeconds,
|
||
writeWindowRecordCount: windowCheck.counter.recordCount,
|
||
maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow,
|
||
};
|
||
}
|
||
|
||
const partitionState = await this.getPartitionState(businessName, options.date);
|
||
const jsonLine = this.stringifyRecord(record);
|
||
const payload = `${jsonLine}\n`;
|
||
const payloadSize = Buffer.byteLength(payload);
|
||
|
||
if (
|
||
partitionState.currentLineCount >= this.maxFileLines
|
||
|| (partitionState.currentSize > 0 && partitionState.currentSize + payloadSize > this.maxFileSize)
|
||
) {
|
||
partitionState.sequence += 1;
|
||
partitionState.currentFilePath = this.buildDataFilePath(partitionState, partitionState.sequence);
|
||
partitionState.currentSize = 0;
|
||
partitionState.currentLineCount = 0;
|
||
}
|
||
|
||
await this.ensureParentDirectory(partitionState.partitionDir);
|
||
|
||
const handle = await fsp.open(partitionState.currentFilePath, 'a');
|
||
try {
|
||
await handle.write(payload);
|
||
await handle.sync();
|
||
} finally {
|
||
await handle.close();
|
||
}
|
||
|
||
partitionState.currentSize += payloadSize;
|
||
partitionState.currentLineCount += 1;
|
||
await this.maybeCleanup();
|
||
|
||
return {
|
||
businessName: partitionState.businessName,
|
||
dateKey: partitionState.dateKey,
|
||
filePath: partitionState.currentFilePath,
|
||
size: partitionState.currentSize,
|
||
lineCount: partitionState.currentLineCount,
|
||
bytesWritten: payloadSize,
|
||
sequence: partitionState.sequence,
|
||
machineId: this.machineId,
|
||
workerId: this.workerId,
|
||
pid: this.pid,
|
||
throttled: false,
|
||
writeWindowRecordCount: windowCheck.counter.recordCount,
|
||
maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow,
|
||
};
|
||
});
|
||
}
|
||
|
||
/**
|
||
* 写入方法别名,便于业务侧直接调用。
|
||
* @param {string} businessName 业务名称。
|
||
* @param {*} record 需要写入的数据。
|
||
* @param {Object} [options] 写入选项。
|
||
* @return {Promise<Object>} 写入结果。
|
||
*/
|
||
async write(businessName, record, options = {}) {
|
||
return this.writeRecord(businessName, record, options);
|
||
}
|
||
|
||
/**
|
||
* 获取某个业务在指定日期下的目录信息。
|
||
* @param {string} businessName 业务名称。
|
||
* @param {Date|string|number} [date] 指定日期。
|
||
* @return {Object} 目录信息。
|
||
*/
|
||
getPartitionInfo(businessName, date = new Date()) {
|
||
const businessSegment = this.sanitizeSegment(businessName || 'default');
|
||
const dateParts = this.getDateParts(date);
|
||
const partitionDir = path.join(this.baseDir, businessSegment, dateParts.year, dateParts.month, dateParts.day);
|
||
|
||
return {
|
||
businessName,
|
||
businessSegment,
|
||
dateKey: dateParts.dateKey,
|
||
partitionDir,
|
||
filePrefix: `${businessSegment}-${this.processTag}`,
|
||
processTag: this.processTag,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 获取消费进度文件路径。
|
||
* @param {string} businessName 业务名称。
|
||
* @param {Date|string|number|null} [date] 指定日期;传空时使用业务级共享进度文件。
|
||
* @param {string} [consumerName] 消费者名称。
|
||
* @return {string} 进度文件路径。
|
||
*/
|
||
getProgressFilePath(businessName, date = new Date(), consumerName = 'default') {
|
||
const businessSegment = this.sanitizeSegment(businessName || 'default');
|
||
const safeConsumerName = this.sanitizeSegment(consumerName || 'default');
|
||
if (date == null) {
|
||
return path.join(this.baseDir, businessSegment, `.${safeConsumerName}${this.progressExtension}`);
|
||
}
|
||
|
||
const partitionInfo = this.getPartitionInfo(businessName, date);
|
||
return path.join(partitionInfo.partitionDir, `.${safeConsumerName}${this.progressExtension}`);
|
||
}
|
||
|
||
/**
|
||
* 获取某个数据文件的消费完成标记路径。
|
||
* @param {string} filePath 数据文件路径。
|
||
* @return {string} 完成标记路径。
|
||
*/
|
||
getDoneFilePath(filePath) {
|
||
const resolvedFilePath = path.resolve(filePath);
|
||
return `${resolvedFilePath}${this.doneExtension}`;
|
||
}
|
||
|
||
/**
|
||
* 列出某个业务日期分区下的所有日志文件。
|
||
* @param {string} businessName 业务名称。
|
||
* @param {Date|string|number|null} [date] 指定日期;传空时递归列出业务目录下所有日志文件。
|
||
* @return {Promise<Array<string>>} 排序后的文件路径列表。
|
||
*/
|
||
async listDataFiles(businessName, date = new Date()) {
|
||
const businessSegment = this.sanitizeSegment(businessName || 'default');
|
||
const businessDir = path.join(this.baseDir, businessSegment);
|
||
const targetDir = date == null ? businessDir : this.getPartitionInfo(businessName, date).partitionDir;
|
||
const files = await this.collectDataFiles(targetDir);
|
||
|
||
return files
|
||
.sort((left, right) => {
|
||
return this.compareDataFileSortKey(left.sortKey, right.sortKey);
|
||
})
|
||
.map(entry => entry.filePath);
|
||
}
|
||
|
||
/**
|
||
* 读取单个日志文件中的 JSON Lines 数据。
|
||
* @param {string} filePath 日志文件路径。
|
||
* @param {Object} [options] 读取选项。
|
||
* @param {number} [options.startOffset=0] 起始字节偏移量。
|
||
* @param {number} [options.maxRecords=0] 最大读取条数,0 或不传不限制。
|
||
* @param {boolean} [options.parseJson=true] 是否自动解析 JSON。
|
||
* @return {Promise<Object>} 读取结果。
|
||
*/
|
||
async readRecords(filePath, options = {}) {
|
||
const resolvedFilePath = path.resolve(filePath);
|
||
const startOffset = this.normalizeInteger(options.startOffset, 0);
|
||
const maxRecords = this.normalizeInteger(options.maxRecords, 0);
|
||
const parseJson = options.parseJson !== false;
|
||
const records = [];
|
||
let nextOffset = startOffset;
|
||
let lineNumber = 0;
|
||
|
||
try {
|
||
for await (const lineEntry of this.iterateJsonLines(resolvedFilePath, startOffset)) {
|
||
lineNumber = lineEntry.lineNumber;
|
||
nextOffset = lineEntry.nextOffset;
|
||
|
||
if (!lineEntry.line.trim()) {
|
||
continue;
|
||
}
|
||
|
||
if (!parseJson) {
|
||
records.push(lineEntry.line);
|
||
|
||
if (maxRecords > 0 && records.length >= maxRecords) {
|
||
break;
|
||
}
|
||
|
||
continue;
|
||
}
|
||
|
||
try {
|
||
records.push(JSON.parse(lineEntry.line));
|
||
|
||
if (maxRecords > 0 && records.length >= maxRecords) {
|
||
break;
|
||
}
|
||
} catch (parseError) {
|
||
parseError.message = `解析文件 ${resolvedFilePath} 第 ${lineNumber} 行失败: ${parseError.message}`;
|
||
throw parseError;
|
||
}
|
||
}
|
||
} catch (error) {
|
||
if (error.code === 'ENOENT') {
|
||
return {
|
||
filePath: resolvedFilePath,
|
||
records: [],
|
||
nextOffset: startOffset,
|
||
lineNumber: 0,
|
||
};
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
|
||
return {
|
||
filePath: resolvedFilePath,
|
||
records,
|
||
nextOffset,
|
||
lineNumber,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 消费某个文件,并在每条记录处理完成后更新进度文件。
|
||
* @param {string} filePath 日志文件路径。
|
||
* @param {Object} options 消费选项。
|
||
* @param {string} options.progressFilePath 进度文件路径。
|
||
* @param {Function} options.onRecord 每条记录的处理函数。
|
||
* @param {number} [options.startOffset=0] 起始偏移量。
|
||
* @param {boolean} [options.parseJson=true] 是否自动解析 JSON。
|
||
* @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。
|
||
* @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。
|
||
* @param {Object|null} [options.progress] 已加载的进度数据,供内部复用。
|
||
* @return {Promise<Object>} 消费结果,包含 filePath、processedCount、nextOffset、limitReached、maxConsumeRecordsPerWindow。
|
||
*/
|
||
async consumeFile(filePath, options = {}) {
|
||
const progressFilePath = options.progressFilePath;
|
||
const onRecord = options.onRecord;
|
||
|
||
if (!progressFilePath) {
|
||
throw new Error('progressFilePath 不能为空');
|
||
}
|
||
|
||
if (typeof onRecord !== 'function') {
|
||
throw new Error('onRecord 必须是函数');
|
||
}
|
||
|
||
const progress = options.progress || await this.loadProgress(progressFilePath);
|
||
const consumeLimit = this.normalizePositiveInteger(
|
||
options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun,
|
||
this.maxConsumeRecordsPerWindow
|
||
);
|
||
|
||
const resolvedFilePath = path.resolve(filePath);
|
||
const startOffset = this.resolveConsumeStartOffset(progress, resolvedFilePath, options);
|
||
const parseJson = options.parseJson !== false;
|
||
|
||
let offset = startOffset;
|
||
let processedCount = 0;
|
||
let lineNumber = 0;
|
||
let limitReached = false;
|
||
|
||
try {
|
||
for await (const lineEntry of this.iterateJsonLines(resolvedFilePath, startOffset)) {
|
||
if (consumeLimit > 0 && processedCount >= consumeLimit) {
|
||
limitReached = true;
|
||
break;
|
||
}
|
||
|
||
lineNumber = lineEntry.lineNumber;
|
||
offset = lineEntry.nextOffset;
|
||
|
||
if (!lineEntry.line.trim()) {
|
||
continue;
|
||
}
|
||
|
||
let record = lineEntry.line;
|
||
if (parseJson) {
|
||
try {
|
||
record = JSON.parse(lineEntry.line);
|
||
} catch (parseError) {
|
||
parseError.message = `解析文件 ${resolvedFilePath} 第 ${lineNumber} 行失败: ${parseError.message}`;
|
||
throw parseError;
|
||
}
|
||
}
|
||
|
||
await onRecord(record, {
|
||
filePath: resolvedFilePath,
|
||
offset,
|
||
index: processedCount,
|
||
});
|
||
processedCount += 1;
|
||
|
||
const progressData = {
|
||
filePath: resolvedFilePath,
|
||
offset,
|
||
processedCount,
|
||
updatedAt: new Date().toISOString(),
|
||
};
|
||
await this.saveProgress(progressFilePath, progressData);
|
||
|
||
if (consumeLimit > 0 && processedCount >= consumeLimit) {
|
||
limitReached = true;
|
||
break;
|
||
}
|
||
}
|
||
} catch (error) {
|
||
if (error.code === 'ENOENT') {
|
||
return {
|
||
filePath: resolvedFilePath,
|
||
processedCount: 0,
|
||
nextOffset: startOffset,
|
||
limitReached: false,
|
||
maxConsumeRecordsPerRun: consumeLimit,
|
||
maxConsumeRecordsPerWindow: consumeLimit,
|
||
};
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
|
||
if (limitReached) {
|
||
return {
|
||
filePath: resolvedFilePath,
|
||
processedCount,
|
||
nextOffset: offset,
|
||
limitReached: true,
|
||
maxConsumeRecordsPerRun: consumeLimit,
|
||
maxConsumeRecordsPerWindow: consumeLimit,
|
||
};
|
||
}
|
||
|
||
const completionData = {
|
||
filePath: resolvedFilePath,
|
||
offset,
|
||
processedCount,
|
||
completedAt: new Date().toISOString(),
|
||
updatedAt: new Date().toISOString(),
|
||
};
|
||
|
||
await this.saveProgress(progressFilePath, completionData);
|
||
await this.saveCompletionMarker(resolvedFilePath, completionData);
|
||
|
||
return {
|
||
filePath: resolvedFilePath,
|
||
processedCount,
|
||
nextOffset: offset,
|
||
limitReached: false,
|
||
maxConsumeRecordsPerRun: consumeLimit,
|
||
maxConsumeRecordsPerWindow: consumeLimit,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 按业务目录消费所有日志文件,支持断点续传。
|
||
* @param {Object} options 消费选项。
|
||
* @param {string} options.businessName 业务名称。
|
||
* @param {string} options.progressFilePath 进度文件路径。
|
||
* @param {Function} options.onRecord 每条记录的处理函数。
|
||
* @param {boolean} [options.parseJson=true] 是否自动解析 JSON。
|
||
* @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。
|
||
* @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。
|
||
* @return {Promise<Object>} 消费结果,包含 businessName、dataFiles、processedCount、progress、limitReached、maxConsumeRecordsPerWindow。
|
||
*/
|
||
async consumeFiles(options = {}) {
|
||
const businessName = options.businessName;
|
||
const progressFilePath = options.progressFilePath;
|
||
const onRecord = options.onRecord;
|
||
|
||
if (!businessName) {
|
||
throw new Error('businessName 不能为空');
|
||
}
|
||
|
||
if (!progressFilePath) {
|
||
throw new Error('progressFilePath 不能为空');
|
||
}
|
||
|
||
if (typeof onRecord !== 'function') {
|
||
throw new Error('onRecord 必须是函数');
|
||
}
|
||
|
||
const progress = await this.loadProgress(progressFilePath);
|
||
const consumeLimit = this.normalizePositiveInteger(
|
||
options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun,
|
||
this.maxConsumeRecordsPerWindow
|
||
);
|
||
|
||
const dataFiles = await this.listDataFiles(businessName, null);
|
||
|
||
if (dataFiles.length === 0) {
|
||
return {
|
||
businessName,
|
||
dataFiles,
|
||
processedCount: 0,
|
||
progress,
|
||
limitReached: false,
|
||
maxConsumeRecordsPerRun: consumeLimit,
|
||
maxConsumeRecordsPerWindow: consumeLimit,
|
||
};
|
||
}
|
||
|
||
let startIndex = 0;
|
||
let startOffset = 0;
|
||
let limitReached = false;
|
||
|
||
if (progress && progress.filePath) {
|
||
const matchedIndex = dataFiles.findIndex(filePath => path.resolve(filePath) === path.resolve(progress.filePath));
|
||
if (matchedIndex >= 0) {
|
||
startIndex = matchedIndex;
|
||
startOffset = this.normalizeInteger(progress.offset, 0);
|
||
} else {
|
||
const progressSortKey = this.getDataFileSortKey(progress.filePath);
|
||
const fallbackIndex = dataFiles.findIndex(filePath => this.compareDataFileSortKey(
|
||
this.getDataFileSortKey(filePath),
|
||
progressSortKey
|
||
) > 0);
|
||
if (fallbackIndex >= 0) {
|
||
startIndex = fallbackIndex;
|
||
startOffset = 0;
|
||
} else {
|
||
startIndex = dataFiles.length;
|
||
}
|
||
}
|
||
}
|
||
|
||
let processedCount = 0;
|
||
let currentProgress = progress;
|
||
let totalConsumedCount = 0;
|
||
|
||
for (let index = startIndex; index < dataFiles.length; index += 1) {
|
||
if (consumeLimit > 0 && totalConsumedCount >= consumeLimit) {
|
||
limitReached = true;
|
||
break;
|
||
}
|
||
|
||
const filePath = dataFiles[index];
|
||
const completion = await this.loadCompletionMarker(filePath);
|
||
if (completion && path.resolve(completion.filePath || filePath) === path.resolve(filePath)) {
|
||
currentProgress = {
|
||
...completion,
|
||
filePath: completion.filePath || filePath,
|
||
updatedAt: new Date().toISOString(),
|
||
};
|
||
await this.saveProgress(progressFilePath, currentProgress);
|
||
continue;
|
||
}
|
||
|
||
const remainingLimit = consumeLimit > 0 ? consumeLimit - totalConsumedCount : consumeLimit;
|
||
const fileResult = await this.consumeFile(filePath, {
|
||
progressFilePath,
|
||
onRecord,
|
||
startOffset: index === startIndex ? startOffset : 0,
|
||
parseJson: options.parseJson,
|
||
maxConsumeRecordsPerWindow: remainingLimit,
|
||
progress,
|
||
});
|
||
processedCount += fileResult.processedCount;
|
||
totalConsumedCount += fileResult.processedCount;
|
||
currentProgress = {
|
||
filePath,
|
||
offset: fileResult.nextOffset,
|
||
processedCount,
|
||
updatedAt: new Date().toISOString(),
|
||
};
|
||
await this.saveProgress(progressFilePath, currentProgress);
|
||
|
||
if (fileResult.limitReached) {
|
||
limitReached = true;
|
||
break;
|
||
}
|
||
}
|
||
|
||
return {
|
||
businessName,
|
||
dataFiles,
|
||
processedCount,
|
||
progress: currentProgress,
|
||
limitReached,
|
||
maxConsumeRecordsPerRun: consumeLimit,
|
||
maxConsumeRecordsPerWindow: consumeLimit,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 递归收集目录下的所有数据文件,并记录用于排序的分区日期与序号。
|
||
* @param {string} dirPath 目录路径。
|
||
* @param {Array<Object>} [result=[]] 累积结果。
|
||
* @return {Promise<Array<Object>>} 收集结果。
|
||
*/
|
||
async collectDataFiles(dirPath, result = []) {
|
||
let entries;
|
||
try {
|
||
entries = await fsp.readdir(dirPath, { withFileTypes: true });
|
||
} catch (error) {
|
||
if (error.code === 'ENOENT') {
|
||
return result;
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
|
||
for (const entry of entries) {
|
||
const entryPath = path.join(dirPath, entry.name);
|
||
if (entry.isDirectory()) {
|
||
await this.collectDataFiles(entryPath, result);
|
||
continue;
|
||
}
|
||
|
||
if (!entry.isFile() || !entry.name.endsWith(this.fileExtension)) {
|
||
continue;
|
||
}
|
||
|
||
result.push({
|
||
filePath: entryPath,
|
||
sortKey: this.getDataFileSortKey(entryPath),
|
||
});
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* 解析数据文件的消费顺序键,按分区日期和文件序号排序。
|
||
* @param {string} filePath 数据文件路径。
|
||
* @return {Object} 排序键。
|
||
*/
|
||
getDataFileSortKey(filePath) {
|
||
const resolvedFilePath = path.resolve(filePath);
|
||
const relativeParts = path.relative(this.baseDir, resolvedFilePath).split(path.sep).filter(Boolean);
|
||
const fileName = path.basename(resolvedFilePath);
|
||
const sequenceMatch = fileName.match(/-(\d+)\.[^.]+$/);
|
||
|
||
let partitionDateKey = '';
|
||
if (relativeParts.length >= 5) {
|
||
const year = relativeParts[1];
|
||
const month = relativeParts[2];
|
||
const day = relativeParts[3];
|
||
|
||
if (/^\d+$/.test(year) && /^\d+$/.test(month) && /^\d+$/.test(day)) {
|
||
partitionDateKey = `${year.padStart(4, '0')}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`;
|
||
}
|
||
}
|
||
|
||
return {
|
||
partitionDateKey,
|
||
sequence: sequenceMatch ? Number.parseInt(sequenceMatch[1], 10) : Number.MAX_SAFE_INTEGER,
|
||
filePath: resolvedFilePath,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 比较两个数据文件的排序键。
|
||
* @param {Object} left 左侧排序键。
|
||
* @param {Object} right 右侧排序键。
|
||
* @return {number} 排序结果。
|
||
*/
|
||
compareDataFileSortKey(left, right) {
|
||
if (left.partitionDateKey !== right.partitionDateKey) {
|
||
return left.partitionDateKey.localeCompare(right.partitionDateKey, 'en');
|
||
}
|
||
|
||
if (left.sequence !== right.sequence) {
|
||
return left.sequence - right.sequence;
|
||
}
|
||
|
||
return left.filePath.localeCompare(right.filePath, 'en', { numeric: true });
|
||
}
|
||
|
||
/**
|
||
* 读取进度文件。
|
||
* @param {string} progressFilePath 进度文件路径。
|
||
* @return {Promise<Object|null>} 进度内容。
|
||
*/
|
||
async loadProgress(progressFilePath) {
|
||
const resolvedPath = path.resolve(progressFilePath);
|
||
try {
|
||
const content = await fsp.readFile(resolvedPath, 'utf8');
|
||
if (!content.trim()) {
|
||
return null;
|
||
}
|
||
|
||
return this.normalizeProgressData(JSON.parse(content), resolvedPath);
|
||
} catch (error) {
|
||
if (error.code === 'ENOENT') {
|
||
return null;
|
||
}
|
||
|
||
if (error instanceof SyntaxError) {
|
||
this.app.logger.warn('[fileWriter.loadProgress] 忽略损坏的进度文件', resolvedPath, error.message);
|
||
return null;
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 保存进度文件。
|
||
* @param {string} progressFilePath 进度文件路径。
|
||
* @param {Object} progressData 进度数据。
|
||
* @return {Promise<Object>} 保存后的进度数据。
|
||
*/
|
||
async saveProgress(progressFilePath, progressData) {
|
||
const resolvedPath = path.resolve(progressFilePath);
|
||
const payload = `${JSON.stringify({
|
||
...progressData,
|
||
updatedAt: progressData.updatedAt || new Date().toISOString(),
|
||
}, null, 2)}\n`;
|
||
|
||
await this.writeAtomicTextFile(resolvedPath, payload);
|
||
|
||
return {
|
||
...progressData,
|
||
updatedAt: progressData.updatedAt || new Date().toISOString(),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 保存消费完成标记。
|
||
* @param {string} filePath 数据文件路径。
|
||
* @param {Object} completionData 完成数据。
|
||
* @return {Promise<Object>} 保存后的完成数据。
|
||
*/
|
||
async saveCompletionMarker(filePath, completionData) {
|
||
const resolvedFilePath = path.resolve(filePath);
|
||
const doneFilePath = this.getDoneFilePath(resolvedFilePath);
|
||
const payload = `${JSON.stringify({
|
||
...completionData,
|
||
filePath: resolvedFilePath,
|
||
completedAt: completionData.completedAt || new Date().toISOString(),
|
||
}, null, 2)}\n`;
|
||
|
||
await this.writeAtomicTextFile(doneFilePath, payload);
|
||
|
||
return {
|
||
...completionData,
|
||
filePath: resolvedFilePath,
|
||
completedAt: completionData.completedAt || new Date().toISOString(),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 读取消费完成标记。
|
||
* @param {string} filePath 数据文件路径。
|
||
* @return {Promise<Object|null>} 完成标记内容。
|
||
*/
|
||
async loadCompletionMarker(filePath) {
|
||
const resolvedFilePath = path.resolve(filePath);
|
||
const doneFilePath = this.getDoneFilePath(resolvedFilePath);
|
||
try {
|
||
const content = await fsp.readFile(doneFilePath, 'utf8');
|
||
if (!content.trim()) {
|
||
return null;
|
||
}
|
||
|
||
return this.normalizeCompletionData(JSON.parse(content), resolvedFilePath);
|
||
} catch (error) {
|
||
if (error.code === 'ENOENT') {
|
||
return null;
|
||
}
|
||
|
||
if (error instanceof SyntaxError) {
|
||
this.app.logger.warn('[fileWriter.loadCompletionMarker] 忽略损坏的完成标记', doneFilePath, error.message);
|
||
return null;
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 清理超过保留期的日志文件与进度文件。
|
||
* @return {Promise<Object>} 清理结果。
|
||
*/
|
||
async cleanupExpiredFiles() {
|
||
if (!this.retentionDays || this.retentionDays <= 0) {
|
||
return {
|
||
removedFiles: 0,
|
||
removedDirectories: 0,
|
||
};
|
||
}
|
||
|
||
const cutoffTime = Date.now() - (this.retentionDays * 24 * 60 * 60 * 1000);
|
||
const result = {
|
||
removedFiles: 0,
|
||
removedDirectories: 0,
|
||
};
|
||
|
||
await this.walkAndCleanup(this.baseDir, cutoffTime, result);
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* 规范化日期并返回目录分段。
|
||
* @param {Date|string|number} date 日期。
|
||
* @return {Object} 日期分段。
|
||
*/
|
||
getDateParts(date = new Date()) {
|
||
const targetDate = date instanceof Date ? date : new Date(date);
|
||
if (Number.isNaN(targetDate.getTime())) {
|
||
throw new Error(`无效的日期参数: ${date}`);
|
||
}
|
||
|
||
const year = String(targetDate.getFullYear());
|
||
const month = String(targetDate.getMonth() + 1).padStart(2, '0');
|
||
const day = String(targetDate.getDate()).padStart(2, '0');
|
||
|
||
return {
|
||
year,
|
||
month,
|
||
day,
|
||
dateKey: `${year}-${month}-${day}`,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 获取写入分区状态。
|
||
* @param {string} businessName 业务名称。
|
||
* @param {Date|string|number} [date] 指定日期。
|
||
* @return {Promise<Object>} 分区状态。
|
||
*/
|
||
async getPartitionState(businessName, date = new Date()) {
|
||
const partitionInfo = this.getPartitionInfo(businessName, date);
|
||
const stateKey = `${partitionInfo.businessSegment}:${partitionInfo.dateKey}`;
|
||
|
||
if (this.stateMap.has(stateKey)) {
|
||
const existingState = this.stateMap.get(stateKey);
|
||
this.stateMap.delete(stateKey);
|
||
this.stateMap.set(stateKey, existingState);
|
||
return existingState;
|
||
}
|
||
|
||
await this.ensureParentDirectory(partitionInfo.partitionDir);
|
||
|
||
const state = {
|
||
businessName,
|
||
businessSegment: partitionInfo.businessSegment,
|
||
dateKey: partitionInfo.dateKey,
|
||
partitionDir: partitionInfo.partitionDir,
|
||
filePrefix: partitionInfo.filePrefix,
|
||
sequence: 1,
|
||
currentSize: 0,
|
||
currentLineCount: 0,
|
||
currentFilePath: this.buildDataFilePath({
|
||
partitionDir: partitionInfo.partitionDir,
|
||
filePrefix: partitionInfo.filePrefix,
|
||
}, 1),
|
||
};
|
||
|
||
const latestDataFileState = await this.loadLatestDataFileState(partitionInfo.partitionDir, partitionInfo.filePrefix);
|
||
if (latestDataFileState) {
|
||
state.sequence = latestDataFileState.sequence;
|
||
state.currentSize = latestDataFileState.currentSize;
|
||
state.currentLineCount = latestDataFileState.currentLineCount;
|
||
state.currentFilePath = latestDataFileState.currentFilePath;
|
||
}
|
||
|
||
this.evictOldestStateIfNeeded();
|
||
this.stateMap.set(stateKey, state);
|
||
return state;
|
||
}
|
||
|
||
/**
|
||
* 读取分区中最新数据文件的大小与行数,用于恢复写入状态。
|
||
* @param {string} partitionDir 分区目录。
|
||
* @param {string} filePrefix 文件前缀。
|
||
* @return {Promise<Object|null>} 最新文件状态。
|
||
*/
|
||
async loadLatestDataFileState(partitionDir, filePrefix) {
|
||
const dataFiles = await this.collectDataFiles(partitionDir);
|
||
if (dataFiles.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
dataFiles.sort((left, right) => this.compareDataFileSortKey(left.sortKey, right.sortKey));
|
||
const latestFile = dataFiles[dataFiles.length - 1];
|
||
const stat = await fsp.stat(latestFile.filePath);
|
||
const currentLineCount = await this.countFileLines(latestFile.filePath);
|
||
|
||
return {
|
||
filePrefix,
|
||
sequence: latestFile.sortKey.sequence > 0 ? latestFile.sortKey.sequence : 1,
|
||
currentFilePath: latestFile.filePath,
|
||
currentSize: stat.size,
|
||
currentLineCount,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 统计文件中的行数。
|
||
* @param {string} filePath 文件路径。
|
||
* @return {Promise<number>} 行数。
|
||
*/
|
||
async countFileLines(filePath) {
|
||
let lineCount = 0;
|
||
let lastByte = null;
|
||
const stream = fs.createReadStream(filePath);
|
||
|
||
try {
|
||
for await (const chunk of stream) {
|
||
lastByte = chunk.length > 0 ? chunk[chunk.length - 1] : lastByte;
|
||
for (let i = 0; i < chunk.length; i += 1) {
|
||
if (chunk[i] === 0x0A) {
|
||
lineCount += 1;
|
||
}
|
||
}
|
||
}
|
||
} finally {
|
||
stream.destroy();
|
||
}
|
||
|
||
if (lineCount === 0) {
|
||
return 0;
|
||
}
|
||
|
||
if (lastByte !== 0x0A) {
|
||
lineCount += 1;
|
||
}
|
||
|
||
return lineCount;
|
||
}
|
||
|
||
/**
|
||
* 逐行读取 JSON Lines 文件,并返回每行的精确字节偏移。
|
||
* @param {string} filePath 文件路径。
|
||
* @param {number} [startOffset=0] 起始字节偏移量。
|
||
* @return {AsyncGenerator<Object>} 逐行迭代器。
|
||
*/
|
||
async* iterateJsonLines(filePath, startOffset = 0) {
|
||
const stream = fs.createReadStream(filePath, {
|
||
start: startOffset,
|
||
});
|
||
|
||
let buffer = Buffer.alloc(0);
|
||
let offset = startOffset;
|
||
let lineNumber = 0;
|
||
|
||
try {
|
||
for await (const chunk of stream) {
|
||
buffer = Buffer.concat([ buffer, chunk ]);
|
||
|
||
while (buffer.length > 0) {
|
||
const lineFeedIndex = buffer.indexOf(0x0A);
|
||
if (lineFeedIndex < 0) {
|
||
break;
|
||
}
|
||
|
||
const lineEndIndex = lineFeedIndex > 0 && buffer[lineFeedIndex - 1] === 0x0D
|
||
? lineFeedIndex - 1
|
||
: lineFeedIndex;
|
||
const consumedBytes = lineFeedIndex + 1;
|
||
const lineBuffer = buffer.slice(0, lineEndIndex);
|
||
|
||
lineNumber += 1;
|
||
offset += consumedBytes;
|
||
buffer = buffer.slice(consumedBytes);
|
||
|
||
yield {
|
||
lineNumber,
|
||
line: lineBuffer.toString('utf8'),
|
||
nextOffset: offset,
|
||
};
|
||
}
|
||
}
|
||
|
||
if (buffer.length > 0) {
|
||
lineNumber += 1;
|
||
offset += buffer.length;
|
||
|
||
yield {
|
||
lineNumber,
|
||
line: buffer.toString('utf8'),
|
||
nextOffset: offset,
|
||
};
|
||
}
|
||
} finally {
|
||
stream.destroy();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 归一化进度文件内容,兼容损坏或旧字段。
|
||
* @param {*} progressData 原始进度内容。
|
||
* @param {string} [fallbackFilePath] 兜底文件路径。
|
||
* @return {Object|null} 归一化后的进度内容。
|
||
*/
|
||
normalizeProgressData(progressData, fallbackFilePath = null) {
|
||
if (!progressData || typeof progressData !== 'object') {
|
||
return null;
|
||
}
|
||
|
||
const filePath = typeof progressData.filePath === 'string' && progressData.filePath.trim()
|
||
? path.resolve(progressData.filePath)
|
||
: (fallbackFilePath || null);
|
||
const offset = this.normalizeInteger(progressData.offset ?? progressData.nextOffset, 0);
|
||
const processedCount = this.normalizeInteger(progressData.processedCount ?? progressData.processed ?? progressData.count, 0);
|
||
|
||
return {
|
||
...progressData,
|
||
filePath,
|
||
offset,
|
||
processedCount,
|
||
updatedAt: typeof progressData.updatedAt === 'string' && progressData.updatedAt.trim()
|
||
? progressData.updatedAt
|
||
: undefined,
|
||
completedAt: typeof progressData.completedAt === 'string' && progressData.completedAt.trim()
|
||
? progressData.completedAt
|
||
: undefined,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 归一化完成标记内容,兼容损坏或旧字段。
|
||
* @param {*} completionData 原始完成标记内容。
|
||
* @param {string} fallbackFilePath 兜底文件路径。
|
||
* @return {Object|null} 归一化后的完成标记内容。
|
||
*/
|
||
normalizeCompletionData(completionData, fallbackFilePath) {
|
||
if (!completionData || typeof completionData !== 'object') {
|
||
return null;
|
||
}
|
||
|
||
const filePath = typeof completionData.filePath === 'string' && completionData.filePath.trim()
|
||
? path.resolve(completionData.filePath)
|
||
: fallbackFilePath;
|
||
if (!filePath) {
|
||
return null;
|
||
}
|
||
|
||
const completedAtMs = this.resolveTimestamp(completionData.completedAt ?? completionData.updatedAt);
|
||
|
||
return {
|
||
...completionData,
|
||
filePath,
|
||
completedAt: this.formatTimestamp(completedAtMs),
|
||
updatedAt: typeof completionData.updatedAt === 'string' && completionData.updatedAt.trim()
|
||
? completionData.updatedAt
|
||
: undefined,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 回收最旧的分区状态,避免 stateMap 无界增长。
|
||
* @return {void}
|
||
*/
|
||
evictOldestStateIfNeeded() {
|
||
if (this.stateMap.size < this.stateMapMaxEntries) {
|
||
return;
|
||
}
|
||
|
||
const oldestKey = this.stateMap.keys().next().value;
|
||
if (oldestKey !== undefined) {
|
||
this.stateMap.delete(oldestKey);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 构建日志文件路径。
|
||
* @param {Object} partitionState 分区状态。
|
||
* @param {number} sequence 序号。
|
||
* @return {string} 文件路径。
|
||
*/
|
||
buildDataFilePath(partitionState, sequence) {
|
||
const sequenceText = String(sequence).padStart(this.sequenceWidth, '0');
|
||
const fileName = `${partitionState.filePrefix}-${sequenceText}${this.fileExtension}`;
|
||
return path.join(partitionState.partitionDir, fileName);
|
||
}
|
||
|
||
/**
|
||
* 把要写入的记录转换为 JSON 字符串。
|
||
* @param {*} record 记录内容。
|
||
* @return {string} JSON 字符串。
|
||
*/
|
||
stringifyRecord(record) {
|
||
const json = JSON.stringify(record, this.jsonReplacer);
|
||
if (json === undefined) {
|
||
throw new Error('写入内容无法被序列化为 JSON');
|
||
}
|
||
|
||
return json;
|
||
}
|
||
|
||
/**
|
||
* JSON 序列化时处理 BigInt。
|
||
* @param {string} key 属性名。
|
||
* @param {*} value 属性值。
|
||
* @return {*} 序列化后的值。
|
||
*/
|
||
jsonReplacer(key, value) {
|
||
if (typeof value === 'bigint') {
|
||
return value.toString();
|
||
}
|
||
|
||
return value;
|
||
}
|
||
|
||
/**
|
||
* 规范化目录片段,避免路径穿越。
|
||
* @param {*} value 待处理值。
|
||
* @return {string} 安全片段。
|
||
*/
|
||
sanitizeSegment(value) {
|
||
const normalized = String(value || '').trim();
|
||
if (!normalized) {
|
||
return 'default';
|
||
}
|
||
|
||
const safeValue = Array.from(normalized).map(character => {
|
||
const code = character.charCodeAt(0);
|
||
if (code < 32 || '<>:"/\\|?*'.includes(character)) {
|
||
return '_';
|
||
}
|
||
|
||
return character;
|
||
}).join('');
|
||
|
||
return safeValue
|
||
.replace(/\s+/g, '_')
|
||
.replace(/\.+/g, '.')
|
||
.replace(/^\.+|\.+$/g, '')
|
||
.replace(/_+/g, '_')
|
||
.slice(0, 80) || 'default';
|
||
}
|
||
|
||
/**
|
||
* 规范化扩展名。
|
||
* @param {*} value 扩展名。
|
||
* @param {string} fallback 回退扩展名。
|
||
* @return {string} 规范化后的扩展名。
|
||
*/
|
||
normalizeExtension(value, fallback) {
|
||
const raw = String(value || fallback || '').trim();
|
||
if (!raw) {
|
||
return fallback;
|
||
}
|
||
|
||
return raw.startsWith('.') ? raw : `.${raw}`;
|
||
}
|
||
|
||
/**
|
||
* 规范化数字。
|
||
* @param {*} value 数值或字符串。
|
||
* @param {number} fallback 回退值。
|
||
* @return {number} 规范化后的整数。
|
||
*/
|
||
normalizeInteger(value, fallback) {
|
||
const parsed = Number.parseInt(value, 10);
|
||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
||
}
|
||
|
||
/**
|
||
* 规范化大小配置。
|
||
* @param {*} value 大小配置。
|
||
* @param {number} fallback 回退值。
|
||
* @return {number} 字节数。
|
||
*/
|
||
normalizeSize(value, fallback) {
|
||
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
|
||
return Math.floor(value);
|
||
}
|
||
|
||
if (typeof value === 'string') {
|
||
const trimmed = value.trim();
|
||
const match = trimmed.match(/^(\d+(?:\.\d+)?)([kKmMgGtT]?)$/);
|
||
if (match) {
|
||
const number = Number.parseFloat(match[1]);
|
||
const unit = match[2].toLowerCase();
|
||
const unitMap = {
|
||
'': 1,
|
||
k: 1024,
|
||
m: 1024 * 1024,
|
||
g: 1024 * 1024 * 1024,
|
||
t: 1024 * 1024 * 1024 * 1024,
|
||
};
|
||
return Math.floor(number * (unitMap[unit] || 1));
|
||
}
|
||
}
|
||
|
||
return fallback;
|
||
}
|
||
|
||
/**
|
||
* 规范化正整数。
|
||
* @param {*} value 数值或字符串。
|
||
* @param {number} fallback 回退值。
|
||
* @return {number} 规范化后的正整数。
|
||
*/
|
||
normalizePositiveInteger(value, fallback) {
|
||
const parsed = Number.parseInt(value, 10);
|
||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||
}
|
||
|
||
/**
|
||
* 解析时间值并返回毫秒时间戳。
|
||
* @param {Date|string|number} [value] 时间值。
|
||
* @return {number} 毫秒时间戳。
|
||
*/
|
||
resolveTimestamp(value) {
|
||
if (value == null) {
|
||
return 0;
|
||
}
|
||
|
||
if (value instanceof Date) {
|
||
return value.getTime();
|
||
}
|
||
|
||
if (typeof value === 'number' && Number.isFinite(value)) {
|
||
return value;
|
||
}
|
||
|
||
if (typeof value === 'string') {
|
||
const trimmed = value.trim();
|
||
if (!trimmed) {
|
||
return 0;
|
||
}
|
||
|
||
if (/^\d+$/.test(trimmed)) {
|
||
return Number.parseInt(trimmed, 10);
|
||
}
|
||
|
||
const parsed = Date.parse(trimmed);
|
||
if (!Number.isNaN(parsed)) {
|
||
return parsed;
|
||
}
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/**
|
||
* 把毫秒时间戳转换为 ISO 字符串。
|
||
* @param {number} timestamp 毫秒时间戳。
|
||
* @return {string|null} ISO 字符串。
|
||
*/
|
||
formatTimestamp(timestamp) {
|
||
if (!Number.isFinite(timestamp) || timestamp <= 0) {
|
||
return null;
|
||
}
|
||
|
||
return new Date(timestamp).toISOString();
|
||
}
|
||
|
||
/**
|
||
* 获取本次消费限额。
|
||
* @param {Object} [options] 消费选项。
|
||
* @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。
|
||
* @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。
|
||
* @return {number} 当前消费限额。
|
||
*/
|
||
getConsumeLimit(options = {}) {
|
||
return this.normalizePositiveInteger(
|
||
options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun,
|
||
this.maxConsumeRecordsPerWindow
|
||
);
|
||
}
|
||
|
||
/**
|
||
* 解析消费起始偏移量。
|
||
* @param {Object|null} progress 进度数据。
|
||
* @param {string} filePath 当前文件路径。
|
||
* @param {Object} [options] 消费选项。
|
||
* @param {number} [options.startOffset] 显式起始偏移量。
|
||
* @return {number} 起始偏移量。
|
||
*/
|
||
resolveConsumeStartOffset(progress, filePath, options = {}) {
|
||
if (options.startOffset != null) {
|
||
return this.normalizeInteger(options.startOffset, 0);
|
||
}
|
||
|
||
if (progress && progress.filePath && path.resolve(progress.filePath) === path.resolve(filePath)) {
|
||
return this.normalizeInteger(progress.offset, 0);
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/**
|
||
* 获取某个桶的共享写入窗口计数器文件路径。
|
||
* @param {number} bucketIndex 桶索引。
|
||
* @return {string} 计数器文件路径。
|
||
*/
|
||
getWriteWindowCounterPath(bucketIndex = 0) {
|
||
return path.join(this.baseDir, `.write-window-counter-${bucketIndex}.json`);
|
||
}
|
||
|
||
/**
|
||
* 多桶分片原子限流。
|
||
* 将总限额均分到 N 个桶,从首选桶开始尝试,竞争失败或桶满时自动漂移到下一个桶。
|
||
* @return {Promise<Object>} { allowed: boolean, counter: { startedAt, recordCount }, source: 'file' }。
|
||
*/
|
||
async atomicCheckAndIncrementWriteWindow() {
|
||
if (this.maxWriteRecordsPerWindow <= 0) {
|
||
return { allowed: true, counter: { startedAt: 0, recordCount: 0 } };
|
||
}
|
||
|
||
const bucketCount = this.writeWindowBucketCount;
|
||
const perBucketLimit = Math.ceil(this.maxWriteRecordsPerWindow / bucketCount);
|
||
const writeWindowMs = this.writeWindowSeconds * 1000;
|
||
const now = Date.now();
|
||
|
||
await this.ensureParentDirectory(this.baseDir);
|
||
|
||
let lastCounter = null;
|
||
|
||
for (let i = 0; i < bucketCount; i++) {
|
||
const bucketIndex = (this.preferredBucket + i) % bucketCount;
|
||
const counterPath = this.getWriteWindowCounterPath(bucketIndex);
|
||
const lockPath = `${counterPath}.lock`;
|
||
|
||
let lockHandle = null;
|
||
try {
|
||
// eslint-disable-next-line no-bitwise
|
||
lockHandle = await fsp.open(lockPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY);
|
||
} catch (error) {
|
||
if (error.code === 'EEXIST' || error.code === 'EPERM') {
|
||
continue;
|
||
}
|
||
throw error;
|
||
}
|
||
|
||
try {
|
||
const counter = { startedAt: 0, recordCount: 0 };
|
||
try {
|
||
const content = await fsp.readFile(counterPath, 'utf8');
|
||
if (content.trim()) {
|
||
const data = JSON.parse(content);
|
||
counter.startedAt = typeof data.startedAt === 'number' && Number.isFinite(data.startedAt) ? data.startedAt : 0;
|
||
counter.recordCount = typeof data.recordCount === 'number' && Number.isFinite(data.recordCount) ? data.recordCount : 0;
|
||
}
|
||
} catch (error) {
|
||
if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) {
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
if (counter.startedAt > 0 && now - counter.startedAt >= writeWindowMs) {
|
||
counter.startedAt = now;
|
||
counter.recordCount = 0;
|
||
}
|
||
|
||
if (counter.startedAt === 0) {
|
||
counter.startedAt = now;
|
||
}
|
||
|
||
lastCounter = counter;
|
||
|
||
if (counter.recordCount >= perBucketLimit) {
|
||
continue;
|
||
}
|
||
|
||
counter.recordCount += 1;
|
||
await fsp.writeFile(counterPath, `${JSON.stringify(counter)}\n`, 'utf8');
|
||
return { allowed: true, counter, source: 'file' };
|
||
} finally {
|
||
await lockHandle.close();
|
||
await fsp.rm(lockPath, { force: true });
|
||
}
|
||
}
|
||
|
||
return { allowed: false, counter: lastCounter || { startedAt: now, recordCount: 0 }, source: 'file' };
|
||
}
|
||
|
||
/**
|
||
* 获取当前写入窗口状态(汇总所有桶)。
|
||
* @return {Promise<Object>} 写入窗口状态。
|
||
*/
|
||
async getWriteWindowState() {
|
||
const writeWindowMs = this.writeWindowSeconds * 1000;
|
||
const now = Date.now();
|
||
let totalRecordCount = 0;
|
||
let earliestStartedAt = 0;
|
||
|
||
for (let i = 0; i < this.writeWindowBucketCount; i++) {
|
||
try {
|
||
const content = await fsp.readFile(this.getWriteWindowCounterPath(i), 'utf8');
|
||
if (content.trim()) {
|
||
const data = JSON.parse(content);
|
||
const startedAt = typeof data.startedAt === 'number' && Number.isFinite(data.startedAt) ? data.startedAt : 0;
|
||
const recordCount = typeof data.recordCount === 'number' && Number.isFinite(data.recordCount) ? data.recordCount : 0;
|
||
if (startedAt > 0 && now - startedAt < writeWindowMs) {
|
||
totalRecordCount += recordCount;
|
||
if (earliestStartedAt === 0 || startedAt < earliestStartedAt) {
|
||
earliestStartedAt = startedAt;
|
||
}
|
||
}
|
||
}
|
||
} catch { /* ignore missing/corrupt counter files */ }
|
||
}
|
||
|
||
const windowExpired = earliestStartedAt > 0 && now - earliestStartedAt >= writeWindowMs;
|
||
const resumeAt = earliestStartedAt > 0
|
||
? new Date(earliestStartedAt + writeWindowMs).toISOString()
|
||
: null;
|
||
|
||
return {
|
||
writeWindowSeconds: this.writeWindowSeconds,
|
||
maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow,
|
||
writeWindowBucketCount: this.writeWindowBucketCount,
|
||
writeWindowRecordCount: totalRecordCount,
|
||
writeWindowStartedAt: earliestStartedAt > 0
|
||
? new Date(earliestStartedAt).toISOString()
|
||
: null,
|
||
windowExpired,
|
||
resumeAt: windowExpired ? null : resumeAt,
|
||
source: 'file',
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 将字符串映射到 [0, max) 范围内的索引。
|
||
* @param {string} str 输入字符串。
|
||
* @param {number} max 范围上限。
|
||
* @return {number} 索引。
|
||
*/
|
||
hashStringToIndex(str, max) {
|
||
let hash = 0;
|
||
for (let i = 0; i < str.length; i++) {
|
||
// eslint-disable-next-line no-bitwise
|
||
hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0;
|
||
}
|
||
return ((hash % max) + max) % max;
|
||
}
|
||
|
||
/**
|
||
* 确保目录存在。
|
||
* @param {string} dirPath 目录路径。
|
||
* @return {Promise<void>} Promise。
|
||
*/
|
||
async ensureParentDirectory(dirPath) {
|
||
await fsp.mkdir(dirPath, { recursive: true });
|
||
}
|
||
|
||
/**
|
||
* 判断是否需要执行清理。
|
||
* @return {Promise<void>} Promise。
|
||
*/
|
||
async maybeCleanup() {
|
||
if (!this.cleanupIntervalMs || this.cleanupIntervalMs <= 0) {
|
||
return;
|
||
}
|
||
|
||
const now = Date.now();
|
||
if (now - this.lastCleanupAt < this.cleanupIntervalMs) {
|
||
return;
|
||
}
|
||
|
||
this.lastCleanupAt = now;
|
||
try {
|
||
await this.cleanupExpiredFiles();
|
||
} catch (error) {
|
||
this.app.logger.warn('[fileWriter.cleanup] 清理本地文件失败', error.message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 递归清理目录中的过期文件。
|
||
* @param {string} dirPath 当前目录。
|
||
* @param {number} cutoffTime 过期时间戳。
|
||
* @param {Object} result 清理统计。
|
||
* @return {Promise<void>} Promise。
|
||
*/
|
||
async walkAndCleanup(dirPath, cutoffTime, result) {
|
||
let entries;
|
||
try {
|
||
entries = await fsp.readdir(dirPath, { withFileTypes: true });
|
||
} catch (error) {
|
||
if (error.code === 'ENOENT') {
|
||
return;
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
|
||
for (const entry of entries) {
|
||
const entryPath = path.join(dirPath, entry.name);
|
||
if (entry.isDirectory()) {
|
||
await this.walkAndCleanup(entryPath, cutoffTime, result);
|
||
await this.removeEmptyDirectory(entryPath, result);
|
||
continue;
|
||
}
|
||
|
||
if (!entry.isFile()) {
|
||
continue;
|
||
}
|
||
|
||
if (!entry.name.endsWith(this.fileExtension)) {
|
||
// 【风险点】清理只覆盖数据文件,.progress 文件不会被清理,长期会残留陈旧进度文件。
|
||
continue;
|
||
}
|
||
|
||
const completion = await this.loadCompletionMarker(entryPath);
|
||
if (!completion || path.resolve(completion.filePath || entryPath) !== path.resolve(entryPath)) {
|
||
continue;
|
||
}
|
||
|
||
const stat = await fsp.stat(entryPath).catch(error => {
|
||
if (error.code === 'ENOENT') {
|
||
return null;
|
||
}
|
||
|
||
throw error;
|
||
});
|
||
|
||
if (!stat || stat.mtimeMs >= cutoffTime) {
|
||
continue;
|
||
}
|
||
|
||
await fsp.rm(entryPath, { force: true });
|
||
await fsp.rm(this.getDoneFilePath(entryPath), { force: true });
|
||
result.removedFiles += 1;
|
||
|
||
const siblingEntries = await fsp.readdir(dirPath, { withFileTypes: true });
|
||
for (const siblingEntry of siblingEntries) {
|
||
if (!siblingEntry.isFile() || !siblingEntry.name.endsWith(this.progressExtension)) {
|
||
continue;
|
||
}
|
||
|
||
const progressPath = path.join(dirPath, siblingEntry.name);
|
||
const progressData = await this.loadProgress(progressPath);
|
||
if (!progressData || !progressData.completedAt) {
|
||
continue;
|
||
}
|
||
|
||
if (path.resolve(progressData.filePath || '') !== path.resolve(entryPath)) {
|
||
continue;
|
||
}
|
||
|
||
await fsp.rm(progressPath, { force: true });
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 原子写入文本文件。
|
||
* @param {string} targetPath 目标文件路径。
|
||
* @param {string} content 文件内容。
|
||
* @return {Promise<void>} Promise。
|
||
*/
|
||
async writeAtomicTextFile(targetPath, content) {
|
||
const directory = path.dirname(targetPath);
|
||
const tempPath = `${targetPath}.${this.pid}.${Date.now()}.tmp`;
|
||
|
||
await this.ensureParentDirectory(directory);
|
||
|
||
const handle = await fsp.open(tempPath, 'w');
|
||
try {
|
||
await handle.write(content);
|
||
await handle.sync();
|
||
} finally {
|
||
await handle.close();
|
||
}
|
||
|
||
await fsp.rename(tempPath, targetPath);
|
||
}
|
||
|
||
/**
|
||
* 删除空目录。
|
||
* @param {string} dirPath 目录路径。
|
||
* @param {Object} result 清理统计。
|
||
* @return {Promise<void>} Promise。
|
||
*/
|
||
async removeEmptyDirectory(dirPath, result) {
|
||
try {
|
||
const entries = await fsp.readdir(dirPath);
|
||
if (entries.length > 0) {
|
||
return;
|
||
}
|
||
|
||
await fsp.rmdir(dirPath);
|
||
result.removedDirectories += 1;
|
||
} catch (error) {
|
||
if (error.code === 'ENOENT' || error.code === 'ENOTEMPTY') {
|
||
return;
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
}
|
||
}
|
||
|
||
module.exports = LocalFileWriter;
|