'use strict'; const fs = require('node:fs'); const fsp = require('node:fs/promises'); const os = require('node:os'); const path = require('node:path'); const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; const DEFAULT_MAX_FILE_LINES = 2000; const DEFAULT_MAX_CONSUME_RECORDS_PER_WINDOW = 1000; const DEFAULT_WRITE_WINDOW_SECONDS = 1; const DEFAULT_MAX_WRITE_RECORDS_PER_WINDOW = 2000; const DEFAULT_RETENTION_DAYS = 7; const DEFAULT_CLEANUP_INTERVAL_MS = 60 * 60 * 1000; const DEFAULT_STATE_MAP_MAX_ENTRIES = 256; const DEFAULT_WRITE_WINDOW_BUCKET_COUNT = 8; /** * 本地文件队列写入器。 * 负责把业务数据按 JSON Lines 追加到本地文件,并保证单条写入串行、fsync 落盘和自动轮转。 */ class LocalFileWriter { /** * 创建本地文件写入器实例。 * @param {Egg.Application} app Egg 应用实例。 */ constructor(app) { this.app = app; this.config = app.config.fileWriter || {}; this.baseDir = path.resolve(this.config.baseDir || path.join(app.baseDir, 'runtime', 'file-writer')); this.maxFileSize = this.normalizeSize(this.config.maxFileSize, DEFAULT_MAX_FILE_SIZE); this.maxFileLines = this.normalizePositiveInteger(this.config.maxFileLines, DEFAULT_MAX_FILE_LINES); this.maxConsumeRecordsPerWindow = this.normalizePositiveInteger( this.config.maxConsumeRecordsPerWindow ?? this.config.maxConsumeRecordsPerRun ?? app.config.maxConsumeRecordsPerWindow ?? app.config.maxConsumeRecordsPerRun, DEFAULT_MAX_CONSUME_RECORDS_PER_WINDOW ); this.maxConsumeRecordsPerRun = this.maxConsumeRecordsPerWindow; this.writeWindowSeconds = this.normalizeInteger( this.config.writeWindowSeconds ?? app.config.writeWindowSeconds, DEFAULT_WRITE_WINDOW_SECONDS ); this.maxWriteRecordsPerWindow = this.normalizeInteger( this.config.maxWriteRecordsPerWindow ?? app.config.maxWriteRecordsPerWindow, DEFAULT_MAX_WRITE_RECORDS_PER_WINDOW ); this.writeWindowStartedAt = 0; this.writeWindowRecordCount = 0; this.writeWindowBucketCount = this.normalizePositiveInteger( this.config.writeWindowBucketCount ?? app.config.writeWindowBucketCount, DEFAULT_WRITE_WINDOW_BUCKET_COUNT ); this.preferredBucket = this.hashStringToIndex(this.workerId + '-' + this.pid, this.writeWindowBucketCount); this.retentionDays = this.normalizeInteger(this.config.retentionDays, DEFAULT_RETENTION_DAYS); this.cleanupIntervalMs = this.normalizeInteger(this.config.cleanupIntervalMs, DEFAULT_CLEANUP_INTERVAL_MS); this.machineId = this.sanitizeSegment(this.config.machineId || os.hostname() || 'machine'); this.workerId = this.sanitizeSegment(this.config.workerId || process.env.EGG_WORKER_ID || 'worker'); this.pid = process.pid; this.processTag = `${this.machineId}-${this.workerId}-p${this.pid}`; this.sequenceWidth = this.normalizeInteger(this.config.sequenceWidth, 4); this.stateMapMaxEntries = this.normalizePositiveInteger(this.config.stateMapMaxEntries, DEFAULT_STATE_MAP_MAX_ENTRIES); this.fileExtension = this.normalizeExtension(this.config.fileExtension, '.log'); this.progressExtension = this.normalizeExtension(this.config.progressExtension, '.progress'); this.doneExtension = this.normalizeExtension(this.config.doneExtension, '.done'); this.stateMap = new Map(); this.queue = Promise.resolve(); this.lastCleanupAt = 0; } /** * 将写入任务串行排队执行。 * @param {Function} task 需要排队执行的异步任务。 * @return {Promise<*>} 任务执行结果。 */ enqueue(task) { const currentTask = this.queue.then(() => task()); this.queue = currentTask.then(() => undefined, () => undefined); return currentTask; } /** * 等待当前队列里的所有写入完成。 * @return {Promise} 等待完成的 Promise。 */ async flush() { await this.queue; } /** * 把业务数据写入本地 JSON Lines 文件。 * @param {string} businessName 业务名称。 * @param {*} record 需要写入的数据。 * @param {Object} [options] 写入选项。 * @param {Date|string|number} [options.date] 指定归档日期,默认使用当前时间。 * @return {Promise} 写入结果。 */ async writeRecord(businessName, record, options = {}) { return this.enqueue(async () => { const windowCheck = await this.atomicCheckAndIncrementWriteWindow(); this.writeWindowStartedAt = windowCheck.counter.startedAt; this.writeWindowRecordCount = windowCheck.counter.recordCount; if (!windowCheck.allowed) { const writeWindowMs = this.writeWindowSeconds * 1000; const resumeAt = new Date(windowCheck.counter.startedAt + writeWindowMs).toISOString(); return { throttled: true, businessName, resumeAt, writeWindowSeconds: this.writeWindowSeconds, writeWindowRecordCount: windowCheck.counter.recordCount, maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow, }; } const partitionState = await this.getPartitionState(businessName, options.date); const jsonLine = this.stringifyRecord(record); const payload = `${jsonLine}\n`; const payloadSize = Buffer.byteLength(payload); if ( partitionState.currentLineCount >= this.maxFileLines || (partitionState.currentSize > 0 && partitionState.currentSize + payloadSize > this.maxFileSize) ) { partitionState.sequence += 1; partitionState.currentFilePath = this.buildDataFilePath(partitionState, partitionState.sequence); partitionState.currentSize = 0; partitionState.currentLineCount = 0; } await this.ensureParentDirectory(partitionState.partitionDir); const handle = await fsp.open(partitionState.currentFilePath, 'a'); try { await handle.write(payload); await handle.sync(); } finally { await handle.close(); } partitionState.currentSize += payloadSize; partitionState.currentLineCount += 1; await this.maybeCleanup(); return { businessName: partitionState.businessName, dateKey: partitionState.dateKey, filePath: partitionState.currentFilePath, size: partitionState.currentSize, lineCount: partitionState.currentLineCount, bytesWritten: payloadSize, sequence: partitionState.sequence, machineId: this.machineId, workerId: this.workerId, pid: this.pid, throttled: false, writeWindowRecordCount: windowCheck.counter.recordCount, maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow, }; }); } /** * 写入方法别名,便于业务侧直接调用。 * @param {string} businessName 业务名称。 * @param {*} record 需要写入的数据。 * @param {Object} [options] 写入选项。 * @return {Promise} 写入结果。 */ async write(businessName, record, options = {}) { return this.writeRecord(businessName, record, options); } /** * 获取某个业务在指定日期下的目录信息。 * @param {string} businessName 业务名称。 * @param {Date|string|number} [date] 指定日期。 * @return {Object} 目录信息。 */ getPartitionInfo(businessName, date = new Date()) { const businessSegment = this.sanitizeSegment(businessName || 'default'); const dateParts = this.getDateParts(date); const partitionDir = path.join(this.baseDir, businessSegment, dateParts.year, dateParts.month, dateParts.day); return { businessName, businessSegment, dateKey: dateParts.dateKey, partitionDir, filePrefix: `${businessSegment}-${this.processTag}`, processTag: this.processTag, }; } /** * 获取消费进度文件路径。 * @param {string} businessName 业务名称。 * @param {Date|string|number|null} [date] 指定日期;传空时使用业务级共享进度文件。 * @param {string} [consumerName] 消费者名称。 * @return {string} 进度文件路径。 */ getProgressFilePath(businessName, date = new Date(), consumerName = 'default') { const businessSegment = this.sanitizeSegment(businessName || 'default'); const safeConsumerName = this.sanitizeSegment(consumerName || 'default'); if (date == null) { return path.join(this.baseDir, businessSegment, `.${safeConsumerName}${this.progressExtension}`); } const partitionInfo = this.getPartitionInfo(businessName, date); return path.join(partitionInfo.partitionDir, `.${safeConsumerName}${this.progressExtension}`); } /** * 获取某个数据文件的消费完成标记路径。 * @param {string} filePath 数据文件路径。 * @return {string} 完成标记路径。 */ getDoneFilePath(filePath) { const resolvedFilePath = path.resolve(filePath); return `${resolvedFilePath}${this.doneExtension}`; } /** * 列出某个业务日期分区下的所有日志文件。 * @param {string} businessName 业务名称。 * @param {Date|string|number|null} [date] 指定日期;传空时递归列出业务目录下所有日志文件。 * @return {Promise>} 排序后的文件路径列表。 */ async listDataFiles(businessName, date = new Date()) { const businessSegment = this.sanitizeSegment(businessName || 'default'); const businessDir = path.join(this.baseDir, businessSegment); const targetDir = date == null ? businessDir : this.getPartitionInfo(businessName, date).partitionDir; const files = await this.collectDataFiles(targetDir); return files .sort((left, right) => { return this.compareDataFileSortKey(left.sortKey, right.sortKey); }) .map(entry => entry.filePath); } /** * 读取单个日志文件中的 JSON Lines 数据。 * @param {string} filePath 日志文件路径。 * @param {Object} [options] 读取选项。 * @param {number} [options.startOffset=0] 起始字节偏移量。 * @param {number} [options.maxRecords=0] 最大读取条数,0 或不传不限制。 * @param {boolean} [options.parseJson=true] 是否自动解析 JSON。 * @return {Promise} 读取结果。 */ async readRecords(filePath, options = {}) { const resolvedFilePath = path.resolve(filePath); const startOffset = this.normalizeInteger(options.startOffset, 0); const maxRecords = this.normalizeInteger(options.maxRecords, 0); const parseJson = options.parseJson !== false; const records = []; let nextOffset = startOffset; let lineNumber = 0; try { for await (const lineEntry of this.iterateJsonLines(resolvedFilePath, startOffset)) { lineNumber = lineEntry.lineNumber; nextOffset = lineEntry.nextOffset; if (!lineEntry.line.trim()) { continue; } if (!parseJson) { records.push(lineEntry.line); if (maxRecords > 0 && records.length >= maxRecords) { break; } continue; } try { records.push(JSON.parse(lineEntry.line)); if (maxRecords > 0 && records.length >= maxRecords) { break; } } catch (parseError) { parseError.message = `解析文件 ${resolvedFilePath} 第 ${lineNumber} 行失败: ${parseError.message}`; throw parseError; } } } catch (error) { if (error.code === 'ENOENT') { return { filePath: resolvedFilePath, records: [], nextOffset: startOffset, lineNumber: 0, }; } throw error; } return { filePath: resolvedFilePath, records, nextOffset, lineNumber, }; } /** * 消费某个文件,并在每条记录处理完成后更新进度文件。 * @param {string} filePath 日志文件路径。 * @param {Object} options 消费选项。 * @param {string} options.progressFilePath 进度文件路径。 * @param {Function} options.onRecord 每条记录的处理函数。 * @param {number} [options.startOffset=0] 起始偏移量。 * @param {boolean} [options.parseJson=true] 是否自动解析 JSON。 * @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。 * @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。 * @param {Object|null} [options.progress] 已加载的进度数据,供内部复用。 * @return {Promise} 消费结果,包含 filePath、processedCount、nextOffset、limitReached、maxConsumeRecordsPerWindow。 */ async consumeFile(filePath, options = {}) { const progressFilePath = options.progressFilePath; const onRecord = options.onRecord; if (!progressFilePath) { throw new Error('progressFilePath 不能为空'); } if (typeof onRecord !== 'function') { throw new Error('onRecord 必须是函数'); } const progress = options.progress || await this.loadProgress(progressFilePath); const consumeLimit = this.normalizePositiveInteger( options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun, this.maxConsumeRecordsPerWindow ); const resolvedFilePath = path.resolve(filePath); const startOffset = this.resolveConsumeStartOffset(progress, resolvedFilePath, options); const parseJson = options.parseJson !== false; let offset = startOffset; let processedCount = 0; let lineNumber = 0; let limitReached = false; try { for await (const lineEntry of this.iterateJsonLines(resolvedFilePath, startOffset)) { if (consumeLimit > 0 && processedCount >= consumeLimit) { limitReached = true; break; } lineNumber = lineEntry.lineNumber; offset = lineEntry.nextOffset; if (!lineEntry.line.trim()) { continue; } let record = lineEntry.line; if (parseJson) { try { record = JSON.parse(lineEntry.line); } catch (parseError) { parseError.message = `解析文件 ${resolvedFilePath} 第 ${lineNumber} 行失败: ${parseError.message}`; throw parseError; } } await onRecord(record, { filePath: resolvedFilePath, offset, index: processedCount, }); processedCount += 1; const progressData = { filePath: resolvedFilePath, offset, processedCount, updatedAt: new Date().toISOString(), }; await this.saveProgress(progressFilePath, progressData); if (consumeLimit > 0 && processedCount >= consumeLimit) { limitReached = true; break; } } } catch (error) { if (error.code === 'ENOENT') { return { filePath: resolvedFilePath, processedCount: 0, nextOffset: startOffset, limitReached: false, maxConsumeRecordsPerRun: consumeLimit, maxConsumeRecordsPerWindow: consumeLimit, }; } throw error; } if (limitReached) { return { filePath: resolvedFilePath, processedCount, nextOffset: offset, limitReached: true, maxConsumeRecordsPerRun: consumeLimit, maxConsumeRecordsPerWindow: consumeLimit, }; } const completionData = { filePath: resolvedFilePath, offset, processedCount, completedAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }; await this.saveProgress(progressFilePath, completionData); await this.saveCompletionMarker(resolvedFilePath, completionData); return { filePath: resolvedFilePath, processedCount, nextOffset: offset, limitReached: false, maxConsumeRecordsPerRun: consumeLimit, maxConsumeRecordsPerWindow: consumeLimit, }; } /** * 按业务目录消费所有日志文件,支持断点续传。 * @param {Object} options 消费选项。 * @param {string} options.businessName 业务名称。 * @param {string} options.progressFilePath 进度文件路径。 * @param {Function} options.onRecord 每条记录的处理函数。 * @param {boolean} [options.parseJson=true] 是否自动解析 JSON。 * @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。 * @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。 * @return {Promise} 消费结果,包含 businessName、dataFiles、processedCount、progress、limitReached、maxConsumeRecordsPerWindow。 */ async consumeFiles(options = {}) { const businessName = options.businessName; const progressFilePath = options.progressFilePath; const onRecord = options.onRecord; if (!businessName) { throw new Error('businessName 不能为空'); } if (!progressFilePath) { throw new Error('progressFilePath 不能为空'); } if (typeof onRecord !== 'function') { throw new Error('onRecord 必须是函数'); } const progress = await this.loadProgress(progressFilePath); const consumeLimit = this.normalizePositiveInteger( options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun, this.maxConsumeRecordsPerWindow ); const dataFiles = await this.listDataFiles(businessName, null); if (dataFiles.length === 0) { return { businessName, dataFiles, processedCount: 0, progress, limitReached: false, maxConsumeRecordsPerRun: consumeLimit, maxConsumeRecordsPerWindow: consumeLimit, }; } let startIndex = 0; let startOffset = 0; let limitReached = false; if (progress && progress.filePath) { const matchedIndex = dataFiles.findIndex(filePath => path.resolve(filePath) === path.resolve(progress.filePath)); if (matchedIndex >= 0) { startIndex = matchedIndex; startOffset = this.normalizeInteger(progress.offset, 0); } else { const progressSortKey = this.getDataFileSortKey(progress.filePath); const fallbackIndex = dataFiles.findIndex(filePath => this.compareDataFileSortKey( this.getDataFileSortKey(filePath), progressSortKey ) > 0); if (fallbackIndex >= 0) { startIndex = fallbackIndex; startOffset = 0; } else { startIndex = dataFiles.length; } } } let processedCount = 0; let currentProgress = progress; let totalConsumedCount = 0; for (let index = startIndex; index < dataFiles.length; index += 1) { if (consumeLimit > 0 && totalConsumedCount >= consumeLimit) { limitReached = true; break; } const filePath = dataFiles[index]; const completion = await this.loadCompletionMarker(filePath); if (completion && path.resolve(completion.filePath || filePath) === path.resolve(filePath)) { currentProgress = { ...completion, filePath: completion.filePath || filePath, updatedAt: new Date().toISOString(), }; await this.saveProgress(progressFilePath, currentProgress); continue; } const remainingLimit = consumeLimit > 0 ? consumeLimit - totalConsumedCount : consumeLimit; const fileResult = await this.consumeFile(filePath, { progressFilePath, onRecord, startOffset: index === startIndex ? startOffset : 0, parseJson: options.parseJson, maxConsumeRecordsPerWindow: remainingLimit, progress, }); processedCount += fileResult.processedCount; totalConsumedCount += fileResult.processedCount; currentProgress = { filePath, offset: fileResult.nextOffset, processedCount, updatedAt: new Date().toISOString(), }; await this.saveProgress(progressFilePath, currentProgress); if (fileResult.limitReached) { limitReached = true; break; } } return { businessName, dataFiles, processedCount, progress: currentProgress, limitReached, maxConsumeRecordsPerRun: consumeLimit, maxConsumeRecordsPerWindow: consumeLimit, }; } /** * 递归收集目录下的所有数据文件,并记录用于排序的分区日期与序号。 * @param {string} dirPath 目录路径。 * @param {Array} [result=[]] 累积结果。 * @return {Promise>} 收集结果。 */ async collectDataFiles(dirPath, result = []) { let entries; try { entries = await fsp.readdir(dirPath, { withFileTypes: true }); } catch (error) { if (error.code === 'ENOENT') { return result; } throw error; } for (const entry of entries) { const entryPath = path.join(dirPath, entry.name); if (entry.isDirectory()) { await this.collectDataFiles(entryPath, result); continue; } if (!entry.isFile() || !entry.name.endsWith(this.fileExtension)) { continue; } result.push({ filePath: entryPath, sortKey: this.getDataFileSortKey(entryPath), }); } return result; } /** * 解析数据文件的消费顺序键,按分区日期和文件序号排序。 * @param {string} filePath 数据文件路径。 * @return {Object} 排序键。 */ getDataFileSortKey(filePath) { const resolvedFilePath = path.resolve(filePath); const relativeParts = path.relative(this.baseDir, resolvedFilePath).split(path.sep).filter(Boolean); const fileName = path.basename(resolvedFilePath); const sequenceMatch = fileName.match(/-(\d+)\.[^.]+$/); let partitionDateKey = ''; if (relativeParts.length >= 5) { const year = relativeParts[1]; const month = relativeParts[2]; const day = relativeParts[3]; if (/^\d+$/.test(year) && /^\d+$/.test(month) && /^\d+$/.test(day)) { partitionDateKey = `${year.padStart(4, '0')}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`; } } return { partitionDateKey, sequence: sequenceMatch ? Number.parseInt(sequenceMatch[1], 10) : Number.MAX_SAFE_INTEGER, filePath: resolvedFilePath, }; } /** * 比较两个数据文件的排序键。 * @param {Object} left 左侧排序键。 * @param {Object} right 右侧排序键。 * @return {number} 排序结果。 */ compareDataFileSortKey(left, right) { if (left.partitionDateKey !== right.partitionDateKey) { return left.partitionDateKey.localeCompare(right.partitionDateKey, 'en'); } if (left.sequence !== right.sequence) { return left.sequence - right.sequence; } return left.filePath.localeCompare(right.filePath, 'en', { numeric: true }); } /** * 读取进度文件。 * @param {string} progressFilePath 进度文件路径。 * @return {Promise} 进度内容。 */ async loadProgress(progressFilePath) { const resolvedPath = path.resolve(progressFilePath); try { const content = await fsp.readFile(resolvedPath, 'utf8'); if (!content.trim()) { return null; } return this.normalizeProgressData(JSON.parse(content), resolvedPath); } catch (error) { if (error.code === 'ENOENT') { return null; } if (error instanceof SyntaxError) { this.app.logger.warn('[fileWriter.loadProgress] 忽略损坏的进度文件', resolvedPath, error.message); return null; } throw error; } } /** * 保存进度文件。 * @param {string} progressFilePath 进度文件路径。 * @param {Object} progressData 进度数据。 * @return {Promise} 保存后的进度数据。 */ async saveProgress(progressFilePath, progressData) { const resolvedPath = path.resolve(progressFilePath); const payload = `${JSON.stringify({ ...progressData, updatedAt: progressData.updatedAt || new Date().toISOString(), }, null, 2)}\n`; await this.writeAtomicTextFile(resolvedPath, payload); return { ...progressData, updatedAt: progressData.updatedAt || new Date().toISOString(), }; } /** * 保存消费完成标记。 * @param {string} filePath 数据文件路径。 * @param {Object} completionData 完成数据。 * @return {Promise} 保存后的完成数据。 */ async saveCompletionMarker(filePath, completionData) { const resolvedFilePath = path.resolve(filePath); const doneFilePath = this.getDoneFilePath(resolvedFilePath); const payload = `${JSON.stringify({ ...completionData, filePath: resolvedFilePath, completedAt: completionData.completedAt || new Date().toISOString(), }, null, 2)}\n`; await this.writeAtomicTextFile(doneFilePath, payload); return { ...completionData, filePath: resolvedFilePath, completedAt: completionData.completedAt || new Date().toISOString(), }; } /** * 读取消费完成标记。 * @param {string} filePath 数据文件路径。 * @return {Promise} 完成标记内容。 */ async loadCompletionMarker(filePath) { const resolvedFilePath = path.resolve(filePath); const doneFilePath = this.getDoneFilePath(resolvedFilePath); try { const content = await fsp.readFile(doneFilePath, 'utf8'); if (!content.trim()) { return null; } return this.normalizeCompletionData(JSON.parse(content), resolvedFilePath); } catch (error) { if (error.code === 'ENOENT') { return null; } if (error instanceof SyntaxError) { this.app.logger.warn('[fileWriter.loadCompletionMarker] 忽略损坏的完成标记', doneFilePath, error.message); return null; } throw error; } } /** * 清理超过保留期的日志文件与进度文件。 * @return {Promise} 清理结果。 */ async cleanupExpiredFiles() { if (!this.retentionDays || this.retentionDays <= 0) { return { removedFiles: 0, removedDirectories: 0, }; } const cutoffTime = Date.now() - (this.retentionDays * 24 * 60 * 60 * 1000); const result = { removedFiles: 0, removedDirectories: 0, }; await this.walkAndCleanup(this.baseDir, cutoffTime, result); return result; } /** * 规范化日期并返回目录分段。 * @param {Date|string|number} date 日期。 * @return {Object} 日期分段。 */ getDateParts(date = new Date()) { const targetDate = date instanceof Date ? date : new Date(date); if (Number.isNaN(targetDate.getTime())) { throw new Error(`无效的日期参数: ${date}`); } const year = String(targetDate.getFullYear()); const month = String(targetDate.getMonth() + 1).padStart(2, '0'); const day = String(targetDate.getDate()).padStart(2, '0'); return { year, month, day, dateKey: `${year}-${month}-${day}`, }; } /** * 获取写入分区状态。 * @param {string} businessName 业务名称。 * @param {Date|string|number} [date] 指定日期。 * @return {Promise} 分区状态。 */ async getPartitionState(businessName, date = new Date()) { const partitionInfo = this.getPartitionInfo(businessName, date); const stateKey = `${partitionInfo.businessSegment}:${partitionInfo.dateKey}`; if (this.stateMap.has(stateKey)) { const existingState = this.stateMap.get(stateKey); this.stateMap.delete(stateKey); this.stateMap.set(stateKey, existingState); return existingState; } await this.ensureParentDirectory(partitionInfo.partitionDir); const state = { businessName, businessSegment: partitionInfo.businessSegment, dateKey: partitionInfo.dateKey, partitionDir: partitionInfo.partitionDir, filePrefix: partitionInfo.filePrefix, sequence: 1, currentSize: 0, currentLineCount: 0, currentFilePath: this.buildDataFilePath({ partitionDir: partitionInfo.partitionDir, filePrefix: partitionInfo.filePrefix, }, 1), }; const latestDataFileState = await this.loadLatestDataFileState(partitionInfo.partitionDir, partitionInfo.filePrefix); if (latestDataFileState) { state.sequence = latestDataFileState.sequence; state.currentSize = latestDataFileState.currentSize; state.currentLineCount = latestDataFileState.currentLineCount; state.currentFilePath = latestDataFileState.currentFilePath; } this.evictOldestStateIfNeeded(); this.stateMap.set(stateKey, state); return state; } /** * 读取分区中最新数据文件的大小与行数,用于恢复写入状态。 * @param {string} partitionDir 分区目录。 * @param {string} filePrefix 文件前缀。 * @return {Promise} 最新文件状态。 */ async loadLatestDataFileState(partitionDir, filePrefix) { const dataFiles = await this.collectDataFiles(partitionDir); if (dataFiles.length === 0) { return null; } dataFiles.sort((left, right) => this.compareDataFileSortKey(left.sortKey, right.sortKey)); const latestFile = dataFiles[dataFiles.length - 1]; const stat = await fsp.stat(latestFile.filePath); const currentLineCount = await this.countFileLines(latestFile.filePath); return { filePrefix, sequence: latestFile.sortKey.sequence > 0 ? latestFile.sortKey.sequence : 1, currentFilePath: latestFile.filePath, currentSize: stat.size, currentLineCount, }; } /** * 统计文件中的行数。 * @param {string} filePath 文件路径。 * @return {Promise} 行数。 */ async countFileLines(filePath) { let lineCount = 0; let lastByte = null; const stream = fs.createReadStream(filePath); try { for await (const chunk of stream) { lastByte = chunk.length > 0 ? chunk[chunk.length - 1] : lastByte; for (let i = 0; i < chunk.length; i += 1) { if (chunk[i] === 0x0A) { lineCount += 1; } } } } finally { stream.destroy(); } if (lineCount === 0) { return 0; } if (lastByte !== 0x0A) { lineCount += 1; } return lineCount; } /** * 逐行读取 JSON Lines 文件,并返回每行的精确字节偏移。 * @param {string} filePath 文件路径。 * @param {number} [startOffset=0] 起始字节偏移量。 * @return {AsyncGenerator} 逐行迭代器。 */ async* iterateJsonLines(filePath, startOffset = 0) { const stream = fs.createReadStream(filePath, { start: startOffset, }); let buffer = Buffer.alloc(0); let offset = startOffset; let lineNumber = 0; try { for await (const chunk of stream) { buffer = Buffer.concat([ buffer, chunk ]); while (buffer.length > 0) { const lineFeedIndex = buffer.indexOf(0x0A); if (lineFeedIndex < 0) { break; } const lineEndIndex = lineFeedIndex > 0 && buffer[lineFeedIndex - 1] === 0x0D ? lineFeedIndex - 1 : lineFeedIndex; const consumedBytes = lineFeedIndex + 1; const lineBuffer = buffer.slice(0, lineEndIndex); lineNumber += 1; offset += consumedBytes; buffer = buffer.slice(consumedBytes); yield { lineNumber, line: lineBuffer.toString('utf8'), nextOffset: offset, }; } } if (buffer.length > 0) { lineNumber += 1; offset += buffer.length; yield { lineNumber, line: buffer.toString('utf8'), nextOffset: offset, }; } } finally { stream.destroy(); } } /** * 归一化进度文件内容,兼容损坏或旧字段。 * @param {*} progressData 原始进度内容。 * @param {string} [fallbackFilePath] 兜底文件路径。 * @return {Object|null} 归一化后的进度内容。 */ normalizeProgressData(progressData, fallbackFilePath = null) { if (!progressData || typeof progressData !== 'object') { return null; } const filePath = typeof progressData.filePath === 'string' && progressData.filePath.trim() ? path.resolve(progressData.filePath) : (fallbackFilePath || null); const offset = this.normalizeInteger(progressData.offset ?? progressData.nextOffset, 0); const processedCount = this.normalizeInteger(progressData.processedCount ?? progressData.processed ?? progressData.count, 0); return { ...progressData, filePath, offset, processedCount, updatedAt: typeof progressData.updatedAt === 'string' && progressData.updatedAt.trim() ? progressData.updatedAt : undefined, completedAt: typeof progressData.completedAt === 'string' && progressData.completedAt.trim() ? progressData.completedAt : undefined, }; } /** * 归一化完成标记内容,兼容损坏或旧字段。 * @param {*} completionData 原始完成标记内容。 * @param {string} fallbackFilePath 兜底文件路径。 * @return {Object|null} 归一化后的完成标记内容。 */ normalizeCompletionData(completionData, fallbackFilePath) { if (!completionData || typeof completionData !== 'object') { return null; } const filePath = typeof completionData.filePath === 'string' && completionData.filePath.trim() ? path.resolve(completionData.filePath) : fallbackFilePath; if (!filePath) { return null; } const completedAtMs = this.resolveTimestamp(completionData.completedAt ?? completionData.updatedAt); return { ...completionData, filePath, completedAt: this.formatTimestamp(completedAtMs), updatedAt: typeof completionData.updatedAt === 'string' && completionData.updatedAt.trim() ? completionData.updatedAt : undefined, }; } /** * 回收最旧的分区状态,避免 stateMap 无界增长。 * @return {void} */ evictOldestStateIfNeeded() { if (this.stateMap.size < this.stateMapMaxEntries) { return; } const oldestKey = this.stateMap.keys().next().value; if (oldestKey !== undefined) { this.stateMap.delete(oldestKey); } } /** * 构建日志文件路径。 * @param {Object} partitionState 分区状态。 * @param {number} sequence 序号。 * @return {string} 文件路径。 */ buildDataFilePath(partitionState, sequence) { const sequenceText = String(sequence).padStart(this.sequenceWidth, '0'); const fileName = `${partitionState.filePrefix}-${sequenceText}${this.fileExtension}`; return path.join(partitionState.partitionDir, fileName); } /** * 把要写入的记录转换为 JSON 字符串。 * @param {*} record 记录内容。 * @return {string} JSON 字符串。 */ stringifyRecord(record) { const json = JSON.stringify(record, this.jsonReplacer); if (json === undefined) { throw new Error('写入内容无法被序列化为 JSON'); } return json; } /** * JSON 序列化时处理 BigInt。 * @param {string} key 属性名。 * @param {*} value 属性值。 * @return {*} 序列化后的值。 */ jsonReplacer(key, value) { if (typeof value === 'bigint') { return value.toString(); } return value; } /** * 规范化目录片段,避免路径穿越。 * @param {*} value 待处理值。 * @return {string} 安全片段。 */ sanitizeSegment(value) { const normalized = String(value || '').trim(); if (!normalized) { return 'default'; } const safeValue = Array.from(normalized).map(character => { const code = character.charCodeAt(0); if (code < 32 || '<>:"/\\|?*'.includes(character)) { return '_'; } return character; }).join(''); return safeValue .replace(/\s+/g, '_') .replace(/\.+/g, '.') .replace(/^\.+|\.+$/g, '') .replace(/_+/g, '_') .slice(0, 80) || 'default'; } /** * 规范化扩展名。 * @param {*} value 扩展名。 * @param {string} fallback 回退扩展名。 * @return {string} 规范化后的扩展名。 */ normalizeExtension(value, fallback) { const raw = String(value || fallback || '').trim(); if (!raw) { return fallback; } return raw.startsWith('.') ? raw : `.${raw}`; } /** * 规范化数字。 * @param {*} value 数值或字符串。 * @param {number} fallback 回退值。 * @return {number} 规范化后的整数。 */ normalizeInteger(value, fallback) { const parsed = Number.parseInt(value, 10); return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; } /** * 规范化大小配置。 * @param {*} value 大小配置。 * @param {number} fallback 回退值。 * @return {number} 字节数。 */ normalizeSize(value, fallback) { if (typeof value === 'number' && Number.isFinite(value) && value > 0) { return Math.floor(value); } if (typeof value === 'string') { const trimmed = value.trim(); const match = trimmed.match(/^(\d+(?:\.\d+)?)([kKmMgGtT]?)$/); if (match) { const number = Number.parseFloat(match[1]); const unit = match[2].toLowerCase(); const unitMap = { '': 1, k: 1024, m: 1024 * 1024, g: 1024 * 1024 * 1024, t: 1024 * 1024 * 1024 * 1024, }; return Math.floor(number * (unitMap[unit] || 1)); } } return fallback; } /** * 规范化正整数。 * @param {*} value 数值或字符串。 * @param {number} fallback 回退值。 * @return {number} 规范化后的正整数。 */ normalizePositiveInteger(value, fallback) { const parsed = Number.parseInt(value, 10); return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } /** * 解析时间值并返回毫秒时间戳。 * @param {Date|string|number} [value] 时间值。 * @return {number} 毫秒时间戳。 */ resolveTimestamp(value) { if (value == null) { return 0; } if (value instanceof Date) { return value.getTime(); } if (typeof value === 'number' && Number.isFinite(value)) { return value; } if (typeof value === 'string') { const trimmed = value.trim(); if (!trimmed) { return 0; } if (/^\d+$/.test(trimmed)) { return Number.parseInt(trimmed, 10); } const parsed = Date.parse(trimmed); if (!Number.isNaN(parsed)) { return parsed; } } return 0; } /** * 把毫秒时间戳转换为 ISO 字符串。 * @param {number} timestamp 毫秒时间戳。 * @return {string|null} ISO 字符串。 */ formatTimestamp(timestamp) { if (!Number.isFinite(timestamp) || timestamp <= 0) { return null; } return new Date(timestamp).toISOString(); } /** * 获取本次消费限额。 * @param {Object} [options] 消费选项。 * @param {number} [options.maxConsumeRecordsPerWindow] 单次最大消费条数。 * @param {number} [options.maxConsumeRecordsPerRun] 兼容旧字段的单次最大消费条数。 * @return {number} 当前消费限额。 */ getConsumeLimit(options = {}) { return this.normalizePositiveInteger( options.maxConsumeRecordsPerWindow ?? options.maxConsumeRecordsPerRun, this.maxConsumeRecordsPerWindow ); } /** * 解析消费起始偏移量。 * @param {Object|null} progress 进度数据。 * @param {string} filePath 当前文件路径。 * @param {Object} [options] 消费选项。 * @param {number} [options.startOffset] 显式起始偏移量。 * @return {number} 起始偏移量。 */ resolveConsumeStartOffset(progress, filePath, options = {}) { if (options.startOffset != null) { return this.normalizeInteger(options.startOffset, 0); } if (progress && progress.filePath && path.resolve(progress.filePath) === path.resolve(filePath)) { return this.normalizeInteger(progress.offset, 0); } return 0; } /** * 获取某个桶的共享写入窗口计数器文件路径。 * @param {number} bucketIndex 桶索引。 * @return {string} 计数器文件路径。 */ getWriteWindowCounterPath(bucketIndex = 0) { return path.join(this.baseDir, `.write-window-counter-${bucketIndex}.json`); } /** * 多桶分片原子限流。 * 将总限额均分到 N 个桶,从首选桶开始尝试,竞争失败或桶满时自动漂移到下一个桶。 * @return {Promise} { allowed: boolean, counter: { startedAt, recordCount }, source: 'file' }。 */ async atomicCheckAndIncrementWriteWindow() { if (this.maxWriteRecordsPerWindow <= 0) { return { allowed: true, counter: { startedAt: 0, recordCount: 0 } }; } const bucketCount = this.writeWindowBucketCount; const perBucketLimit = Math.ceil(this.maxWriteRecordsPerWindow / bucketCount); const writeWindowMs = this.writeWindowSeconds * 1000; const now = Date.now(); await this.ensureParentDirectory(this.baseDir); let lastCounter = null; for (let i = 0; i < bucketCount; i++) { const bucketIndex = (this.preferredBucket + i) % bucketCount; const counterPath = this.getWriteWindowCounterPath(bucketIndex); const lockPath = `${counterPath}.lock`; let lockHandle = null; try { // eslint-disable-next-line no-bitwise lockHandle = await fsp.open(lockPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY); } catch (error) { if (error.code === 'EEXIST' || error.code === 'EPERM') { continue; } throw error; } try { const counter = { startedAt: 0, recordCount: 0 }; try { const content = await fsp.readFile(counterPath, 'utf8'); if (content.trim()) { const data = JSON.parse(content); counter.startedAt = typeof data.startedAt === 'number' && Number.isFinite(data.startedAt) ? data.startedAt : 0; counter.recordCount = typeof data.recordCount === 'number' && Number.isFinite(data.recordCount) ? data.recordCount : 0; } } catch (error) { if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) { throw error; } } if (counter.startedAt > 0 && now - counter.startedAt >= writeWindowMs) { counter.startedAt = now; counter.recordCount = 0; } if (counter.startedAt === 0) { counter.startedAt = now; } lastCounter = counter; if (counter.recordCount >= perBucketLimit) { continue; } counter.recordCount += 1; await fsp.writeFile(counterPath, `${JSON.stringify(counter)}\n`, 'utf8'); return { allowed: true, counter, source: 'file' }; } finally { await lockHandle.close(); await fsp.rm(lockPath, { force: true }); } } return { allowed: false, counter: lastCounter || { startedAt: now, recordCount: 0 }, source: 'file' }; } /** * 获取当前写入窗口状态(汇总所有桶)。 * @return {Promise} 写入窗口状态。 */ async getWriteWindowState() { const writeWindowMs = this.writeWindowSeconds * 1000; const now = Date.now(); let totalRecordCount = 0; let earliestStartedAt = 0; for (let i = 0; i < this.writeWindowBucketCount; i++) { try { const content = await fsp.readFile(this.getWriteWindowCounterPath(i), 'utf8'); if (content.trim()) { const data = JSON.parse(content); const startedAt = typeof data.startedAt === 'number' && Number.isFinite(data.startedAt) ? data.startedAt : 0; const recordCount = typeof data.recordCount === 'number' && Number.isFinite(data.recordCount) ? data.recordCount : 0; if (startedAt > 0 && now - startedAt < writeWindowMs) { totalRecordCount += recordCount; if (earliestStartedAt === 0 || startedAt < earliestStartedAt) { earliestStartedAt = startedAt; } } } } catch { /* ignore missing/corrupt counter files */ } } const windowExpired = earliestStartedAt > 0 && now - earliestStartedAt >= writeWindowMs; const resumeAt = earliestStartedAt > 0 ? new Date(earliestStartedAt + writeWindowMs).toISOString() : null; return { writeWindowSeconds: this.writeWindowSeconds, maxWriteRecordsPerWindow: this.maxWriteRecordsPerWindow, writeWindowBucketCount: this.writeWindowBucketCount, writeWindowRecordCount: totalRecordCount, writeWindowStartedAt: earliestStartedAt > 0 ? new Date(earliestStartedAt).toISOString() : null, windowExpired, resumeAt: windowExpired ? null : resumeAt, source: 'file', }; } /** * 将字符串映射到 [0, max) 范围内的索引。 * @param {string} str 输入字符串。 * @param {number} max 范围上限。 * @return {number} 索引。 */ hashStringToIndex(str, max) { let hash = 0; for (let i = 0; i < str.length; i++) { // eslint-disable-next-line no-bitwise hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0; } return ((hash % max) + max) % max; } /** * 确保目录存在。 * @param {string} dirPath 目录路径。 * @return {Promise} Promise。 */ async ensureParentDirectory(dirPath) { await fsp.mkdir(dirPath, { recursive: true }); } /** * 判断是否需要执行清理。 * @return {Promise} Promise。 */ async maybeCleanup() { if (!this.cleanupIntervalMs || this.cleanupIntervalMs <= 0) { return; } const now = Date.now(); if (now - this.lastCleanupAt < this.cleanupIntervalMs) { return; } this.lastCleanupAt = now; try { await this.cleanupExpiredFiles(); } catch (error) { this.app.logger.warn('[fileWriter.cleanup] 清理本地文件失败', error.message); } } /** * 递归清理目录中的过期文件。 * @param {string} dirPath 当前目录。 * @param {number} cutoffTime 过期时间戳。 * @param {Object} result 清理统计。 * @return {Promise} Promise。 */ async walkAndCleanup(dirPath, cutoffTime, result) { let entries; try { entries = await fsp.readdir(dirPath, { withFileTypes: true }); } catch (error) { if (error.code === 'ENOENT') { return; } throw error; } for (const entry of entries) { const entryPath = path.join(dirPath, entry.name); if (entry.isDirectory()) { await this.walkAndCleanup(entryPath, cutoffTime, result); await this.removeEmptyDirectory(entryPath, result); continue; } if (!entry.isFile()) { continue; } if (!entry.name.endsWith(this.fileExtension)) { // 【风险点】清理只覆盖数据文件,.progress 文件不会被清理,长期会残留陈旧进度文件。 continue; } const completion = await this.loadCompletionMarker(entryPath); if (!completion || path.resolve(completion.filePath || entryPath) !== path.resolve(entryPath)) { continue; } const stat = await fsp.stat(entryPath).catch(error => { if (error.code === 'ENOENT') { return null; } throw error; }); if (!stat || stat.mtimeMs >= cutoffTime) { continue; } await fsp.rm(entryPath, { force: true }); await fsp.rm(this.getDoneFilePath(entryPath), { force: true }); result.removedFiles += 1; const siblingEntries = await fsp.readdir(dirPath, { withFileTypes: true }); for (const siblingEntry of siblingEntries) { if (!siblingEntry.isFile() || !siblingEntry.name.endsWith(this.progressExtension)) { continue; } const progressPath = path.join(dirPath, siblingEntry.name); const progressData = await this.loadProgress(progressPath); if (!progressData || !progressData.completedAt) { continue; } if (path.resolve(progressData.filePath || '') !== path.resolve(entryPath)) { continue; } await fsp.rm(progressPath, { force: true }); } } } /** * 原子写入文本文件。 * @param {string} targetPath 目标文件路径。 * @param {string} content 文件内容。 * @return {Promise} Promise。 */ async writeAtomicTextFile(targetPath, content) { const directory = path.dirname(targetPath); const tempPath = `${targetPath}.${this.pid}.${Date.now()}.tmp`; await this.ensureParentDirectory(directory); const handle = await fsp.open(tempPath, 'w'); try { await handle.write(content); await handle.sync(); } finally { await handle.close(); } await fsp.rename(tempPath, targetPath); } /** * 删除空目录。 * @param {string} dirPath 目录路径。 * @param {Object} result 清理统计。 * @return {Promise} Promise。 */ async removeEmptyDirectory(dirPath, result) { try { const entries = await fsp.readdir(dirPath); if (entries.length > 0) { return; } await fsp.rmdir(dirPath); result.removedDirectories += 1; } catch (error) { if (error.code === 'ENOENT' || error.code === 'ENOTEMPTY') { return; } throw error; } } } module.exports = LocalFileWriter;