'use strict'; const assert = require('node:assert/strict'); const fs = require('node:fs'); const fsp = require('node:fs/promises'); const os = require('node:os'); const path = require('node:path'); const LocalFileWriter = require('../../../app/lib/file_writer'); function createWriter(baseDir, fileWriterOverrides = {}) { return new LocalFileWriter({ baseDir: process.cwd(), config: { fileWriter: { baseDir, maxFileSize: 10485760, maxFileLines: 2000, maxConsumeRecordsPerWindow: 1000, maxConsumeRecordsPerRun: 1000, writeWindowSeconds: 1, maxWriteRecordsPerWindow: 2000, writeWindowBucketCount: 1, retentionDays: 1, cleanupIntervalMs: 0, sequenceWidth: 2, machineId: 'test-machine', workerId: 'test-worker', fileExtension: '.log', progressExtension: '.progress', doneExtension: '.done', ...fileWriterOverrides, }, }, logger: { warn() {}, info() {}, error() {}, debug() {}, }, }); } function touchOld(filePath, daysAgo = 2) { const time = new Date(Date.now() - daysAgo * 24 * 60 * 60 * 1000); fs.utimesSync(filePath, time, time); } describe('test/app/lib/file_writer.test.js', () => { let baseDir = ''; beforeEach(() => { baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wms-file-writer-')); }); afterEach(() => { fs.rmSync(baseDir, { recursive: true, force: true }); }); it('should keep unfinished data files and progress files during cleanup', async () => { const writer = createWriter(baseDir); const archiveDate = new Date(2026, 3, 9); const partitionInfo = writer.getPartitionInfo('orders', archiveDate); const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log'); const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer'); await fsp.mkdir(partitionInfo.partitionDir, { recursive: true }); await fsp.writeFile(dataFilePath, '{"id":1}\n', 'utf8'); await writer.saveProgress(progressFilePath, { filePath: dataFilePath, offset: 0, processedCount: 0, }); touchOld(dataFilePath); touchOld(progressFilePath); const result = await writer.cleanupExpiredFiles(); assert.equal(result.removedFiles, 0); assert.equal(fs.existsSync(dataFilePath), true); assert.equal(fs.existsSync(progressFilePath), true); assert.equal(fs.existsSync(writer.getDoneFilePath(dataFilePath)), false); }); it('should rotate to a new file after reaching the line threshold within the same writer instance', async () => { const writer = createWriter(baseDir, { maxFileLines: 2, maxFileSize: 10485760, }); const archiveDate = new Date(2026, 3, 9); const firstResult = await writer.writeRecord('orders', { id: 1 }, { date: archiveDate }); const secondResult = await writer.writeRecord('orders', { id: 2 }, { date: archiveDate }); const thirdResult = await writer.writeRecord('orders', { id: 3 }, { date: archiveDate }); assert.equal(firstResult.sequence, 1); assert.equal(firstResult.lineCount, 1); assert.equal(secondResult.sequence, 1); assert.equal(secondResult.lineCount, 2); assert.equal(thirdResult.sequence, 2); assert.equal(thirdResult.lineCount, 1); const dataFiles = await writer.listDataFiles('orders', archiveDate); assert.deepEqual(dataFiles, [ path.resolve(firstResult.filePath), path.resolve(thirdResult.filePath), ]); const firstFileRecords = await writer.readRecords(firstResult.filePath); const secondFileRecords = await writer.readRecords(thirdResult.filePath); assert.equal(firstFileRecords.records.length, 2); assert.equal(secondFileRecords.records.length, 1); }); it('should restore the current line count from disk after writer restart and continue rotating correctly', async () => { const archiveDate = new Date(2026, 3, 9); const firstWriter = createWriter(baseDir, { maxFileLines: 2, maxFileSize: 10485760, }); await firstWriter.writeRecord('orders', { id: 1 }, { date: archiveDate }); await firstWriter.writeRecord('orders', { id: 2 }, { date: archiveDate }); const restartedWriter = createWriter(baseDir, { maxFileLines: 2, maxFileSize: 10485760, }); const restoredState = await restartedWriter.getPartitionState('orders', archiveDate); const restoredFilePath = restoredState.currentFilePath; assert.equal(restoredState.sequence, 1); assert.equal(restoredState.currentLineCount, 2); assert.equal(path.basename(restoredState.currentFilePath).endsWith('-01.log'), true); const thirdResult = await restartedWriter.writeRecord('orders', { id: 3 }, { date: archiveDate }); assert.equal(thirdResult.sequence, 2); assert.equal(thirdResult.lineCount, 1); const firstFileRecords = await restartedWriter.readRecords(restoredFilePath); const secondFileRecords = await restartedWriter.readRecords(thirdResult.filePath); assert.equal(firstFileRecords.records.length, 2); assert.equal(secondFileRecords.records.length, 1); }); it('should mark completed files, skip them on restart, and clean them up later', async () => { const writer = createWriter(baseDir); const archiveDate = new Date(2026, 3, 9); const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer'); const writeResult = await writer.writeRecord('orders', { id: 1, name: 'completed-record', }, { date: archiveDate, }); let replayedCount = 0; const consumeResult = await writer.consumeFile(writeResult.filePath, { progressFilePath, onRecord: async () => { replayedCount += 1; }, }); assert.equal(consumeResult.processedCount, 1); assert.equal(replayedCount, 1); assert.equal(fs.existsSync(writer.getDoneFilePath(writeResult.filePath)), true); const replayCount = await writer.consumeFiles({ businessName: 'orders', date: archiveDate, progressFilePath, onRecord: async () => { replayedCount += 1; }, }); assert.equal(replayCount.processedCount, 0); assert.equal(replayedCount, 1); touchOld(writeResult.filePath); touchOld(writer.getDoneFilePath(writeResult.filePath)); const cleanupResult = await writer.cleanupExpiredFiles(); assert.equal(cleanupResult.removedFiles, 1); assert.equal(fs.existsSync(writeResult.filePath), false); assert.equal(fs.existsSync(writer.getDoneFilePath(writeResult.filePath)), false); assert.equal(fs.existsSync(progressFilePath), false); }); it('should keep the size-based rollover behavior unchanged', async () => { const writer = createWriter(baseDir, { maxFileSize: 1, maxFileLines: 2000, }); const archiveDate = new Date(2026, 3, 9); const firstResult = await writer.writeRecord('orders', { id: 1, name: 'record-1' }, { date: archiveDate }); const secondResult = await writer.writeRecord('orders', { id: 2, name: 'record-2' }, { date: archiveDate }); assert.equal(firstResult.sequence, 1); assert.equal(secondResult.sequence, 2); assert.equal(firstResult.lineCount, 1); assert.equal(secondResult.lineCount, 1); const dataFiles = await writer.listDataFiles('orders', archiveDate); assert.deepEqual(dataFiles, [ path.resolve(firstResult.filePath), path.resolve(secondResult.filePath), ]); }); it('should keep consuming across files and partitions until the shared limit is reached', async () => { const writer = createWriter(baseDir, { maxConsumeRecordsPerWindow: 4, }); const businessName = 'orders'; const firstArchiveDate = new Date(2026, 3, 9); const secondArchiveDate = new Date(2026, 3, 10); const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer'); const firstPartition = writer.getPartitionInfo(businessName, firstArchiveDate); const secondPartition = writer.getPartitionInfo(businessName, secondArchiveDate); const firstFilePath = path.join(firstPartition.partitionDir, 'manual-01.log'); const secondFilePath = path.join(secondPartition.partitionDir, 'manual-01.log'); const consumedRecords = []; await fsp.mkdir(firstPartition.partitionDir, { recursive: true }); await fsp.mkdir(secondPartition.partitionDir, { recursive: true }); await fsp.writeFile(firstFilePath, JSON.stringify({ id: 1, name: 'record-1' }), 'utf8'); await fsp.writeFile(secondFilePath, [ JSON.stringify({ id: 2, name: 'record-2' }), JSON.stringify({ id: 3, name: 'record-3' }), JSON.stringify({ id: 4, name: 'record-4' }), JSON.stringify({ id: 5, name: 'record-5' }), ].join('\n'), 'utf8'); const consumeResult = await writer.consumeFiles({ businessName, date: null, progressFilePath, now: Date.UTC(2026, 3, 9, 10, 0, 0), onRecord: async record => { consumedRecords.push(record.name); }, }); assert.equal(consumeResult.processedCount, 4); assert.equal(consumeResult.limitReached, true); assert.deepEqual(consumedRecords, [ 'record-1', 'record-2', 'record-3', 'record-4', ]); assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), true); assert.equal(fs.existsSync(writer.getDoneFilePath(secondFilePath)), false); assert.equal(consumeResult.progress.filePath, path.resolve(secondFilePath)); }); it('should throttle writes when write window record limit is reached', async () => { const writer = createWriter(baseDir, { maxWriteRecordsPerWindow: 3, writeWindowSeconds: 1, }); const archiveDate = new Date(2026, 3, 9); const firstResult = await writer.writeRecord('orders', { id: 1 }, { date: archiveDate }); assert.equal(firstResult.throttled, false); assert.equal(firstResult.writeWindowRecordCount, 1); const secondResult = await writer.writeRecord('orders', { id: 2 }, { date: archiveDate }); assert.equal(secondResult.throttled, false); assert.equal(secondResult.writeWindowRecordCount, 2); const thirdResult = await writer.writeRecord('orders', { id: 3 }, { date: archiveDate }); assert.equal(thirdResult.throttled, false); assert.equal(thirdResult.writeWindowRecordCount, 3); const throttledResult = await writer.writeRecord('orders', { id: 4 }, { date: archiveDate }); assert.equal(throttledResult.throttled, true); assert.equal(throttledResult.writeWindowRecordCount, 3); assert.equal(throttledResult.maxWriteRecordsPerWindow, 3); assert.ok(throttledResult.resumeAt); assert.equal(throttledResult.filePath, undefined); const readResult = await writer.readRecords(thirdResult.filePath); assert.equal(readResult.records.length, 3); }); it('should consume with a simple per-call limit without time-based windowing', async () => { const writer = createWriter(baseDir, { maxConsumeRecordsPerWindow: 2, }); const archiveDate = new Date(2026, 3, 9); const partitionInfo = writer.getPartitionInfo('orders', archiveDate); const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer'); const consumedRecords = []; const firstFilePath = writer.buildDataFilePath(partitionInfo, 1); await fsp.mkdir(partitionInfo.partitionDir, { recursive: true }); await fsp.writeFile(firstFilePath, [ JSON.stringify({ id: 1, name: 'record-1' }), JSON.stringify({ id: 2, name: 'record-2' }), JSON.stringify({ id: 3, name: 'record-3' }), ].join('\n'), 'utf8'); const firstConsumeResult = await writer.consumeFiles({ businessName: 'orders', date: archiveDate, progressFilePath, onRecord: async record => { consumedRecords.push(record.name); }, }); assert.equal(firstConsumeResult.processedCount, 2); assert.equal(firstConsumeResult.limitReached, true); assert.equal(firstConsumeResult.maxConsumeRecordsPerRun, 2); assert.equal(consumedRecords.length, 2); assert.deepEqual(consumedRecords, [ 'record-1', 'record-2' ]); assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), false); const secondConsumeResult = await writer.consumeFiles({ businessName: 'orders', date: archiveDate, progressFilePath, onRecord: async record => { consumedRecords.push(record.name); }, }); assert.equal(secondConsumeResult.processedCount, 1); assert.equal(secondConsumeResult.limitReached, false); assert.equal(secondConsumeResult.maxConsumeRecordsPerRun, 2); assert.deepEqual(consumedRecords, [ 'record-1', 'record-2', 'record-3' ]); assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), true); assert.equal(fs.existsSync(progressFilePath), true); }); it('should consume files across multiple date partitions in write order without a date filter', async () => { const writer = createWriter(baseDir); const businessName = 'orders'; const firstArchiveDate = new Date(2026, 3, 10); const secondArchiveDate = new Date(2026, 3, 9); const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer'); const consumedRecords = []; try { const firstWriteResult = await writer.writeRecord(businessName, { id: 1, name: 'record-1', }, { date: firstArchiveDate, }); const secondWriteResult = await writer.writeRecord(businessName, { id: 2, name: 'record-2', }, { date: secondArchiveDate, }); fs.utimesSync(firstWriteResult.filePath, new Date('2026-04-09T08:00:00.000Z'), new Date('2026-04-09T08:00:00.000Z')); fs.utimesSync(secondWriteResult.filePath, new Date('2026-04-09T08:00:01.000Z'), new Date('2026-04-09T08:00:01.000Z')); const consumeResult = await writer.consumeFiles({ businessName, progressFilePath, onRecord: async record => { consumedRecords.push(record.name); }, }); assert.equal(consumeResult.processedCount, 2); assert.equal(consumeResult.limitReached, false); assert.equal(Array.isArray(consumeResult.dataFiles), true); assert.equal(consumeResult.dataFiles.length, 2); assert.deepEqual(consumedRecords, [ 'record-2', 'record-1' ]); assert.deepEqual(consumeResult.dataFiles, [ path.resolve(secondWriteResult.filePath), path.resolve(firstWriteResult.filePath), ]); assert.equal(consumeResult.progress.filePath, path.resolve(firstWriteResult.filePath)); assert.equal(fs.existsSync(writer.getDoneFilePath(firstWriteResult.filePath)), true); assert.equal(fs.existsSync(writer.getDoneFilePath(secondWriteResult.filePath)), true); } finally { fs.rmSync(path.join(baseDir, writer.sanitizeSegment(businessName)), { recursive: true, force: true }); } }); it('should sort files within the same date partition by sequence instead of mtime', async () => { const writer = createWriter(baseDir, { maxFileSize: 1, }); const businessName = 'orders'; const archiveDate = new Date(2026, 3, 9); const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer'); try { const firstWriteResult = await writer.writeRecord(businessName, { id: 1, name: 'record-1', }, { date: archiveDate, }); const secondWriteResult = await writer.writeRecord(businessName, { id: 2, name: 'record-2', }, { date: archiveDate, }); fs.utimesSync(firstWriteResult.filePath, new Date('2026-04-09T08:00:01.000Z'), new Date('2026-04-09T08:00:01.000Z')); fs.utimesSync(secondWriteResult.filePath, new Date('2026-04-09T08:00:00.000Z'), new Date('2026-04-09T08:00:00.000Z')); const dataFiles = await writer.listDataFiles(businessName, null); const consumedRecords = []; const consumeResult = await writer.consumeFiles({ businessName, progressFilePath, onRecord: async record => { consumedRecords.push(record.name); }, }); assert.deepEqual(dataFiles, [ path.resolve(firstWriteResult.filePath), path.resolve(secondWriteResult.filePath), ]); assert.deepEqual(consumedRecords, [ 'record-1', 'record-2' ]); assert.equal(consumeResult.progress.filePath, path.resolve(secondWriteResult.filePath)); } finally { fs.rmSync(path.join(baseDir, writer.sanitizeSegment(businessName)), { recursive: true, force: true }); } }); it('should keep exact offsets for CRLF files without a trailing newline', async () => { const writer = createWriter(baseDir); const archiveDate = new Date(2026, 3, 9); const partitionInfo = writer.getPartitionInfo('orders', archiveDate); const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log'); const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer'); const content = [ JSON.stringify({ id: 1, name: 'record-1' }), JSON.stringify({ id: 2, name: 'record-2' }), ].join('\r\n'); const expectedOffset = Buffer.byteLength(content); const consumedRecords = []; await fsp.mkdir(partitionInfo.partitionDir, { recursive: true }); await fsp.writeFile(dataFilePath, content, 'utf8'); const readResult = await writer.readRecords(dataFilePath); assert.equal(readResult.nextOffset, expectedOffset); assert.deepEqual(readResult.records.map(record => record.id), [ 1, 2 ]); const consumeResult = await writer.consumeFile(dataFilePath, { progressFilePath, onRecord: async record => { consumedRecords.push(record.id); }, }); assert.equal(consumeResult.nextOffset, expectedOffset); assert.equal(consumeResult.processedCount, 2); assert.deepEqual(consumedRecords, [ 1, 2 ]); }); it('should ignore corrupt progress and completion markers and keep consuming', async () => { const writer = createWriter(baseDir); const archiveDate = new Date(2026, 3, 9); const partitionInfo = writer.getPartitionInfo('orders', archiveDate); const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log'); const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer'); const completionFilePath = writer.getDoneFilePath(dataFilePath); const consumedRecords = []; await fsp.mkdir(partitionInfo.partitionDir, { recursive: true }); await fsp.writeFile(dataFilePath, [ JSON.stringify({ id: 1, name: 'record-1' }), JSON.stringify({ id: 2, name: 'record-2' }), ].join('\n'), 'utf8'); await fsp.writeFile(progressFilePath, '{"filePath":123', 'utf8'); await fsp.writeFile(completionFilePath, '{"filePath":123', 'utf8'); const loadedProgress = await writer.loadProgress(progressFilePath); const loadedCompletion = await writer.loadCompletionMarker(dataFilePath); assert.equal(loadedProgress, null); assert.equal(loadedCompletion, null); const consumeResult = await writer.consumeFiles({ businessName: 'orders', date: archiveDate, progressFilePath, onRecord: async record => { consumedRecords.push(record.id); }, }); assert.equal(consumeResult.processedCount, 2); assert.deepEqual(consumedRecords, [ 1, 2 ]); assert.equal(fs.existsSync(writer.getDoneFilePath(dataFilePath)), true); }); it('should rethrow filesystem errors other than missing files when loading progress markers', async () => { const writer = createWriter(baseDir); const originalReadFile = fsp.readFile; const error = new Error('permission denied'); error.code = 'EACCES'; try { fsp.readFile = async () => { throw error; }; await assert.rejects( () => writer.loadProgress(path.join(baseDir, 'orders.progress')), thrownError => thrownError.code === 'EACCES' ); await assert.rejects( () => writer.loadCompletionMarker(path.join(baseDir, 'orders.log')), thrownError => thrownError.code === 'EACCES' ); } finally { fsp.readFile = originalReadFile; } }); it('should recycle old partition states when the cache reaches its limit', async () => { const writer = createWriter(baseDir, { stateMapMaxEntries: 2, }); const firstState = await writer.getPartitionState('orders-a', new Date(2026, 3, 9)); const secondState = await writer.getPartitionState('orders-b', new Date(2026, 3, 9)); const firstStateKey = `${writer.sanitizeSegment(firstState.businessSegment)}:${firstState.dateKey}`; const secondStateKey = `${writer.sanitizeSegment(secondState.businessSegment)}:${secondState.dateKey}`; assert.equal(writer.stateMap.size, 2); assert.equal(writer.stateMap.has(firstStateKey), true); assert.equal(writer.stateMap.has(secondStateKey), true); const thirdState = await writer.getPartitionState('orders-c', new Date(2026, 3, 9)); const thirdStateKey = `${writer.sanitizeSegment(thirdState.businessSegment)}:${thirdState.dateKey}`; assert.equal(writer.stateMap.size, 2); assert.equal(writer.stateMap.has(firstStateKey), false); assert.equal(writer.stateMap.has(secondStateKey), true); assert.equal(writer.stateMap.has(thirdStateKey), true); const firstStateReloaded = await writer.getPartitionState('orders-a', new Date(2026, 3, 9)); const firstStateReloadedKey = `${writer.sanitizeSegment(firstStateReloaded.businessSegment)}:${firstStateReloaded.dateKey}`; assert.equal(writer.stateMap.size, 2); assert.equal(writer.stateMap.has(firstStateReloadedKey), true); }); it('should limit records when maxRecords is specified', async () => { const writer = createWriter(baseDir, { maxFileSize: 10485760, }); for (let i = 0; i < 10; i += 1) { await writer.writeRecord('batch-read', { index: i }); } const files = await writer.listDataFiles('batch-read', null); assert(files.length > 0); const res1 = await writer.readRecords(files[0], { maxRecords: 3 }); assert.equal(res1.records.length, 3); assert.equal(res1.records[0].index, 0); assert.equal(res1.records[2].index, 2); const res2 = await writer.readRecords(files[0], { startOffset: res1.nextOffset, maxRecords: 3 }); assert.equal(res2.records.length, 3); assert.equal(res2.records[0].index, 3); const res3 = await writer.readRecords(files[0], { startOffset: res2.nextOffset, maxRecords: 100 }); assert.equal(res3.records.length, 4); assert.equal(res3.records[3].index, 9); const resAll = await writer.readRecords(files[0]); assert.equal(resAll.records.length, 10); }); });