WMS_MAIN/test/app/lib/file_writer.test.js
2026-04-27 16:09:26 +08:00

596 lines
22 KiB
JavaScript

'use strict';
const assert = require('node:assert/strict');
const fs = require('node:fs');
const fsp = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const LocalFileWriter = require('../../../app/lib/file_writer');
function createWriter(baseDir, fileWriterOverrides = {}) {
return new LocalFileWriter({
baseDir: process.cwd(),
config: {
fileWriter: {
baseDir,
maxFileSize: 10485760,
maxFileLines: 2000,
maxConsumeRecordsPerWindow: 1000,
maxConsumeRecordsPerRun: 1000,
writeWindowSeconds: 1,
maxWriteRecordsPerWindow: 2000,
writeWindowBucketCount: 1,
retentionDays: 1,
cleanupIntervalMs: 0,
sequenceWidth: 2,
machineId: 'test-machine',
workerId: 'test-worker',
fileExtension: '.log',
progressExtension: '.progress',
doneExtension: '.done',
...fileWriterOverrides,
},
},
logger: {
warn() {},
info() {},
error() {},
debug() {},
},
});
}
function touchOld(filePath, daysAgo = 2) {
const time = new Date(Date.now() - daysAgo * 24 * 60 * 60 * 1000);
fs.utimesSync(filePath, time, time);
}
describe('test/app/lib/file_writer.test.js', () => {
let baseDir = '';
beforeEach(() => {
baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wms-file-writer-'));
});
afterEach(() => {
fs.rmSync(baseDir, { recursive: true, force: true });
});
it('should keep unfinished data files and progress files during cleanup', async () => {
const writer = createWriter(baseDir);
const archiveDate = new Date(2026, 3, 9);
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log');
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
await fsp.writeFile(dataFilePath, '{"id":1}\n', 'utf8');
await writer.saveProgress(progressFilePath, {
filePath: dataFilePath,
offset: 0,
processedCount: 0,
});
touchOld(dataFilePath);
touchOld(progressFilePath);
const result = await writer.cleanupExpiredFiles();
assert.equal(result.removedFiles, 0);
assert.equal(fs.existsSync(dataFilePath), true);
assert.equal(fs.existsSync(progressFilePath), true);
assert.equal(fs.existsSync(writer.getDoneFilePath(dataFilePath)), false);
});
it('should rotate to a new file after reaching the line threshold within the same writer instance', async () => {
const writer = createWriter(baseDir, {
maxFileLines: 2,
maxFileSize: 10485760,
});
const archiveDate = new Date(2026, 3, 9);
const firstResult = await writer.writeRecord('orders', { id: 1 }, { date: archiveDate });
const secondResult = await writer.writeRecord('orders', { id: 2 }, { date: archiveDate });
const thirdResult = await writer.writeRecord('orders', { id: 3 }, { date: archiveDate });
assert.equal(firstResult.sequence, 1);
assert.equal(firstResult.lineCount, 1);
assert.equal(secondResult.sequence, 1);
assert.equal(secondResult.lineCount, 2);
assert.equal(thirdResult.sequence, 2);
assert.equal(thirdResult.lineCount, 1);
const dataFiles = await writer.listDataFiles('orders', archiveDate);
assert.deepEqual(dataFiles, [
path.resolve(firstResult.filePath),
path.resolve(thirdResult.filePath),
]);
const firstFileRecords = await writer.readRecords(firstResult.filePath);
const secondFileRecords = await writer.readRecords(thirdResult.filePath);
assert.equal(firstFileRecords.records.length, 2);
assert.equal(secondFileRecords.records.length, 1);
});
it('should restore the current line count from disk after writer restart and continue rotating correctly', async () => {
const archiveDate = new Date(2026, 3, 9);
const firstWriter = createWriter(baseDir, {
maxFileLines: 2,
maxFileSize: 10485760,
});
await firstWriter.writeRecord('orders', { id: 1 }, { date: archiveDate });
await firstWriter.writeRecord('orders', { id: 2 }, { date: archiveDate });
const restartedWriter = createWriter(baseDir, {
maxFileLines: 2,
maxFileSize: 10485760,
});
const restoredState = await restartedWriter.getPartitionState('orders', archiveDate);
const restoredFilePath = restoredState.currentFilePath;
assert.equal(restoredState.sequence, 1);
assert.equal(restoredState.currentLineCount, 2);
assert.equal(path.basename(restoredState.currentFilePath).endsWith('-01.log'), true);
const thirdResult = await restartedWriter.writeRecord('orders', { id: 3 }, { date: archiveDate });
assert.equal(thirdResult.sequence, 2);
assert.equal(thirdResult.lineCount, 1);
const firstFileRecords = await restartedWriter.readRecords(restoredFilePath);
const secondFileRecords = await restartedWriter.readRecords(thirdResult.filePath);
assert.equal(firstFileRecords.records.length, 2);
assert.equal(secondFileRecords.records.length, 1);
});
it('should mark completed files, skip them on restart, and clean them up later', async () => {
const writer = createWriter(baseDir);
const archiveDate = new Date(2026, 3, 9);
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
const writeResult = await writer.writeRecord('orders', {
id: 1,
name: 'completed-record',
}, {
date: archiveDate,
});
let replayedCount = 0;
const consumeResult = await writer.consumeFile(writeResult.filePath, {
progressFilePath,
onRecord: async () => {
replayedCount += 1;
},
});
assert.equal(consumeResult.processedCount, 1);
assert.equal(replayedCount, 1);
assert.equal(fs.existsSync(writer.getDoneFilePath(writeResult.filePath)), true);
const replayCount = await writer.consumeFiles({
businessName: 'orders',
date: archiveDate,
progressFilePath,
onRecord: async () => {
replayedCount += 1;
},
});
assert.equal(replayCount.processedCount, 0);
assert.equal(replayedCount, 1);
touchOld(writeResult.filePath);
touchOld(writer.getDoneFilePath(writeResult.filePath));
const cleanupResult = await writer.cleanupExpiredFiles();
assert.equal(cleanupResult.removedFiles, 1);
assert.equal(fs.existsSync(writeResult.filePath), false);
assert.equal(fs.existsSync(writer.getDoneFilePath(writeResult.filePath)), false);
assert.equal(fs.existsSync(progressFilePath), false);
});
it('should keep the size-based rollover behavior unchanged', async () => {
const writer = createWriter(baseDir, {
maxFileSize: 1,
maxFileLines: 2000,
});
const archiveDate = new Date(2026, 3, 9);
const firstResult = await writer.writeRecord('orders', { id: 1, name: 'record-1' }, { date: archiveDate });
const secondResult = await writer.writeRecord('orders', { id: 2, name: 'record-2' }, { date: archiveDate });
assert.equal(firstResult.sequence, 1);
assert.equal(secondResult.sequence, 2);
assert.equal(firstResult.lineCount, 1);
assert.equal(secondResult.lineCount, 1);
const dataFiles = await writer.listDataFiles('orders', archiveDate);
assert.deepEqual(dataFiles, [
path.resolve(firstResult.filePath),
path.resolve(secondResult.filePath),
]);
});
it('should keep consuming across files and partitions until the shared limit is reached', async () => {
const writer = createWriter(baseDir, {
maxConsumeRecordsPerWindow: 4,
});
const businessName = 'orders';
const firstArchiveDate = new Date(2026, 3, 9);
const secondArchiveDate = new Date(2026, 3, 10);
const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer');
const firstPartition = writer.getPartitionInfo(businessName, firstArchiveDate);
const secondPartition = writer.getPartitionInfo(businessName, secondArchiveDate);
const firstFilePath = path.join(firstPartition.partitionDir, 'manual-01.log');
const secondFilePath = path.join(secondPartition.partitionDir, 'manual-01.log');
const consumedRecords = [];
await fsp.mkdir(firstPartition.partitionDir, { recursive: true });
await fsp.mkdir(secondPartition.partitionDir, { recursive: true });
await fsp.writeFile(firstFilePath, JSON.stringify({ id: 1, name: 'record-1' }), 'utf8');
await fsp.writeFile(secondFilePath, [
JSON.stringify({ id: 2, name: 'record-2' }),
JSON.stringify({ id: 3, name: 'record-3' }),
JSON.stringify({ id: 4, name: 'record-4' }),
JSON.stringify({ id: 5, name: 'record-5' }),
].join('\n'), 'utf8');
const consumeResult = await writer.consumeFiles({
businessName,
date: null,
progressFilePath,
now: Date.UTC(2026, 3, 9, 10, 0, 0),
onRecord: async record => {
consumedRecords.push(record.name);
},
});
assert.equal(consumeResult.processedCount, 4);
assert.equal(consumeResult.limitReached, true);
assert.deepEqual(consumedRecords, [
'record-1',
'record-2',
'record-3',
'record-4',
]);
assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), true);
assert.equal(fs.existsSync(writer.getDoneFilePath(secondFilePath)), false);
assert.equal(consumeResult.progress.filePath, path.resolve(secondFilePath));
});
it('should throttle writes when write window record limit is reached', async () => {
const writer = createWriter(baseDir, {
maxWriteRecordsPerWindow: 3,
writeWindowSeconds: 1,
});
const archiveDate = new Date(2026, 3, 9);
const firstResult = await writer.writeRecord('orders', { id: 1 }, { date: archiveDate });
assert.equal(firstResult.throttled, false);
assert.equal(firstResult.writeWindowRecordCount, 1);
const secondResult = await writer.writeRecord('orders', { id: 2 }, { date: archiveDate });
assert.equal(secondResult.throttled, false);
assert.equal(secondResult.writeWindowRecordCount, 2);
const thirdResult = await writer.writeRecord('orders', { id: 3 }, { date: archiveDate });
assert.equal(thirdResult.throttled, false);
assert.equal(thirdResult.writeWindowRecordCount, 3);
const throttledResult = await writer.writeRecord('orders', { id: 4 }, { date: archiveDate });
assert.equal(throttledResult.throttled, true);
assert.equal(throttledResult.writeWindowRecordCount, 3);
assert.equal(throttledResult.maxWriteRecordsPerWindow, 3);
assert.ok(throttledResult.resumeAt);
assert.equal(throttledResult.filePath, undefined);
const readResult = await writer.readRecords(thirdResult.filePath);
assert.equal(readResult.records.length, 3);
});
it('should consume with a simple per-call limit without time-based windowing', async () => {
const writer = createWriter(baseDir, {
maxConsumeRecordsPerWindow: 2,
});
const archiveDate = new Date(2026, 3, 9);
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
const consumedRecords = [];
const firstFilePath = writer.buildDataFilePath(partitionInfo, 1);
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
await fsp.writeFile(firstFilePath, [
JSON.stringify({ id: 1, name: 'record-1' }),
JSON.stringify({ id: 2, name: 'record-2' }),
JSON.stringify({ id: 3, name: 'record-3' }),
].join('\n'), 'utf8');
const firstConsumeResult = await writer.consumeFiles({
businessName: 'orders',
date: archiveDate,
progressFilePath,
onRecord: async record => {
consumedRecords.push(record.name);
},
});
assert.equal(firstConsumeResult.processedCount, 2);
assert.equal(firstConsumeResult.limitReached, true);
assert.equal(firstConsumeResult.maxConsumeRecordsPerRun, 2);
assert.equal(consumedRecords.length, 2);
assert.deepEqual(consumedRecords, [ 'record-1', 'record-2' ]);
assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), false);
const secondConsumeResult = await writer.consumeFiles({
businessName: 'orders',
date: archiveDate,
progressFilePath,
onRecord: async record => {
consumedRecords.push(record.name);
},
});
assert.equal(secondConsumeResult.processedCount, 1);
assert.equal(secondConsumeResult.limitReached, false);
assert.equal(secondConsumeResult.maxConsumeRecordsPerRun, 2);
assert.deepEqual(consumedRecords, [ 'record-1', 'record-2', 'record-3' ]);
assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), true);
assert.equal(fs.existsSync(progressFilePath), true);
});
it('should consume files across multiple date partitions in write order without a date filter', async () => {
const writer = createWriter(baseDir);
const businessName = 'orders';
const firstArchiveDate = new Date(2026, 3, 10);
const secondArchiveDate = new Date(2026, 3, 9);
const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer');
const consumedRecords = [];
try {
const firstWriteResult = await writer.writeRecord(businessName, {
id: 1,
name: 'record-1',
}, {
date: firstArchiveDate,
});
const secondWriteResult = await writer.writeRecord(businessName, {
id: 2,
name: 'record-2',
}, {
date: secondArchiveDate,
});
fs.utimesSync(firstWriteResult.filePath, new Date('2026-04-09T08:00:00.000Z'), new Date('2026-04-09T08:00:00.000Z'));
fs.utimesSync(secondWriteResult.filePath, new Date('2026-04-09T08:00:01.000Z'), new Date('2026-04-09T08:00:01.000Z'));
const consumeResult = await writer.consumeFiles({
businessName,
progressFilePath,
onRecord: async record => {
consumedRecords.push(record.name);
},
});
assert.equal(consumeResult.processedCount, 2);
assert.equal(consumeResult.limitReached, false);
assert.equal(Array.isArray(consumeResult.dataFiles), true);
assert.equal(consumeResult.dataFiles.length, 2);
assert.deepEqual(consumedRecords, [ 'record-2', 'record-1' ]);
assert.deepEqual(consumeResult.dataFiles, [
path.resolve(secondWriteResult.filePath),
path.resolve(firstWriteResult.filePath),
]);
assert.equal(consumeResult.progress.filePath, path.resolve(firstWriteResult.filePath));
assert.equal(fs.existsSync(writer.getDoneFilePath(firstWriteResult.filePath)), true);
assert.equal(fs.existsSync(writer.getDoneFilePath(secondWriteResult.filePath)), true);
} finally {
fs.rmSync(path.join(baseDir, writer.sanitizeSegment(businessName)), { recursive: true, force: true });
}
});
it('should sort files within the same date partition by sequence instead of mtime', async () => {
const writer = createWriter(baseDir, {
maxFileSize: 1,
});
const businessName = 'orders';
const archiveDate = new Date(2026, 3, 9);
const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer');
try {
const firstWriteResult = await writer.writeRecord(businessName, {
id: 1,
name: 'record-1',
}, {
date: archiveDate,
});
const secondWriteResult = await writer.writeRecord(businessName, {
id: 2,
name: 'record-2',
}, {
date: archiveDate,
});
fs.utimesSync(firstWriteResult.filePath, new Date('2026-04-09T08:00:01.000Z'), new Date('2026-04-09T08:00:01.000Z'));
fs.utimesSync(secondWriteResult.filePath, new Date('2026-04-09T08:00:00.000Z'), new Date('2026-04-09T08:00:00.000Z'));
const dataFiles = await writer.listDataFiles(businessName, null);
const consumedRecords = [];
const consumeResult = await writer.consumeFiles({
businessName,
progressFilePath,
onRecord: async record => {
consumedRecords.push(record.name);
},
});
assert.deepEqual(dataFiles, [
path.resolve(firstWriteResult.filePath),
path.resolve(secondWriteResult.filePath),
]);
assert.deepEqual(consumedRecords, [ 'record-1', 'record-2' ]);
assert.equal(consumeResult.progress.filePath, path.resolve(secondWriteResult.filePath));
} finally {
fs.rmSync(path.join(baseDir, writer.sanitizeSegment(businessName)), { recursive: true, force: true });
}
});
it('should keep exact offsets for CRLF files without a trailing newline', async () => {
const writer = createWriter(baseDir);
const archiveDate = new Date(2026, 3, 9);
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log');
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
const content = [
JSON.stringify({ id: 1, name: 'record-1' }),
JSON.stringify({ id: 2, name: 'record-2' }),
].join('\r\n');
const expectedOffset = Buffer.byteLength(content);
const consumedRecords = [];
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
await fsp.writeFile(dataFilePath, content, 'utf8');
const readResult = await writer.readRecords(dataFilePath);
assert.equal(readResult.nextOffset, expectedOffset);
assert.deepEqual(readResult.records.map(record => record.id), [ 1, 2 ]);
const consumeResult = await writer.consumeFile(dataFilePath, {
progressFilePath,
onRecord: async record => {
consumedRecords.push(record.id);
},
});
assert.equal(consumeResult.nextOffset, expectedOffset);
assert.equal(consumeResult.processedCount, 2);
assert.deepEqual(consumedRecords, [ 1, 2 ]);
});
it('should ignore corrupt progress and completion markers and keep consuming', async () => {
const writer = createWriter(baseDir);
const archiveDate = new Date(2026, 3, 9);
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log');
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
const completionFilePath = writer.getDoneFilePath(dataFilePath);
const consumedRecords = [];
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
await fsp.writeFile(dataFilePath, [
JSON.stringify({ id: 1, name: 'record-1' }),
JSON.stringify({ id: 2, name: 'record-2' }),
].join('\n'), 'utf8');
await fsp.writeFile(progressFilePath, '{"filePath":123', 'utf8');
await fsp.writeFile(completionFilePath, '{"filePath":123', 'utf8');
const loadedProgress = await writer.loadProgress(progressFilePath);
const loadedCompletion = await writer.loadCompletionMarker(dataFilePath);
assert.equal(loadedProgress, null);
assert.equal(loadedCompletion, null);
const consumeResult = await writer.consumeFiles({
businessName: 'orders',
date: archiveDate,
progressFilePath,
onRecord: async record => {
consumedRecords.push(record.id);
},
});
assert.equal(consumeResult.processedCount, 2);
assert.deepEqual(consumedRecords, [ 1, 2 ]);
assert.equal(fs.existsSync(writer.getDoneFilePath(dataFilePath)), true);
});
it('should rethrow filesystem errors other than missing files when loading progress markers', async () => {
const writer = createWriter(baseDir);
const originalReadFile = fsp.readFile;
const error = new Error('permission denied');
error.code = 'EACCES';
try {
fsp.readFile = async () => {
throw error;
};
await assert.rejects(
() => writer.loadProgress(path.join(baseDir, 'orders.progress')),
thrownError => thrownError.code === 'EACCES'
);
await assert.rejects(
() => writer.loadCompletionMarker(path.join(baseDir, 'orders.log')),
thrownError => thrownError.code === 'EACCES'
);
} finally {
fsp.readFile = originalReadFile;
}
});
it('should recycle old partition states when the cache reaches its limit', async () => {
const writer = createWriter(baseDir, {
stateMapMaxEntries: 2,
});
const firstState = await writer.getPartitionState('orders-a', new Date(2026, 3, 9));
const secondState = await writer.getPartitionState('orders-b', new Date(2026, 3, 9));
const firstStateKey = `${writer.sanitizeSegment(firstState.businessSegment)}:${firstState.dateKey}`;
const secondStateKey = `${writer.sanitizeSegment(secondState.businessSegment)}:${secondState.dateKey}`;
assert.equal(writer.stateMap.size, 2);
assert.equal(writer.stateMap.has(firstStateKey), true);
assert.equal(writer.stateMap.has(secondStateKey), true);
const thirdState = await writer.getPartitionState('orders-c', new Date(2026, 3, 9));
const thirdStateKey = `${writer.sanitizeSegment(thirdState.businessSegment)}:${thirdState.dateKey}`;
assert.equal(writer.stateMap.size, 2);
assert.equal(writer.stateMap.has(firstStateKey), false);
assert.equal(writer.stateMap.has(secondStateKey), true);
assert.equal(writer.stateMap.has(thirdStateKey), true);
const firstStateReloaded = await writer.getPartitionState('orders-a', new Date(2026, 3, 9));
const firstStateReloadedKey = `${writer.sanitizeSegment(firstStateReloaded.businessSegment)}:${firstStateReloaded.dateKey}`;
assert.equal(writer.stateMap.size, 2);
assert.equal(writer.stateMap.has(firstStateReloadedKey), true);
});
it('should limit records when maxRecords is specified', async () => {
const writer = createWriter(baseDir, {
maxFileSize: 10485760,
});
for (let i = 0; i < 10; i += 1) {
await writer.writeRecord('batch-read', { index: i });
}
const files = await writer.listDataFiles('batch-read', null);
assert(files.length > 0);
const res1 = await writer.readRecords(files[0], { maxRecords: 3 });
assert.equal(res1.records.length, 3);
assert.equal(res1.records[0].index, 0);
assert.equal(res1.records[2].index, 2);
const res2 = await writer.readRecords(files[0], { startOffset: res1.nextOffset, maxRecords: 3 });
assert.equal(res2.records.length, 3);
assert.equal(res2.records[0].index, 3);
const res3 = await writer.readRecords(files[0], { startOffset: res2.nextOffset, maxRecords: 100 });
assert.equal(res3.records.length, 4);
assert.equal(res3.records[3].index, 9);
const resAll = await writer.readRecords(files[0]);
assert.equal(resAll.records.length, 10);
});
});