509 lines
19 KiB
JavaScript
509 lines
19 KiB
JavaScript
'use strict';
|
|
|
|
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs');
|
|
const fsp = require('node:fs/promises');
|
|
const os = require('node:os');
|
|
const path = require('node:path');
|
|
|
|
const LocalFileWriter = require('../../../app/lib/file_writer');
|
|
|
|
function createWriter(baseDir, fileWriterOverrides = {}) {
|
|
return new LocalFileWriter({
|
|
baseDir: process.cwd(),
|
|
config: {
|
|
fileWriter: {
|
|
baseDir,
|
|
maxFileSize: 10485760,
|
|
maxConsumeRecordsPerWindow: 1000,
|
|
maxConsumeRecordsPerRun: 1000,
|
|
writeWindowSeconds: 1,
|
|
maxWriteRecordsPerWindow: 2000,
|
|
writeWindowBucketCount: 1,
|
|
retentionDays: 1,
|
|
cleanupIntervalMs: 0,
|
|
sequenceWidth: 2,
|
|
machineId: 'test-machine',
|
|
workerId: 'test-worker',
|
|
fileExtension: '.log',
|
|
progressExtension: '.progress',
|
|
doneExtension: '.done',
|
|
...fileWriterOverrides,
|
|
},
|
|
},
|
|
logger: {
|
|
warn() {},
|
|
info() {},
|
|
error() {},
|
|
debug() {},
|
|
},
|
|
});
|
|
}
|
|
|
|
function touchOld(filePath, daysAgo = 2) {
|
|
const time = new Date(Date.now() - daysAgo * 24 * 60 * 60 * 1000);
|
|
fs.utimesSync(filePath, time, time);
|
|
}
|
|
|
|
describe('test/app/lib/file_writer.test.js', () => {
|
|
let baseDir = '';
|
|
|
|
beforeEach(() => {
|
|
baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wms-file-writer-'));
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(baseDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it('should keep unfinished data files and progress files during cleanup', async () => {
|
|
const writer = createWriter(baseDir);
|
|
const archiveDate = new Date(2026, 3, 9);
|
|
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
|
|
const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log');
|
|
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
|
|
|
|
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
|
|
await fsp.writeFile(dataFilePath, '{"id":1}\n', 'utf8');
|
|
await writer.saveProgress(progressFilePath, {
|
|
filePath: dataFilePath,
|
|
offset: 0,
|
|
processedCount: 0,
|
|
});
|
|
|
|
touchOld(dataFilePath);
|
|
touchOld(progressFilePath);
|
|
|
|
const result = await writer.cleanupExpiredFiles();
|
|
|
|
assert.equal(result.removedFiles, 0);
|
|
assert.equal(fs.existsSync(dataFilePath), true);
|
|
assert.equal(fs.existsSync(progressFilePath), true);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(dataFilePath)), false);
|
|
});
|
|
|
|
it('should mark completed files, skip them on restart, and clean them up later', async () => {
|
|
const writer = createWriter(baseDir);
|
|
const archiveDate = new Date(2026, 3, 9);
|
|
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
|
|
|
|
const writeResult = await writer.writeRecord('orders', {
|
|
id: 1,
|
|
name: 'completed-record',
|
|
}, {
|
|
date: archiveDate,
|
|
});
|
|
|
|
let replayedCount = 0;
|
|
const consumeResult = await writer.consumeFile(writeResult.filePath, {
|
|
progressFilePath,
|
|
onRecord: async () => {
|
|
replayedCount += 1;
|
|
},
|
|
});
|
|
|
|
assert.equal(consumeResult.processedCount, 1);
|
|
assert.equal(replayedCount, 1);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(writeResult.filePath)), true);
|
|
|
|
const replayCount = await writer.consumeFiles({
|
|
businessName: 'orders',
|
|
date: archiveDate,
|
|
progressFilePath,
|
|
onRecord: async () => {
|
|
replayedCount += 1;
|
|
},
|
|
});
|
|
|
|
assert.equal(replayCount.processedCount, 0);
|
|
assert.equal(replayedCount, 1);
|
|
|
|
touchOld(writeResult.filePath);
|
|
touchOld(writer.getDoneFilePath(writeResult.filePath));
|
|
|
|
const cleanupResult = await writer.cleanupExpiredFiles();
|
|
|
|
assert.equal(cleanupResult.removedFiles, 1);
|
|
assert.equal(fs.existsSync(writeResult.filePath), false);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(writeResult.filePath)), false);
|
|
assert.equal(fs.existsSync(progressFilePath), false);
|
|
});
|
|
|
|
it('should keep consuming across files and partitions until the shared limit is reached', async () => {
|
|
const writer = createWriter(baseDir, {
|
|
maxConsumeRecordsPerWindow: 4,
|
|
});
|
|
const businessName = 'orders';
|
|
const firstArchiveDate = new Date(2026, 3, 9);
|
|
const secondArchiveDate = new Date(2026, 3, 10);
|
|
const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer');
|
|
const firstPartition = writer.getPartitionInfo(businessName, firstArchiveDate);
|
|
const secondPartition = writer.getPartitionInfo(businessName, secondArchiveDate);
|
|
const firstFilePath = path.join(firstPartition.partitionDir, 'manual-01.log');
|
|
const secondFilePath = path.join(secondPartition.partitionDir, 'manual-01.log');
|
|
const consumedRecords = [];
|
|
|
|
await fsp.mkdir(firstPartition.partitionDir, { recursive: true });
|
|
await fsp.mkdir(secondPartition.partitionDir, { recursive: true });
|
|
await fsp.writeFile(firstFilePath, JSON.stringify({ id: 1, name: 'record-1' }), 'utf8');
|
|
await fsp.writeFile(secondFilePath, [
|
|
JSON.stringify({ id: 2, name: 'record-2' }),
|
|
JSON.stringify({ id: 3, name: 'record-3' }),
|
|
JSON.stringify({ id: 4, name: 'record-4' }),
|
|
JSON.stringify({ id: 5, name: 'record-5' }),
|
|
].join('\n'), 'utf8');
|
|
|
|
const consumeResult = await writer.consumeFiles({
|
|
businessName,
|
|
date: null,
|
|
progressFilePath,
|
|
now: Date.UTC(2026, 3, 9, 10, 0, 0),
|
|
onRecord: async record => {
|
|
consumedRecords.push(record.name);
|
|
},
|
|
});
|
|
|
|
assert.equal(consumeResult.processedCount, 4);
|
|
assert.equal(consumeResult.limitReached, true);
|
|
assert.deepEqual(consumedRecords, [
|
|
'record-1',
|
|
'record-2',
|
|
'record-3',
|
|
'record-4',
|
|
]);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), true);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(secondFilePath)), false);
|
|
assert.equal(consumeResult.progress.filePath, path.resolve(secondFilePath));
|
|
});
|
|
|
|
it('should throttle writes when write window record limit is reached', async () => {
|
|
const writer = createWriter(baseDir, {
|
|
maxWriteRecordsPerWindow: 3,
|
|
writeWindowSeconds: 1,
|
|
});
|
|
const archiveDate = new Date(2026, 3, 9);
|
|
|
|
const firstResult = await writer.writeRecord('orders', { id: 1 }, { date: archiveDate });
|
|
assert.equal(firstResult.throttled, false);
|
|
assert.equal(firstResult.writeWindowRecordCount, 1);
|
|
|
|
const secondResult = await writer.writeRecord('orders', { id: 2 }, { date: archiveDate });
|
|
assert.equal(secondResult.throttled, false);
|
|
assert.equal(secondResult.writeWindowRecordCount, 2);
|
|
|
|
const thirdResult = await writer.writeRecord('orders', { id: 3 }, { date: archiveDate });
|
|
assert.equal(thirdResult.throttled, false);
|
|
assert.equal(thirdResult.writeWindowRecordCount, 3);
|
|
|
|
const throttledResult = await writer.writeRecord('orders', { id: 4 }, { date: archiveDate });
|
|
assert.equal(throttledResult.throttled, true);
|
|
assert.equal(throttledResult.writeWindowRecordCount, 3);
|
|
assert.equal(throttledResult.maxWriteRecordsPerWindow, 3);
|
|
assert.ok(throttledResult.resumeAt);
|
|
assert.equal(throttledResult.filePath, undefined);
|
|
|
|
const readResult = await writer.readRecords(thirdResult.filePath);
|
|
assert.equal(readResult.records.length, 3);
|
|
});
|
|
|
|
it('should consume with a simple per-call limit without time-based windowing', async () => {
|
|
const writer = createWriter(baseDir, {
|
|
maxConsumeRecordsPerWindow: 2,
|
|
});
|
|
const archiveDate = new Date(2026, 3, 9);
|
|
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
|
|
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
|
|
const consumedRecords = [];
|
|
const firstFilePath = writer.buildDataFilePath(partitionInfo, 1);
|
|
|
|
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
|
|
await fsp.writeFile(firstFilePath, [
|
|
JSON.stringify({ id: 1, name: 'record-1' }),
|
|
JSON.stringify({ id: 2, name: 'record-2' }),
|
|
JSON.stringify({ id: 3, name: 'record-3' }),
|
|
].join('\n'), 'utf8');
|
|
|
|
const firstConsumeResult = await writer.consumeFiles({
|
|
businessName: 'orders',
|
|
date: archiveDate,
|
|
progressFilePath,
|
|
onRecord: async record => {
|
|
consumedRecords.push(record.name);
|
|
},
|
|
});
|
|
|
|
assert.equal(firstConsumeResult.processedCount, 2);
|
|
assert.equal(firstConsumeResult.limitReached, true);
|
|
assert.equal(firstConsumeResult.maxConsumeRecordsPerRun, 2);
|
|
assert.equal(consumedRecords.length, 2);
|
|
assert.deepEqual(consumedRecords, [ 'record-1', 'record-2' ]);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), false);
|
|
|
|
const secondConsumeResult = await writer.consumeFiles({
|
|
businessName: 'orders',
|
|
date: archiveDate,
|
|
progressFilePath,
|
|
onRecord: async record => {
|
|
consumedRecords.push(record.name);
|
|
},
|
|
});
|
|
|
|
assert.equal(secondConsumeResult.processedCount, 1);
|
|
assert.equal(secondConsumeResult.limitReached, false);
|
|
assert.equal(secondConsumeResult.maxConsumeRecordsPerRun, 2);
|
|
assert.deepEqual(consumedRecords, [ 'record-1', 'record-2', 'record-3' ]);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(firstFilePath)), true);
|
|
assert.equal(fs.existsSync(progressFilePath), true);
|
|
});
|
|
|
|
it('should consume files across multiple date partitions in write order without a date filter', async () => {
|
|
const writer = createWriter(baseDir);
|
|
const businessName = 'orders';
|
|
const firstArchiveDate = new Date(2026, 3, 10);
|
|
const secondArchiveDate = new Date(2026, 3, 9);
|
|
const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer');
|
|
const consumedRecords = [];
|
|
|
|
try {
|
|
const firstWriteResult = await writer.writeRecord(businessName, {
|
|
id: 1,
|
|
name: 'record-1',
|
|
}, {
|
|
date: firstArchiveDate,
|
|
});
|
|
|
|
const secondWriteResult = await writer.writeRecord(businessName, {
|
|
id: 2,
|
|
name: 'record-2',
|
|
}, {
|
|
date: secondArchiveDate,
|
|
});
|
|
|
|
fs.utimesSync(firstWriteResult.filePath, new Date('2026-04-09T08:00:00.000Z'), new Date('2026-04-09T08:00:00.000Z'));
|
|
fs.utimesSync(secondWriteResult.filePath, new Date('2026-04-09T08:00:01.000Z'), new Date('2026-04-09T08:00:01.000Z'));
|
|
|
|
const consumeResult = await writer.consumeFiles({
|
|
businessName,
|
|
progressFilePath,
|
|
onRecord: async record => {
|
|
consumedRecords.push(record.name);
|
|
},
|
|
});
|
|
|
|
assert.equal(consumeResult.processedCount, 2);
|
|
assert.equal(consumeResult.limitReached, false);
|
|
assert.equal(Array.isArray(consumeResult.dataFiles), true);
|
|
assert.equal(consumeResult.dataFiles.length, 2);
|
|
assert.deepEqual(consumedRecords, [ 'record-2', 'record-1' ]);
|
|
assert.deepEqual(consumeResult.dataFiles, [
|
|
path.resolve(secondWriteResult.filePath),
|
|
path.resolve(firstWriteResult.filePath),
|
|
]);
|
|
assert.equal(consumeResult.progress.filePath, path.resolve(firstWriteResult.filePath));
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(firstWriteResult.filePath)), true);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(secondWriteResult.filePath)), true);
|
|
} finally {
|
|
fs.rmSync(path.join(baseDir, writer.sanitizeSegment(businessName)), { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it('should sort files within the same date partition by sequence instead of mtime', async () => {
|
|
const writer = createWriter(baseDir, {
|
|
maxFileSize: 1,
|
|
});
|
|
const businessName = 'orders';
|
|
const archiveDate = new Date(2026, 3, 9);
|
|
const progressFilePath = writer.getProgressFilePath(businessName, null, 'consumer');
|
|
|
|
try {
|
|
const firstWriteResult = await writer.writeRecord(businessName, {
|
|
id: 1,
|
|
name: 'record-1',
|
|
}, {
|
|
date: archiveDate,
|
|
});
|
|
|
|
const secondWriteResult = await writer.writeRecord(businessName, {
|
|
id: 2,
|
|
name: 'record-2',
|
|
}, {
|
|
date: archiveDate,
|
|
});
|
|
|
|
fs.utimesSync(firstWriteResult.filePath, new Date('2026-04-09T08:00:01.000Z'), new Date('2026-04-09T08:00:01.000Z'));
|
|
fs.utimesSync(secondWriteResult.filePath, new Date('2026-04-09T08:00:00.000Z'), new Date('2026-04-09T08:00:00.000Z'));
|
|
|
|
const dataFiles = await writer.listDataFiles(businessName, null);
|
|
const consumedRecords = [];
|
|
const consumeResult = await writer.consumeFiles({
|
|
businessName,
|
|
progressFilePath,
|
|
onRecord: async record => {
|
|
consumedRecords.push(record.name);
|
|
},
|
|
});
|
|
|
|
assert.deepEqual(dataFiles, [
|
|
path.resolve(firstWriteResult.filePath),
|
|
path.resolve(secondWriteResult.filePath),
|
|
]);
|
|
assert.deepEqual(consumedRecords, [ 'record-1', 'record-2' ]);
|
|
assert.equal(consumeResult.progress.filePath, path.resolve(secondWriteResult.filePath));
|
|
} finally {
|
|
fs.rmSync(path.join(baseDir, writer.sanitizeSegment(businessName)), { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it('should keep exact offsets for CRLF files without a trailing newline', async () => {
|
|
const writer = createWriter(baseDir);
|
|
const archiveDate = new Date(2026, 3, 9);
|
|
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
|
|
const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log');
|
|
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
|
|
const content = [
|
|
JSON.stringify({ id: 1, name: 'record-1' }),
|
|
JSON.stringify({ id: 2, name: 'record-2' }),
|
|
].join('\r\n');
|
|
const expectedOffset = Buffer.byteLength(content);
|
|
const consumedRecords = [];
|
|
|
|
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
|
|
await fsp.writeFile(dataFilePath, content, 'utf8');
|
|
|
|
const readResult = await writer.readRecords(dataFilePath);
|
|
assert.equal(readResult.nextOffset, expectedOffset);
|
|
assert.deepEqual(readResult.records.map(record => record.id), [ 1, 2 ]);
|
|
|
|
const consumeResult = await writer.consumeFile(dataFilePath, {
|
|
progressFilePath,
|
|
onRecord: async record => {
|
|
consumedRecords.push(record.id);
|
|
},
|
|
});
|
|
|
|
assert.equal(consumeResult.nextOffset, expectedOffset);
|
|
assert.equal(consumeResult.processedCount, 2);
|
|
assert.deepEqual(consumedRecords, [ 1, 2 ]);
|
|
});
|
|
|
|
it('should ignore corrupt progress and completion markers and keep consuming', async () => {
|
|
const writer = createWriter(baseDir);
|
|
const archiveDate = new Date(2026, 3, 9);
|
|
const partitionInfo = writer.getPartitionInfo('orders', archiveDate);
|
|
const dataFilePath = path.join(partitionInfo.partitionDir, 'manual-01.log');
|
|
const progressFilePath = writer.getProgressFilePath('orders', archiveDate, 'consumer');
|
|
const completionFilePath = writer.getDoneFilePath(dataFilePath);
|
|
const consumedRecords = [];
|
|
|
|
await fsp.mkdir(partitionInfo.partitionDir, { recursive: true });
|
|
await fsp.writeFile(dataFilePath, [
|
|
JSON.stringify({ id: 1, name: 'record-1' }),
|
|
JSON.stringify({ id: 2, name: 'record-2' }),
|
|
].join('\n'), 'utf8');
|
|
await fsp.writeFile(progressFilePath, '{"filePath":123', 'utf8');
|
|
await fsp.writeFile(completionFilePath, '{"filePath":123', 'utf8');
|
|
|
|
const loadedProgress = await writer.loadProgress(progressFilePath);
|
|
const loadedCompletion = await writer.loadCompletionMarker(dataFilePath);
|
|
|
|
assert.equal(loadedProgress, null);
|
|
assert.equal(loadedCompletion, null);
|
|
|
|
const consumeResult = await writer.consumeFiles({
|
|
businessName: 'orders',
|
|
date: archiveDate,
|
|
progressFilePath,
|
|
onRecord: async record => {
|
|
consumedRecords.push(record.id);
|
|
},
|
|
});
|
|
|
|
assert.equal(consumeResult.processedCount, 2);
|
|
assert.deepEqual(consumedRecords, [ 1, 2 ]);
|
|
assert.equal(fs.existsSync(writer.getDoneFilePath(dataFilePath)), true);
|
|
});
|
|
|
|
it('should rethrow filesystem errors other than missing files when loading progress markers', async () => {
|
|
const writer = createWriter(baseDir);
|
|
const originalReadFile = fsp.readFile;
|
|
const error = new Error('permission denied');
|
|
error.code = 'EACCES';
|
|
|
|
try {
|
|
fsp.readFile = async () => {
|
|
throw error;
|
|
};
|
|
|
|
await assert.rejects(
|
|
() => writer.loadProgress(path.join(baseDir, 'orders.progress')),
|
|
thrownError => thrownError.code === 'EACCES'
|
|
);
|
|
|
|
await assert.rejects(
|
|
() => writer.loadCompletionMarker(path.join(baseDir, 'orders.log')),
|
|
thrownError => thrownError.code === 'EACCES'
|
|
);
|
|
} finally {
|
|
fsp.readFile = originalReadFile;
|
|
}
|
|
});
|
|
|
|
it('should recycle old partition states when the cache reaches its limit', async () => {
|
|
const writer = createWriter(baseDir, {
|
|
stateMapMaxEntries: 2,
|
|
});
|
|
|
|
const firstState = await writer.getPartitionState('orders-a', new Date(2026, 3, 9));
|
|
const secondState = await writer.getPartitionState('orders-b', new Date(2026, 3, 9));
|
|
const firstStateKey = `${writer.sanitizeSegment(firstState.businessSegment)}:${firstState.dateKey}`;
|
|
const secondStateKey = `${writer.sanitizeSegment(secondState.businessSegment)}:${secondState.dateKey}`;
|
|
|
|
assert.equal(writer.stateMap.size, 2);
|
|
assert.equal(writer.stateMap.has(firstStateKey), true);
|
|
assert.equal(writer.stateMap.has(secondStateKey), true);
|
|
|
|
const thirdState = await writer.getPartitionState('orders-c', new Date(2026, 3, 9));
|
|
const thirdStateKey = `${writer.sanitizeSegment(thirdState.businessSegment)}:${thirdState.dateKey}`;
|
|
|
|
assert.equal(writer.stateMap.size, 2);
|
|
assert.equal(writer.stateMap.has(firstStateKey), false);
|
|
assert.equal(writer.stateMap.has(secondStateKey), true);
|
|
assert.equal(writer.stateMap.has(thirdStateKey), true);
|
|
|
|
const firstStateReloaded = await writer.getPartitionState('orders-a', new Date(2026, 3, 9));
|
|
const firstStateReloadedKey = `${writer.sanitizeSegment(firstStateReloaded.businessSegment)}:${firstStateReloaded.dateKey}`;
|
|
|
|
assert.equal(writer.stateMap.size, 2);
|
|
assert.equal(writer.stateMap.has(firstStateReloadedKey), true);
|
|
});
|
|
|
|
it('should limit records when maxRecords is specified', async () => {
|
|
const writer = createWriter(baseDir, {
|
|
maxFileSize: 10485760,
|
|
});
|
|
|
|
for (let i = 0; i < 10; i += 1) {
|
|
await writer.writeRecord('batch-read', { index: i });
|
|
}
|
|
|
|
const files = await writer.listDataFiles('batch-read', null);
|
|
assert(files.length > 0);
|
|
|
|
const res1 = await writer.readRecords(files[0], { maxRecords: 3 });
|
|
assert.equal(res1.records.length, 3);
|
|
assert.equal(res1.records[0].index, 0);
|
|
assert.equal(res1.records[2].index, 2);
|
|
|
|
const res2 = await writer.readRecords(files[0], { startOffset: res1.nextOffset, maxRecords: 3 });
|
|
assert.equal(res2.records.length, 3);
|
|
assert.equal(res2.records[0].index, 3);
|
|
|
|
const res3 = await writer.readRecords(files[0], { startOffset: res2.nextOffset, maxRecords: 100 });
|
|
assert.equal(res3.records.length, 4);
|
|
assert.equal(res3.records[3].index, 9);
|
|
|
|
const resAll = await writer.readRecords(files[0]);
|
|
assert.equal(resAll.records.length, 10);
|
|
});
|
|
});
|