'use strict'; const fs = require('fs'); const path = require('path'); const { Service } = require('egg'); const ExcelJS = require('exceljs'); const { buildExportSql, STATUS_LABELS, TYPE_LABELS, formatDate } = require('../utils/exportDefs'); /** 任务状态:0待处理 1处理中 2已完成 3失败 */ const STATUS = { PENDING: 0, PROCESSING: 1, COMPLETED: 2, FAILED: 3 }; /** 单批次查询行数(边查边写,控制内存占用并上报进度) */ const CHUNK = 2000; /** * 解析任务参数(params 以 JSON 字符串存储) * @param {string|Object} s 参数 * @return {Object} 解析后的筛选对象 */ function safeParse(s) { if (!s) return {}; if (typeof s === 'object') return s; try { return JSON.parse(s); } catch { return {}; } } /** * 给任务对象补上中文标签(避免前后端枚举漂移) * @param {Object} item 任务纯对象 * @return {Object} 带 label 的任务 */ function withLabels(item) { item.type_label = TYPE_LABELS[item.type] || item.type; item.status_label = STATUS_LABELS[item.status] != null ? STATUS_LABELS[item.status] : String(item.status); return item; } class ExportTaskService extends Service { /** * 提交导出任务:仅构造并落库一条「待处理」记录,立即返回任务ID(实际导出由定时任务处理) * @param {string} type 导出类型 * @param {Object} filters 已白名单过滤的筛选条件 * @param {number} principalId 主体ID * @param {number} userId 提交人ID * @return {Promise} 任务实例 */ async submit(type, filters, principalId, userId) { const { sql } = buildExportSql(type, filters, principalId, userId); return this.ctx.model.ExportTask.create({ principal_id: principalId, type, status: STATUS.PENDING, sql, params: JSON.stringify(filters || {}), created_by: userId || null, }); } /** * 分页查询当前主体的导出任务 * @param {number} principalId 主体ID * @param {Object} opts page/pageSize/type/status * @return {Promise<{list: Array, total: number}>} */ async list(principalId, opts) { const { page, pageSize, type, status } = opts; const where = { principal_id: principalId }; if (type) where.type = type; if (status !== undefined && status !== null && status !== '') where.status = Number(status); const { count, rows } = await this.ctx.model.ExportTask.findAndCountAll({ where, offset: (Number(page) - 1) * Number(pageSize), limit: Number(pageSize), order: [[ 'id', 'DESC' ]], }); const list = rows.map(r => withLabels(r.get({ plain: true }))); return { list, total: count }; } /** * 查询单个任务(带中文标签) * @param {number} id 任务ID * @return {Promise} */ async progress(id) { const task = await this.ctx.model.ExportTask.findByPk(id); if (!task) return null; return withLabels(task.get({ plain: true })); } /** * 认领并处理一条「待处理」任务(全局单_worker队列,靠行锁保证集群下不重复认领) * 业务流程:事务内 FOR UPDATE 锁定最旧待处理任务 → 置为处理中 → 提交;随后分块查询并写 XLSX; * 完成时置已完成;异常时置失败并记录原因。 * @return {Promise} */ async processPending() { const ctx = this.ctx; const Model = ctx.model.ExportTask; const transaction = await ctx.model.transaction(); let task = null; try { task = await Model.findOne({ where: { status: STATUS.PENDING }, order: [[ 'id', 'ASC' ]], lock: true, transaction, }); if (!task) { await transaction.rollback(); return; } task.status = STATUS.PROCESSING; await task.save({ transaction }); await transaction.commit(); } catch (e) { await transaction.rollback().catch(() => {}); ctx.logger.error('[export-worker] 认领任务失败', e); return; } try { await this.run(task, ctx); } catch (e) { ctx.logger.error('[export-worker] 处理任务失败 id=' + task.id, e); } } /** * 执行导出:分块查询 + 写 XLSX + 实时更新进度,文件按 年/月/日 分目录 * @param {Object} task 任务实例 * @param {Egg.Context} ctx egg 上下文(含 model / app) * @return {Promise} */ async run(task, ctx) { const Model = ctx.model.ExportTask; const QueryTypes = ctx.model.QueryTypes; const workbook = new ExcelJS.Workbook(); const sheet = workbook.addWorksheet(TYPE_LABELS[task.type] || 'export'); try { const filters = safeParse(task.params); const { sql, countSql, columns, replacements } = buildExportSql(task.type, filters, task.principal_id, task.created_by); const [ countRow ] = await ctx.model.query(countSql, { replacements, type: QueryTypes.SELECT }); const total = Number(countRow && countRow.cnt) || 0; const uploadDir = ctx.app.config.upload.dir; const now = new Date(); const y = String(now.getFullYear()); const m = String(now.getMonth() + 1).padStart(2, '0'); const d = String(now.getDate()).padStart(2, '0'); const relDir = [ 'exports', y, m, d ].join('/'); fs.mkdirSync(path.join(uploadDir, relDir), { recursive: true }); const fileName = `export_${task.type}_${task.id}_${now.getTime()}.xlsx`; const relPath = relDir + '/' + fileName; const absPath = path.join(uploadDir, relPath); sheet.columns = columns.map(c => ({ header: c.header, key: c.key, width: 18 })); sheet.getRow(1).font = { bold: true }; sheet.views = [{ state: 'frozen', ySplit: 1 }]; let offset = 0; let processed = 0; if (total > 0) { // 循环分块:offset/CHUNK 为整数,直接内联拼接,无注入风险 let finished = false; while (!finished) { const rows = await ctx.model.query(`${sql} LIMIT ${offset}, ${CHUNK}`, { replacements, type: QueryTypes.SELECT, }); for (const row of rows) { const vals = {}; for (const c of columns) { let v = c.format ? c.format(row[c.key], row) : row[c.key]; if (v instanceof Date) v = formatDate(v); vals[c.key] = v; } sheet.addRow(vals); } processed += rows.length; offset += CHUNK; const progress = Math.min(99, Math.floor(processed / total * 100)); await Model.update( { processed_count: processed, progress, file_path: relPath, file_name: fileName }, { where: { id: task.id } } ); if (rows.length < CHUNK) finished = true; } } await workbook.xlsx.writeFile(absPath); await Model.update( { status: STATUS.COMPLETED, total_count: total, processed_count: total, progress: 100, file_path: relPath, file_name: fileName, }, { where: { id: task.id } } ); ctx.logger.info(`[export-worker] 任务完成 id=${task.id} type=${task.type} rows=${total}`); } catch (e) { const msg = (e && e.message) ? e.message : String(e); await Model.update( { status: STATUS.FAILED, error_message: msg.slice(0, 500) }, { where: { id: task.id } } ).catch(() => {}); ctx.logger.error('[export-worker] 任务失败 id=' + task.id, e); throw e; } } /** * 解析任务对应的导出文件绝对路径(校验防止目录穿越) * @param {Object} task 任务实例 * @return {Promise} 文件绝对路径或 null */ async download(task) { const rel = task.file_path; if (!rel || rel.includes('..') || !/^exports\//.test(rel)) return null; const abs = path.join(this.ctx.app.config.upload.dir, rel); if (!fs.existsSync(abs)) return null; return abs; } } module.exports = ExportTaskService;