oms/backend/app/controller/shopDatabase.js
liufumingren c7a3a01faf feat(shop): 数据库店铺支持拉取商品与订单,一键拉单纳入数据库店铺并收紧货主范围
- 新增 shopDatabaseSync 服务:按商品表/订单表/明细表字段映射读取客户库,写入店铺商品与待审核订单(含明细重建、幂等更新、无明细跳过、已流转单只刷新平台侧数据)
- shop_database 控制器新增 pull-products / pull-orders 接口与路由、contract
- 一键拉单任务新增 database / all 范围,resolveShops 支持按货主收窄,runShop 走数据库店铺分支
- 店铺列表新增「拉取商品」「拉取订单」,订单页一键拉单可按范围选择店铺

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-05 16:20:50 +08:00

556 lines
20 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

'use strict';
const { Controller } = require('egg');
const {
DB_TYPES,
DEFAULT_DB_TYPE,
DATASETS,
datasetDefs,
} = require('../lib/shopDatabaseFields');
const {
SAFE_IDENTIFIER,
normalizeParams,
validateParams,
openConnection,
closeQuietly,
testConnection,
listTables,
listColumns,
} = require('../lib/shopDatabaseClient');
/** 数据库店铺的店铺类型值(shops.type) */
const SHOP_TYPE_DATABASE = 3;
/**
* @controller ShopDatabaseController 数据库店铺配置
*
* 数据库店铺(shops.type=3)直接连客户自有 MySQL 库拉取明文订单,
* 因此需要维护:连接信息(类型/地址/端口/库名/账号/密码)、
* 三张数据集(商品表 / 订单表 / 订单明细表)的选表结果与字段映射。
* 所有连接动作都在服务端完成(浏览器不直连客户库)。
*/
class ShopDatabaseController extends Controller {
/**
* 取店铺并校验可见范围
* @param {number|string} id 店铺ID
* @return {Promise<Object|null>} 店铺实例;失败时已写入响应
*/
async loadShop(id) {
const { ctx } = this;
const shop = await ctx.model.Shop.findByPk(id, {
attributes: [ 'id', 'principal_id', 'name', 'type', 'subsidiary_id' ],
});
if (!shop) {
ctx.status = 404;
ctx.body = { code: 404, message: '店铺不存在' };
return null;
}
if (Number(shop.type) !== SHOP_TYPE_DATABASE) {
ctx.status = 400;
ctx.body = { code: 400, message: '仅「数据库店铺」需要配置数据库连接' };
return null;
}
// 所在公司权限:仅可配置自己所属公司(货主)的店铺
const access = await ctx.service.access.resolve(ctx.userId, shop.principal_id);
if (!access.isGroup && !access.ownCompanyIds.map(Number).includes(Number(shop.subsidiary_id))) {
ctx.status = 403;
ctx.body = { code: 403, message: '仅可配置自己所属公司(货主)的店铺' };
return null;
}
return shop;
}
/**
* 取(或初始化)店铺的数据库配置记录
* @param {Object} shop 店铺实例
* @return {Promise<Object>} 配置 model 实例
*/
async findConfig(shop) {
const { ctx } = this;
const exist = await ctx.model.ShopDatabaseConfig.findOne({ where: { shop_id: shop.id } });
if (exist) return exist;
return ctx.model.ShopDatabaseConfig.build({
principal_id: shop.principal_id,
shop_id: shop.id,
db_type: DEFAULT_DB_TYPE,
});
}
/**
* 合并前端入参与已保存配置(密码留空表示沿用已保存的密码)
* @param {Object} body 请求体
* @param {Object} config 已保存配置(可为空)
* @return {Object} 规范化后的连接参数
*/
resolveParams(body, config) {
const saved = config
? {
db_type: config.db_type,
host: config.host,
port: config.port,
database_name: config.database_name,
username: config.username,
password: config.password,
}
: {};
const input = body || {};
const password = input.password === undefined || input.password === null || input.password === ''
? (saved.password || '')
: input.password;
return normalizeParams({ ...saved, ...input, password });
}
/**
* 输出用的配置视图(密码不回显,只给 has_password)
* @param {Object} config 配置实例
* @param {Object} [shop] 店铺实例
* @return {Object} 配置视图
*/
toView(config, shop) {
const plain = config && config.get ? config.get({ plain: true }) : (config || {});
return {
shop_id: plain.shop_id,
shop_name: shop ? shop.name : undefined,
db_type: plain.db_type || DEFAULT_DB_TYPE,
host: plain.host || '',
port: plain.port == null ? null : Number(plain.port),
database_name: plain.database_name || '',
username: plain.username || '',
has_password: !!plain.password,
product_table: plain.product_table || '',
order_table: plain.order_table || '',
order_item_table: plain.order_item_table || '',
product_fields: plain.product_fields || {},
order_fields: plain.order_fields || {},
order_item_fields: plain.order_item_fields || {},
last_test_at: plain.last_test_at || null,
last_test_success: plain.last_test_success == null ? null : Number(plain.last_test_success),
last_test_message: plain.last_test_message || '',
};
}
/**
* @summary 数据库店铺配置详情
* @description 返回连接信息(密码不回显)、三张数据集的选表与字段映射、以及可映射字段定义
* @router get /api/shops/{id}/database-config
* @apikey
* @request path integer *id 店铺主键ID
* @response 200 shopDatabaseConfigResponse success
*/
async show() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
const config = await this.findConfig(shop);
ctx.body = {
code: 0,
message: 'success',
data: {
config: this.toView(config, shop),
db_types: DB_TYPES,
datasets: datasetDefs(),
},
};
}
/**
* 校验字段映射:键必须在字段定义内,值必须是非空安全标识符,必填字段不能缺
* @param {Object} mapping 前端提交的映射
* @param {Object} dataset DATASETS 中的定义
* @return {string|null} 错误提示,通过返回 null
*/
validateMapping(mapping, dataset) {
const raw = mapping && typeof mapping === 'object' ? mapping : {};
const allowed = new Set(dataset.fields.map(field => field.key));
for (const [ key, value ] of Object.entries(raw)) {
if (!allowed.has(key)) return `${dataset.label}:不支持的字段 ${key}`;
if (value === null || value === undefined || String(value).trim() === '') continue;
if (!SAFE_IDENTIFIER.test(String(value).trim())) return `${dataset.label}:字段「${key}」的源列名不合法`;
}
const missing = dataset.fields
.filter(field => field.required)
.filter(field => {
const value = raw[field.key];
return value === null || value === undefined || String(value).trim() === '';
})
.map(field => field.label);
if (missing.length) return `${dataset.label}:请完成必填字段映射(${missing.join('、')})`;
return null;
}
/**
* 整理要落库的映射(去掉空值,统一 trim)
* @param {Object} mapping 前端提交的映射
* @return {Object} 清洗后的映射
*/
cleanMapping(mapping) {
const result = {};
for (const [ key, value ] of Object.entries(mapping && typeof mapping === 'object' ? mapping : {})) {
const text = value === null || value === undefined ? '' : String(value).trim();
if (text) result[key] = text;
}
return result;
}
/**
* 连库校验:表是否存在、映射列是否存在于对应表
* @param {Object} params 连接参数
* @param {Object} tables 三张数据集选表结果 { product_table: 表名 ... }
* @param {Object} mappings 三张数据集映射 { product_fields: {...} ... }
* @return {Promise<{ok: boolean, connected: boolean, message?: string}>} 校验结果(connected 表示是否连上了库)
*/
async verifyAgainstDatabase(params, tables, mappings) {
let conn = null;
try {
conn = await openConnection(params);
const remoteTables = await listTables(conn, params);
const names = new Set(remoteTables.map(row => row.name));
for (const dataset of Object.values(DATASETS)) {
const table = tables[dataset.tableColumn];
if (!names.has(table)) {
return { ok: false, connected: true, message: `${dataset.label}:库中不存在表 ${table},请重新连接后选择` };
}
}
const columnsByTable = new Map();
for (const dataset of Object.values(DATASETS)) {
const table = tables[dataset.tableColumn];
if (!columnsByTable.has(table)) {
const rows = await listColumns(conn, params, table);
columnsByTable.set(table, new Set(rows.map(row => row.name)));
}
const columns = columnsByTable.get(table);
const mapping = mappings[dataset.fieldColumn] || {};
for (const [ key, column ] of Object.entries(mapping)) {
if (!columns.has(column)) {
const field = dataset.fields.find(item => item.key === key);
return {
ok: false,
connected: true,
message: `${dataset.label}:字段「${field ? field.label : key}」映射的列 ${column} 不存在`,
};
}
}
}
return { ok: true, connected: true };
} catch (err) {
return {
ok: false,
connected: false,
message: `连接失败:${err && err.message ? err.message : '未知错误'}`,
};
} finally {
await closeQuietly(conn);
}
}
/**
* @summary 保存数据库店铺配置
* @description 保存连接信息(密码留空沿用原值)、三张数据集选表与字段映射;
* 保存时会尝试连库校验表与列是否真实存在(库不可达时仍保存,返回提示)
* @router put /api/shops/{id}/database-config
* @apikey
* @request path integer *id 店铺主键ID
* @request body shopDatabaseConnectionRequest *body
* @response 200 baseResponse success
*/
async save() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
const config = await this.findConfig(shop);
const body = ctx.request.body || {};
const params = this.resolveParams(body, config);
const paramsError = validateParams(params);
if (paramsError) {
ctx.status = 400;
ctx.body = { code: 400, message: paramsError };
return;
}
const tables = {};
const mappings = {};
for (const dataset of Object.values(DATASETS)) {
const table = String(body[dataset.tableColumn] || '').trim();
if (!table) {
ctx.status = 400;
ctx.body = { code: 400, message: `请选择${dataset.label}` };
return;
}
if (!SAFE_IDENTIFIER.test(table)) {
ctx.status = 400;
ctx.body = { code: 400, message: `${dataset.label}:表名不合法` };
return;
}
tables[dataset.tableColumn] = table;
const mappingError = this.validateMapping(body[dataset.fieldColumn], dataset);
if (mappingError) {
ctx.status = 400;
ctx.body = { code: 400, message: mappingError };
return;
}
mappings[dataset.fieldColumn] = this.cleanMapping(body[dataset.fieldColumn]);
}
// 能连上就校验表/列是否真实存在:连上但校验不过直接拒绝(避免存下跑不通的映射);
// 连不上时仍保存连接信息,便于先存配置再排查网络
const verified = await this.verifyAgainstDatabase(params, tables, mappings);
if (verified.connected && !verified.ok) {
ctx.status = 400;
ctx.body = { code: 400, message: verified.message };
return;
}
config.set({
principal_id: shop.principal_id,
shop_id: shop.id,
db_type: params.db_type,
host: params.host,
port: params.port,
database_name: params.database_name,
username: params.username,
password: params.password,
...tables,
...mappings,
last_test_at: new Date(),
last_test_success: verified.ok ? 1 : 0,
last_test_message: String(verified.ok ? '配置保存成功,表与字段校验通过' : verified.message).slice(0, 500),
});
const saved = await config.save();
ctx.body = {
code: 0,
message: verified.ok ? 'success' : `配置已保存,但未完成连库校验:${verified.message}`,
data: { id: saved.id, verified: verified.ok },
};
}
/**
* @summary 测试数据库连通
* @description 使用传入参数(密码留空则用已保存密码)在服务端连接客户库,返回版本与耗时
* @router post /api/shops/{id}/database-config/test
* @apikey
* @request path integer *id 店铺主键ID
* @request body shopDatabaseConnectionRequest *body
* @response 200 baseResponse success
*/
async test() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
const config = await this.findConfig(shop);
const params = this.resolveParams(ctx.request.body, config);
const paramsError = validateParams(params);
if (paramsError) {
ctx.status = 400;
ctx.body = { code: 400, message: paramsError };
return;
}
const result = await testConnection(params);
ctx.body = {
code: 0,
message: result.message,
data: { success: result.success, server_version: result.server_version || '', message: result.message },
};
}
/**
* @summary 连接数据库并拉取数据表
* @description 连通后返回库内所有表,供三张数据集选择
* @router post /api/shops/{id}/database-config/connect
* @apikey
* @request path integer *id 店铺主键ID
* @request body shopDatabaseConnectionRequest *body
* @response 200 baseResponse success
*/
async connect() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
const config = await this.findConfig(shop);
const params = this.resolveParams(ctx.request.body, config);
const paramsError = validateParams(params);
if (paramsError) {
ctx.status = 400;
ctx.body = { code: 400, message: paramsError };
return;
}
let conn = null;
try {
conn = await openConnection(params);
const tables = await listTables(conn, params);
ctx.body = {
code: 0,
message: `连接成功,共 ${tables.length} 张表`,
data: { success: true, tables },
};
} catch (err) {
ctx.status = 400;
ctx.body = { code: 400, message: `连接失败:${err && err.message ? err.message : '未知错误'}` };
} finally {
await closeQuietly(conn);
}
}
/**
* @summary 用已保存配置拉取数据表
* @description 配置不完整或库不可达时返回 400,前端可提示重新连接
* @router get /api/shops/{id}/database-config/tables
* @apikey
* @request path integer *id 店铺主键ID
* @response 200 baseResponse success
*/
async tables() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
const config = await this.findConfig(shop);
const params = this.resolveParams({}, config);
const paramsError = validateParams(params);
if (paramsError) {
ctx.status = 400;
ctx.body = { code: 400, message: `请先完善数据库连接信息:${paramsError}` };
return;
}
const result = await testConnection(params);
if (!result.success) {
ctx.status = 400;
ctx.body = { code: 400, message: result.message };
return;
}
let conn = null;
try {
conn = await openConnection(params);
const tables = await listTables(conn, params);
ctx.body = { code: 0, message: 'success', data: { tables } };
} catch (err) {
ctx.status = 400;
ctx.body = { code: 400, message: `连接失败:${err && err.message ? err.message : '未知错误'}` };
} finally {
await closeQuietly(conn);
}
}
/**
* @summary 拉取指定表的字段
* @description 字段映射下拉使用;表名必须真实存在于该库,避免任意 SQL 访问
* @router get /api/shops/{id}/database-config/columns
* @apikey
* @request path integer *id 店铺主键ID
* @request query string *table 表名
* @response 200 baseResponse success
*/
async columns() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
const table = String(ctx.query.table || '').trim();
if (!table) {
ctx.status = 400;
ctx.body = { code: 400, message: 'table 为必填项' };
return;
}
if (!SAFE_IDENTIFIER.test(table)) {
ctx.status = 400;
ctx.body = { code: 400, message: '表名不合法' };
return;
}
const config = await this.findConfig(shop);
const params = this.resolveParams({}, config);
const paramsError = validateParams(params);
if (paramsError) {
ctx.status = 400;
ctx.body = { code: 400, message: `请先完善数据库连接信息:${paramsError}` };
return;
}
let conn = null;
try {
conn = await openConnection(params);
const tables = await listTables(conn, params);
if (!tables.some(row => row.name === table)) {
ctx.status = 400;
ctx.body = { code: 400, message: `库中不存在表 ${table}` };
return;
}
const columns = await listColumns(conn, params, table);
ctx.body = { code: 0, message: 'success', data: { table, columns } };
} catch (err) {
ctx.status = 400;
ctx.body = { code: 400, message: `连接失败:${err && err.message ? err.message : '未知错误'}` };
} finally {
await closeQuietly(conn);
}
}
/**
* @summary 拉取店铺商品(数据库店铺)
* @description 按配置的商品表与字段映射读取客户库,写入/更新店铺商品(供绑定系统商品/组合商品)
* @router post /api/shops/{id}/database-config/pull-products
* @apikey
* @request path integer *id 店铺主键ID
* @response 200 baseResponse success
*/
async pullProducts() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
try {
const result = await ctx.service.shopDatabaseSync.pullProducts(shop);
ctx.body = {
code: 0,
message: `拉取完成:共 ${result.total} 条(新增 ${result.created},更新 ${result.updated})`,
data: result,
};
} catch (err) {
ctx.status = err.status || 400;
ctx.body = { code: ctx.status, message: err.message };
}
}
/**
* @summary 拉取店铺订单(数据库店铺)
* @description 按配置的订单表/明细表与字段映射读取客户库明文订单,写入待审核订单
* @router post /api/shops/{id}/database-config/pull-orders
* @apikey
* @request path integer *id 店铺主键ID
* @request body pullShopDatabaseOrderRequest *body
* @response 200 baseResponse success
*/
async pullOrders() {
const { ctx } = this;
const shop = await this.loadShop(ctx.params.id);
if (!shop) return;
// 回看天数:0 表示不限下单时间(全量)
const days = Number(ctx.request.body && ctx.request.body.days);
const endTime = new Date();
const startTime = Number.isInteger(days) && days > 0
? new Date(endTime.getTime() - days * 24 * 3600 * 1000)
: null;
try {
const result = await ctx.service.shopDatabaseSync.pullOrders(shop, {
startTime,
endTime: startTime ? endTime : null,
});
const scope = startTime ? `最近 ${days} 天` : '全部';
const message = `拉取完成(${scope}):${result.orders} 单(新增 ${result.created},更新 ${result.updated}),明细 ${result.items} 条`
+ (result.skipped ? `;跳过无明细 ${result.skipped} 单` : '');
await ctx.service.shopDatabaseSync.markShopSynced(shop, new Date(), message);
ctx.body = { code: 0, message, data: { ...result, start_time: startTime, end_time: endTime } };
} catch (err) {
await ctx.service.shopDatabaseSync
.markShopSynced(shop, new Date(), `拉取失败:${err.message}`)
.catch(() => {});
ctx.status = err.status || 400;
ctx.body = { code: ctx.status, message: err.message };
}
}
}
module.exports = ShopDatabaseController;