liufumingren 993c7acd30 跨公司下载:云端中转成片(分片上传 + Range 下载)
A(查询机)和 B(仓库)不同网络、B 无公网 IP。B 出站领任务后把成片传回云端中转,
A 再从云端取回本地 —— 全程不需要任何一方有公网入口。

云端
- POST /api/web/edge/download      建任务(按主体选在线设备 → 下发 download)
- GET  /api/web/edge/download/:id  查进度(queued/uploading/done/failed)
- GET  /api/web/edge/download/:id/file  取成片,支持 Range(浏览器播放要拖动进度条)
- POST /api/scanner/tasks/:id/upload  设备分片上传(原始字节流,不是 base64)
- bodyParser 对该上传路由禁用:否则原始请求体会被解析掉
- 成片落 run/edge-files(run/ 属部署保留目录,不会被发布覆盖)
- 每次建任务顺手清理过期文件;元数据按主体校验,越权一律 404

桌面端
- AgentHost 新增 download 任务执行:worker 拉流合并 → 分片上传 → 传完删本地临时文件
- 失败时把 uploadTaskId 带回云端,立刻落 failed,查询机不用干等到超时
- ApiClient 新增建任务 / 查状态 / 取文件 / 分片上传
- PlaybackDownloadService 改为三级选路:
  LAN 设备管理服务 → 云端中转 → 本机直连

实测(真实 NVR)
  建任务 → 设备拉流 1.9MB → 上传 → 下载:7.5 秒,字节数一致,ftypisom
  Range: bytes=0-99 → HTTP 206 + Content-Range 正确

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-10-22 00:30:09 +08:00

464 lines
14 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

'use strict';
const { Service } = require('egg');
const crypto = require('crypto');
const fs = require('fs');
const fsp = require('fs/promises');
const path = require('path');
/**
* 云端「边缘任务」通道
*
* 背景:查询机(A)和打包仓库(B)在不同公司/不同网络,B 没有公网 IP。
* 所以**永远由 B 主动出站**领任务,云端从不反向连它 —— 这条通道复用的就是
* 设备心跳那条出站链路(`POST /api/scanner/tasks/poll` 挂住等任务)。
*
* 只做控制面:
* 云端入队 → 设备长轮询领取 → 设备回结果
*
* 队列用 Redis List(多实例共享,天然不依赖 sticky session),
* 结果放短 TTL 的 key,过期即视为「设备没回」。
*/
/** 超过这个时间没心跳就认为设备离线(与心跳间隔 60s 留足余量) */
const SCANNER_ONLINE_MS = 5 * 60 * 1000;
/** 任务与结果的保留时间:任务没人领就丢,结果没人取就丢 */
const TASK_TTL_SECONDS = 180;
const RESULT_TTL_SECONDS = 180;
/** 单次校验最多等多久(设备侧真拉一小段,正常 2~5 秒) */
const DEFAULT_WAIT_MS = 8000;
/** 同一主体最多试几台设备(多仓库时逐个试,连不上 NVR 的会被跳过) */
const MAX_DEVICES = 2;
/** 中转成片的保留时间:够查询机下完即可,过期自动清理 */
const FILE_TTL_MS = 2 * 60 * 60 * 1000;
/** 成片元数据 TTL(秒) */
const META_TTL_SECONDS = 2 * 60 * 60;
class EdgeTaskService extends Service {
get redis() {
return this.app.redis;
}
get prefix() {
return (this.config.cache && this.config.cache.prefix) || '';
}
queueKey(scannerId) {
return `${this.prefix}edge:task:${Number(scannerId)}`;
}
resultKey(taskId) {
return `${this.prefix}edge:result:${taskId}`;
}
// ==================== 跨网下载:文件中转 ====================
/** 成片存放目录(`run/` 本来就在部署保留目录里,不会被发布覆盖) */
get storageDir() {
return path.join(this.app.config.baseDir, 'run', 'edge-files');
}
/** 成片路径 */
filePath(taskId) {
return path.join(this.storageDir, `${sanitizeId(taskId)}.mp4`);
}
/** 上传中的临时文件(收完最后一个分片才改名,避免被下到半个文件) */
partPath(taskId) {
return `${this.filePath(taskId)}.part`;
}
metaKey(taskId) {
return `${this.prefix}edge:dl:${sanitizeId(taskId)}`;
}
async writeMeta(taskId, meta) {
await this.redis.setex(this.metaKey(taskId), META_TTL_SECONDS, JSON.stringify(meta));
}
async readMeta(taskId) {
const raw = await this.redis.get(this.metaKey(taskId));
if (!raw) return null;
try {
return JSON.parse(raw);
} catch (err) {
return null;
}
}
/**
* 创建跨网下载任务:算回放计划 → 下发给设备 → 等它把成片传上来
*
* 返回的 `taskId` 就是后续查状态 / 取文件的凭据,**只对同主体可见**。
*
* @param {object} params
* @param {number} params.subjectId
* @param {string} params.code 运单号
* @return {Promise<object>} { state, taskId?, message? }
*/
async createDownloadTask({ subjectId, code }) {
// 顺手清一遍过期成片:现场没人会来打扫,靠每次新建任务时收敛
await this.cleanupFiles().catch(() => 0);
const plan = await this.ctx.service.pack.query({ subjectId, code });
const streams = (plan && plan.streams) || [];
if (streams.length === 0) return { state: 'no-plan', message: '这个运单没有可下载的回放' };
const scanners = await this.onlineScanners(subjectId);
if (scanners.length === 0) return { state: 'no-device', message: '没有在线的扫码服务' };
// 凭据:同一台 NVR 只取一次(多路可能来自同一台)
const credentials = [];
const seen = new Set();
for (const stream of streams) {
const id = Number(stream.nvrId);
if (!id || seen.has(id)) continue;
seen.add(id);
const credential = await this.ctx.service.nvr.credentialOf(id, subjectId);
if (!credential.password) {
return { state: 'no-credential', message: `录像机「${credential.name}」没有保存密码` };
}
credentials.push({
nvrId: id,
name: credential.name,
host: credential.host,
port: credential.port,
username: credential.username,
password: credential.password,
});
}
const scanTime = (plan.records && plan.records[0] && plan.records[0].scanTime) || streams[0].startTime;
const payload = { code, streams, credentials, scanTime, rootDir: null };
let lastError = null;
for (const scanner of scanners) {
const target = crypto.randomUUID();
const task = await this.enqueue(scanner.id, 'download', Object.assign({}, payload, { uploadTaskId: target }));
const meta = {
taskId: task.id,
uploadTaskId: target,
subjectId: Number(subjectId),
code,
scannerId: Number(scanner.id),
scannerSn: scanner.sn,
state: 'queued',
size: 0,
createdAt: new Date().toISOString(),
error: null,
};
// 元数据挂在 **上传 id** 上:设备回传时用的是这个 id
await this.writeMeta(target, meta);
// 队列任务 id 也映射一份,方便按任务查
await this.writeMeta(task.id, meta);
const ready = await this.waitUpload(target, 120000);
if (ready && ready.state === 'done') {
return { state: 'ok', taskId: target, size: ready.size, scannerSn: scanner.sn };
}
lastError = (ready && ready.error) || '设备没有完成上传';
await this.markFailed(target, lastError);
}
return { state: 'no-device', message: lastError };
}
/** 设备侧:追加一个分片(顺序追加,最后一片触发改名) */
async appendChunk(uploadTaskId, stream, { last = false } = {}) {
const meta = await this.readMeta(uploadTaskId);
if (!meta) throw new Error('上传任务不存在或已过期');
await fsp.mkdir(this.storageDir, { recursive: true });
const part = this.partPath(uploadTaskId);
await new Promise((resolve, reject) => {
const out = fs.createWriteStream(part, { flags: 'a' });
stream.pipe(out);
out.on('finish', resolve);
out.on('error', reject);
stream.on('error', reject);
});
const size = (await fsp.stat(part)).size;
meta.size = size;
meta.state = last ? 'done' : 'uploading';
if (last) {
await fsp.rename(part, this.filePath(uploadTaskId));
meta.uploadedAt = new Date().toISOString();
}
await this.writeMeta(uploadTaskId, meta);
return meta;
}
/** 设备侧:上传失败(或下载失败)时把状态落到 failed,查询机不用干等 */
async markFailed(uploadTaskId, message) {
const meta = await this.readMeta(uploadTaskId);
if (!meta) return;
meta.state = 'failed';
meta.error = message || '下载失败';
await this.writeMeta(uploadTaskId, meta);
}
/** 查询机侧:等设备传完 */
async waitUpload(uploadTaskId, timeoutMs) {
const deadline = Date.now() + Math.max(0, Number(timeoutMs) || 0);
for (;;) {
const meta = await this.readMeta(uploadTaskId);
if (meta && (meta.state === 'done' || meta.state === 'failed')) return meta;
if (Date.now() >= deadline) return meta;
await sleep(300);
}
}
/**
* 清理过期成片(由定时任务或取文件后调用)
* 只删超过 FILE_TTL 的文件,正在被下载的不动。
*/
async cleanupFiles() {
let entries;
try {
entries = await fsp.readdir(this.storageDir);
} catch (err) {
return 0;
}
const now = Date.now();
let removed = 0;
for (const name of entries) {
const full = path.join(this.storageDir, name);
try {
const stat = await fsp.stat(full);
if (now - stat.mtimeMs > FILE_TTL_MS) {
await fsp.unlink(full);
removed++;
}
} catch (err) {
// 删不掉就下次再试
}
}
return removed;
}
/**
* 同主体里最近有心跳的扫码服务(可代取的候选,按心跳时间倒序)
* @param {number} subjectId
* @return {Promise<Array>} scanners 原始行
*/
async onlineScanners(subjectId) {
const Scanners = this.app.model.Scanner;
const threshold = new Date(Date.now() - SCANNER_ONLINE_MS);
return Scanners.findAll({
where: {
subjectId: Number(subjectId),
status: 1,
lastSeenAt: { [this.app.Sequelize.Op.gte]: threshold },
},
order: [ [ 'last_seen_at', 'DESC' ] ],
limit: MAX_DEVICES,
raw: true,
});
}
/**
* 入队一个任务
* @param {number} scannerId 目标设备
* @param {string} type 任务类型(check / download)
* @param {object} payload 任务参数
* @return {Promise<object>} 含 id 的任务对象
*/
async enqueue(scannerId, type, payload) {
const task = {
id: crypto.randomUUID(),
type,
payload,
createdAt: new Date().toISOString(),
};
const key = this.queueKey(scannerId);
await this.redis.lpush(key, JSON.stringify(task));
// 队列本身也给个 TTL:设备长期离线时不让任务无限堆积
await this.redis.expire(key, TASK_TTL_SECONDS);
return task;
}
/**
* 设备侧领取任务(长轮询)
*
* ⚠️ 不用 BLPOP:egg-redis 是共享连接,阻塞命令会把其它请求一起卡住。
* 这里用短间隔轮询,代价是一次 task 最多 200ms 的额外延迟,可以接受。
*
* @param {number} scannerId
* @param {number} waitMs 最长挂多久
* @return {Promise<object|null>} 没任务返回 null
*/
async poll(scannerId, waitMs) {
const deadline = Date.now() + Math.max(0, Number(waitMs) || 0);
const key = this.queueKey(scannerId);
for (;;) {
const raw = await this.redis.rpop(key);
if (raw) {
try {
return JSON.parse(raw);
} catch (err) {
this.ctx.logger.warn('[edge] 任务解析失败,丢弃:%s', err.message);
}
}
if (Date.now() >= deadline) return null;
await sleep(200);
}
}
/**
* 设备回结果
* @param {string} taskId
* @param {object} payload { ok, data | error }
*/
async submitResult(taskId, payload) {
const id = String(taskId || '').trim();
if (!id) return false;
await this.redis.setex(this.resultKey(id), RESULT_TTL_SECONDS, JSON.stringify(payload || {}));
return true;
}
/**
* 等结果(调用方是 web 请求,用户正等着)
* @param {string} taskId
* @param {number} timeoutMs
* @return {Promise<object|null>} 超时返回 null
*/
async waitResult(taskId, timeoutMs) {
const key = this.resultKey(taskId);
const deadline = Date.now() + Math.max(0, Number(timeoutMs) || DEFAULT_WAIT_MS);
for (;;) {
const raw = await this.redis.get(key);
if (raw) {
try {
return JSON.parse(raw);
} catch (err) {
return null;
}
}
if (Date.now() >= deadline) return null;
await sleep(150);
}
}
/**
* 问「这段录像有没有帧」
*
* 顺序:本主体在线的扫码服务 → 下发 check → 等结果;连不上 NVR 的换下一台。
*
* @param {object} params
* @param {number} params.subjectId
* @param {string} params.code 运单号
* @param {number} [params.waitMs]
* @return {Promise<object>} 见 controller 的注释
*/
async checkVideo({ subjectId, code, waitMs = DEFAULT_WAIT_MS }) {
const plan = await this.ctx.service.pack.query({ subjectId, code });
const streams = (plan && plan.streams) || [];
// 没有流计划:连"该查哪台 NVR 的哪个通道"都算不出来,无需惊动设备
if (streams.length === 0) {
return { state: 'no-plan', hasVideo: false };
}
const scanners = await this.onlineScanners(subjectId);
if (scanners.length === 0) {
return { state: 'no-device', hasVideo: null };
}
const stream = streams[0];
let credential;
try {
credential = await this.ctx.service.nvr.credentialOf(stream.nvrId, subjectId);
} catch (err) {
return { state: 'no-credential', hasVideo: null, message: err.message };
}
if (!credential.password) {
return { state: 'no-credential', hasVideo: null, message: '录像机没有保存密码' };
}
const payload = {
nvrId: stream.nvrId,
nvrHost: credential.host,
nvrPort: credential.port,
username: credential.username,
password: credential.password,
channelNo: stream.channelNo,
startTime: stream.startTime,
endTime: stream.endTime,
};
const startedAt = Date.now();
let lastError = null;
for (const scanner of scanners) {
let task;
try {
task = await this.enqueue(scanner.id, 'check', payload);
} catch (err) {
lastError = err.message;
continue;
}
const result = await this.waitResult(task.id, waitMs);
if (!result) {
lastError = '设备没有响应';
continue;
}
// 设备明确说连不上(或参数被拒):换下一台试
if (result.ok === false) {
lastError = result.error || '设备执行失败';
continue;
}
return {
state: 'ok',
hasVideo: Boolean(result.hasRecord),
frames: Number(result.frames) || 0,
source: result.source || '',
message: result.error || null,
scannerSn: scanner.sn,
elapsedMs: Date.now() - startedAt,
};
}
return { state: 'no-device', hasVideo: null, message: lastError };
}
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/** 只允许十六进制/短横线,防止拼路径穿越 */
function sanitizeId(value) {
const text = String(value || '').trim().toLowerCase();
return /^[a-z0-9-]{8,64}$/.test(text) ? text : crypto.createHash('sha1').update(text).digest('hex');
}
module.exports = EdgeTaskService;