packServer/desktop/Services/Agent/HikWorkerSupervisor.cs
liufumingren 290f29e547 跨公司取流:云端边缘任务通道 + 三级降级
场景:查询机 A 和打包仓库 B 在不同网络,B 没有公网 IP。
B 只能主动出站,所以云端从不反向连它 —— 复用设备出站链路做任务通道。

云端(backend)
- 新增 edge_task 服务:Redis List 做队列、短 TTL 存结果,
  多实例共享,不依赖 sticky session
- POST /api/scanner/tasks/poll:设备长轮询领任务(挂 20 秒,
  有任务立刻返回,没任务 204)
- POST /api/scanner/tasks/:id/result:设备回结果
- POST /api/web/edge/check:查询机问「这段有没有录像」,
  云端按主体选出在线扫码服务 → 下发 check → 等结果(默认 8 秒)
- 不阻塞共享 Redis 连接:任务用短间隔轮询而非 BLPOP
  (egg-redis 是共享连接,阻塞命令会卡住其它请求)

桌面端(desktop)
- AgentHost 新增边缘任务循环:长轮询领任务 → 交 worker 执行 → 回结果,
  与扫码链路完全解耦,异常只退避重试
- worker 新增 check 能力(复用 HikPlaybackService.CheckAsync)
- ApiClient 长轮询单独用不限时客户端(共享 HttpClient 的 15 秒超时会掐断)

三级降级(桌面端校验)
1. 局域网有设备管理服务 → 交给它(共用 NVR 会话池)
2. 没有 → 问云端;云端确认有录像时**隐藏预览、只显示下载**
   (跨网拉流成本高,短片段/下载更合适)
3. 云端也判断不了 → 本机自己校验(仓库内操作员的常规路径)

实测(真实 NVR 192.168.8.241 通道 34)
  设备心跳 → 领任务 → worker 校验 → 回结果 → 网页拿到结论
  端到端 2.17 秒(其中 NVR 拉流约 2 秒)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-10-22 00:26:27 +08:00

385 lines
12 KiB
C#
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Pipes;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace YuncangTrace.LocalService.Services;
/// <summary>
/// 主进程这一侧的 worker 管理:拉起、连接、转发任务、崩溃熔断。
///
/// 设计取舍:
/// - **懒启动**:第一个取流任务来才拉起 worker,空闲时零开销;
/// - **进程级隔离**:worker 崩了只影响取流,扫码/上报不受影响(这是它存在的唯一理由);
/// - **熔断**:如果某个视频一解码就崩,没有熔断会陷入"重启—崩溃"死循环把 CPU 打满。
/// </summary>
public sealed class HikWorkerSupervisor : IAsyncDisposable
{
/// <summary>熔断窗口与阈值(窗口内崩够次数就停自动重启)</summary>
private static readonly TimeSpan CrashWindow = TimeSpan.FromMinutes(5);
private const int CrashLimit = 3;
private static readonly TimeSpan BlockedDuration = TimeSpan.FromMinutes(10);
private readonly object _lifecycleGate = new();
private readonly SemaphoreSlim _connectLock = new(1, 1);
private readonly SemaphoreSlim _writeLock = new(1, 1);
private readonly ConcurrentDictionary<string, TaskCompletionSource<HikWorkerResponse>> _pending = new();
private readonly Queue<DateTime> _crashes = new();
private Process? _process;
private NamedPipeClientStream? _pipe;
private CancellationTokenSource? _readCts;
private Task? _readLoop;
private bool _disposing;
private DateTime _blockedUntil = DateTime.MinValue;
/// <summary>worker 状态变化(供 agent-status.json 更新)</summary>
public event Action? StateChanged;
public bool IsConnected => _pipe?.IsConnected == true;
public bool IsBlocked => DateTime.UtcNow < _blockedUntil;
public DateTime? BlockedUntil => IsBlocked ? _blockedUntil : null;
public int CrashCount
{
get { lock (_lifecycleGate) return _crashes.Count; }
}
public string? LastError { get; private set; }
/// <summary>自检:SDK / ffmpeg 是否就位(失败不影响扫码)</summary>
public async Task<JsonElement?> ProbeAsync(CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Probe, null, ct).ConfigureAwait(false);
/// <summary>派发一份下载计划</summary>
/// <param name="payload">包含 code / streams / credentials / scanTime / rootDir</param>
public async Task<JsonElement?> DownloadAsync(object payload, CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Download, payload, ct).ConfigureAwait(false);
/// <summary>校验某时段有没有录像(真拉一小段)</summary>
public async Task<JsonElement?> CheckAsync(object payload, CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Check, payload, ct).ConfigureAwait(false);
private async Task<JsonElement?> SendAsync(string op, object? payload, CancellationToken ct)
{
await EnsureConnectedAsync(ct).ConfigureAwait(false);
var pipe = _pipe ?? throw new InvalidOperationException("hik-worker 未连接");
var request = HikWorkerRequest.Create(op, payload);
var completion = new TaskCompletionSource<HikWorkerResponse>(TaskCreationOptions.RunContinuationsAsynchronously);
_pending[request.Id] = completion;
try
{
await _writeLock.WaitAsync(ct).ConfigureAwait(false);
try
{
await HikWorkerPipe.WriteAsync(pipe, request, ct).ConfigureAwait(false);
}
finally
{
_writeLock.Release();
}
var response = await completion.Task.WaitAsync(ct).ConfigureAwait(false);
if (!response.Ok) throw new InvalidOperationException(response.Error ?? "hik-worker 执行失败");
return response.Data;
}
finally
{
_pending.TryRemove(request.Id, out _);
}
}
/// <summary>确保 worker 已拉起且管道已连接(失败会抛出可直接展示的原因)</summary>
private async Task EnsureConnectedAsync(CancellationToken ct)
{
if (IsConnected) return;
if (IsBlocked)
{
throw new InvalidOperationException(
$"取流服务连续崩溃已暂停({_blockedUntil:HH:mm:ss} 后自动重试):{LastError}");
}
await _connectLock.WaitAsync(ct).ConfigureAwait(false);
try
{
if (IsConnected) return;
if (IsBlocked) throw new InvalidOperationException($"取流服务连续崩溃已暂停:{LastError}");
StartProcess();
var pipe = await ConnectPipeAsync(ct).ConfigureAwait(false);
_pipe = pipe;
_readCts = new CancellationTokenSource();
_readLoop = Task.Run(() => ReadLoopAsync(pipe, _readCts.Token), CancellationToken.None);
AppLog.Write("[hik-worker] 已连接");
StateChanged?.Invoke();
}
catch (Exception ex)
{
KillProcess();
RecordFailure(ex.Message);
throw;
}
finally
{
_connectLock.Release();
}
}
private void StartProcess()
{
var existing = _process;
if (existing is not null && !existing.HasExited) return;
existing?.Dispose();
_process = null;
var exe = Environment.ProcessPath
?? throw new InvalidOperationException("取不到当前程序路径,无法启动取流进程");
var info = new ProcessStartInfo(exe, "--hik-worker")
{
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = Path.GetDirectoryName(exe) ?? AppContext.BaseDirectory,
RedirectStandardError = true,
RedirectStandardOutput = true,
};
var process = new Process { StartInfo = info, EnableRaisingEvents = true };
process.Exited += (_, _) => OnWorkerGone($"进程退出(code={(TryGetExitCode(process)?.ToString() ?? "?")})");
process.ErrorDataReceived += (_, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stderr] {e.Data}");
};
process.OutputDataReceived += (_, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stdout] {e.Data}");
};
process.Start();
// 必须有读事件订阅,否则重定向缓冲填满会阻塞子进程
process.BeginErrorReadLine();
process.BeginOutputReadLine();
_process = process;
AppLog.Write($"[hik-worker] 已拉起(pid={process.Id})");
}
private static int? TryGetExitCode(Process process)
{
try
{
return process.ExitCode;
}
catch
{
return null;
}
}
private async Task<NamedPipeClientStream> ConnectPipeAsync(CancellationToken ct)
{
Exception? last = null;
// worker 从启动到管道可用通常 < 500ms;给到 6 秒足够覆盖冷启动与磁盘慢的情况
for (var attempt = 0; attempt < 30; attempt++)
{
ct.ThrowIfCancellationRequested();
if (_process is { HasExited: true })
{
throw new InvalidOperationException(
$"取流进程启动后立即退出(code={TryGetExitCode(_process)?.ToString() ?? "?"}),"
+ "多半是缺 HCNetSDK 或 native 目录未复制到服务目录");
}
var pipe = new NamedPipeClientStream(".", HikWorkerPipe.PipeName,
PipeDirection.InOut, PipeOptions.Asynchronous);
try
{
await pipe.ConnectAsync(200, ct).ConfigureAwait(false);
return pipe;
}
catch (OperationCanceledException)
{
pipe.Dispose();
throw;
}
catch (Exception ex)
{
pipe.Dispose();
last = ex;
await Task.Delay(200, ct).ConfigureAwait(false);
}
}
throw new TimeoutException($"连接取流进程超时:{last?.Message}");
}
/// <summary>读循环:把响应按 Id 交回对应请求;断开即视为 worker 已死</summary>
private async Task ReadLoopAsync(Stream pipe, CancellationToken ct)
{
var reason = "连接已关闭";
try
{
while (!ct.IsCancellationRequested)
{
var response = await HikWorkerPipe.ReadAsync<HikWorkerResponse>(pipe, ct).ConfigureAwait(false);
if (response is null) break;
if (_pending.TryRemove(response.Id, out var completion))
{
completion.TrySetResult(response);
}
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
reason = ex.Message;
}
if (_disposing) return;
OnWorkerGone(reason);
}
private void OnWorkerGone(string reason)
{
if (_disposing) return;
CleanupConnection();
KillProcess();
RecordFailure(reason);
}
/// <summary>记一次失败并判断是否熔断(进程退出和启动失败走同一套阈值)</summary>
private void RecordFailure(string reason)
{
lock (_lifecycleGate)
{
if (_disposing) return;
LastError = reason;
var now = DateTime.UtcNow;
_crashes.Enqueue(now);
while (_crashes.Count > 0 && now - _crashes.Peek() > CrashWindow) _crashes.Dequeue();
if (_crashes.Count >= CrashLimit)
{
_blockedUntil = now.Add(BlockedDuration);
AppLog.Write($"[hik-worker] {CrashWindow.TotalMinutes:0} 分钟内失败 {_crashes.Count} 次 → "
+ $"熔断 {BlockedDuration.TotalMinutes:0} 分钟(扫码不受影响):{reason}");
}
else
{
AppLog.Write($"[hik-worker] 失败({reason}),下次任务到来时重试");
}
}
StateChanged?.Invoke();
}
private void CleanupConnection()
{
// 断开连接前先把挂起的请求全部失败掉,避免调用方一直等
foreach (var (_, completion) in _pending)
{
completion.TrySetException(new IOException("取流进程已退出"));
}
_pending.Clear();
try
{
_readCts?.Cancel();
}
catch
{
// 忽略
}
_readCts?.Dispose();
_readCts = null;
_readLoop = null;
try
{
_pipe?.Dispose();
}
catch
{
// 忽略
}
_pipe = null;
}
private void KillProcess()
{
var process = _process;
_process = null;
if (process is null) return;
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
process.WaitForExit(3000);
}
}
catch (Exception ex)
{
AppLog.Write($"[hik-worker] 结束进程失败:{ex.Message}");
}
finally
{
process.Dispose();
}
}
public async ValueTask DisposeAsync()
{
lock (_lifecycleGate)
{
if (_disposing) return;
_disposing = true;
}
// 先请求优雅退出(让 worker 有机会收尾),失败再强杀
if (IsConnected)
{
try
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
await SendAsync(HikWorkerOps.Shutdown, null, timeout.Token).ConfigureAwait(false);
}
catch
{
// 忽略:下面还会 Kill
}
}
CleanupConnection();
KillProcess();
_connectLock.Dispose();
_writeLock.Dispose();
}
}