场景:查询机 A 和打包仓库 B 在不同网络,B 没有公网 IP。 B 只能主动出站,所以云端从不反向连它 —— 复用设备出站链路做任务通道。 云端(backend) - 新增 edge_task 服务:Redis List 做队列、短 TTL 存结果, 多实例共享,不依赖 sticky session - POST /api/scanner/tasks/poll:设备长轮询领任务(挂 20 秒, 有任务立刻返回,没任务 204) - POST /api/scanner/tasks/:id/result:设备回结果 - POST /api/web/edge/check:查询机问「这段有没有录像」, 云端按主体选出在线扫码服务 → 下发 check → 等结果(默认 8 秒) - 不阻塞共享 Redis 连接:任务用短间隔轮询而非 BLPOP (egg-redis 是共享连接,阻塞命令会卡住其它请求) 桌面端(desktop) - AgentHost 新增边缘任务循环:长轮询领任务 → 交 worker 执行 → 回结果, 与扫码链路完全解耦,异常只退避重试 - worker 新增 check 能力(复用 HikPlaybackService.CheckAsync) - ApiClient 长轮询单独用不限时客户端(共享 HttpClient 的 15 秒超时会掐断) 三级降级(桌面端校验) 1. 局域网有设备管理服务 → 交给它(共用 NVR 会话池) 2. 没有 → 问云端;云端确认有录像时**隐藏预览、只显示下载** (跨网拉流成本高,短片段/下载更合适) 3. 云端也判断不了 → 本机自己校验(仓库内操作员的常规路径) 实测(真实 NVR 192.168.8.241 通道 34) 设备心跳 → 领任务 → worker 校验 → 回结果 → 网页拿到结论 端到端 2.17 秒(其中 NVR 拉流约 2 秒) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
385 lines
12 KiB
C#
385 lines
12 KiB
C#
using System;
|
||
using System.Collections.Concurrent;
|
||
using System.Collections.Generic;
|
||
using System.Diagnostics;
|
||
using System.IO;
|
||
using System.IO.Pipes;
|
||
using System.Text.Json;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
|
||
namespace YuncangTrace.LocalService.Services;
|
||
|
||
/// <summary>
|
||
/// 主进程这一侧的 worker 管理:拉起、连接、转发任务、崩溃熔断。
|
||
///
|
||
/// 设计取舍:
|
||
/// - **懒启动**:第一个取流任务来才拉起 worker,空闲时零开销;
|
||
/// - **进程级隔离**:worker 崩了只影响取流,扫码/上报不受影响(这是它存在的唯一理由);
|
||
/// - **熔断**:如果某个视频一解码就崩,没有熔断会陷入"重启—崩溃"死循环把 CPU 打满。
|
||
/// </summary>
|
||
public sealed class HikWorkerSupervisor : IAsyncDisposable
|
||
{
|
||
/// <summary>熔断窗口与阈值(窗口内崩够次数就停自动重启)</summary>
|
||
private static readonly TimeSpan CrashWindow = TimeSpan.FromMinutes(5);
|
||
private const int CrashLimit = 3;
|
||
private static readonly TimeSpan BlockedDuration = TimeSpan.FromMinutes(10);
|
||
|
||
private readonly object _lifecycleGate = new();
|
||
private readonly SemaphoreSlim _connectLock = new(1, 1);
|
||
private readonly SemaphoreSlim _writeLock = new(1, 1);
|
||
private readonly ConcurrentDictionary<string, TaskCompletionSource<HikWorkerResponse>> _pending = new();
|
||
private readonly Queue<DateTime> _crashes = new();
|
||
|
||
private Process? _process;
|
||
private NamedPipeClientStream? _pipe;
|
||
private CancellationTokenSource? _readCts;
|
||
private Task? _readLoop;
|
||
private bool _disposing;
|
||
private DateTime _blockedUntil = DateTime.MinValue;
|
||
|
||
/// <summary>worker 状态变化(供 agent-status.json 更新)</summary>
|
||
public event Action? StateChanged;
|
||
|
||
public bool IsConnected => _pipe?.IsConnected == true;
|
||
|
||
public bool IsBlocked => DateTime.UtcNow < _blockedUntil;
|
||
|
||
public DateTime? BlockedUntil => IsBlocked ? _blockedUntil : null;
|
||
|
||
public int CrashCount
|
||
{
|
||
get { lock (_lifecycleGate) return _crashes.Count; }
|
||
}
|
||
|
||
public string? LastError { get; private set; }
|
||
|
||
/// <summary>自检:SDK / ffmpeg 是否就位(失败不影响扫码)</summary>
|
||
public async Task<JsonElement?> ProbeAsync(CancellationToken ct = default)
|
||
=> await SendAsync(HikWorkerOps.Probe, null, ct).ConfigureAwait(false);
|
||
|
||
/// <summary>派发一份下载计划</summary>
|
||
/// <param name="payload">包含 code / streams / credentials / scanTime / rootDir</param>
|
||
public async Task<JsonElement?> DownloadAsync(object payload, CancellationToken ct = default)
|
||
=> await SendAsync(HikWorkerOps.Download, payload, ct).ConfigureAwait(false);
|
||
|
||
/// <summary>校验某时段有没有录像(真拉一小段)</summary>
|
||
public async Task<JsonElement?> CheckAsync(object payload, CancellationToken ct = default)
|
||
=> await SendAsync(HikWorkerOps.Check, payload, ct).ConfigureAwait(false);
|
||
|
||
private async Task<JsonElement?> SendAsync(string op, object? payload, CancellationToken ct)
|
||
{
|
||
await EnsureConnectedAsync(ct).ConfigureAwait(false);
|
||
|
||
var pipe = _pipe ?? throw new InvalidOperationException("hik-worker 未连接");
|
||
var request = HikWorkerRequest.Create(op, payload);
|
||
var completion = new TaskCompletionSource<HikWorkerResponse>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||
_pending[request.Id] = completion;
|
||
|
||
try
|
||
{
|
||
await _writeLock.WaitAsync(ct).ConfigureAwait(false);
|
||
try
|
||
{
|
||
await HikWorkerPipe.WriteAsync(pipe, request, ct).ConfigureAwait(false);
|
||
}
|
||
finally
|
||
{
|
||
_writeLock.Release();
|
||
}
|
||
|
||
var response = await completion.Task.WaitAsync(ct).ConfigureAwait(false);
|
||
if (!response.Ok) throw new InvalidOperationException(response.Error ?? "hik-worker 执行失败");
|
||
return response.Data;
|
||
}
|
||
finally
|
||
{
|
||
_pending.TryRemove(request.Id, out _);
|
||
}
|
||
}
|
||
|
||
/// <summary>确保 worker 已拉起且管道已连接(失败会抛出可直接展示的原因)</summary>
|
||
private async Task EnsureConnectedAsync(CancellationToken ct)
|
||
{
|
||
if (IsConnected) return;
|
||
|
||
if (IsBlocked)
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"取流服务连续崩溃已暂停({_blockedUntil:HH:mm:ss} 后自动重试):{LastError}");
|
||
}
|
||
|
||
await _connectLock.WaitAsync(ct).ConfigureAwait(false);
|
||
try
|
||
{
|
||
if (IsConnected) return;
|
||
if (IsBlocked) throw new InvalidOperationException($"取流服务连续崩溃已暂停:{LastError}");
|
||
|
||
StartProcess();
|
||
var pipe = await ConnectPipeAsync(ct).ConfigureAwait(false);
|
||
|
||
_pipe = pipe;
|
||
_readCts = new CancellationTokenSource();
|
||
_readLoop = Task.Run(() => ReadLoopAsync(pipe, _readCts.Token), CancellationToken.None);
|
||
|
||
AppLog.Write("[hik-worker] 已连接");
|
||
StateChanged?.Invoke();
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
KillProcess();
|
||
RecordFailure(ex.Message);
|
||
throw;
|
||
}
|
||
finally
|
||
{
|
||
_connectLock.Release();
|
||
}
|
||
}
|
||
|
||
private void StartProcess()
|
||
{
|
||
var existing = _process;
|
||
if (existing is not null && !existing.HasExited) return;
|
||
|
||
existing?.Dispose();
|
||
_process = null;
|
||
|
||
var exe = Environment.ProcessPath
|
||
?? throw new InvalidOperationException("取不到当前程序路径,无法启动取流进程");
|
||
|
||
var info = new ProcessStartInfo(exe, "--hik-worker")
|
||
{
|
||
UseShellExecute = false,
|
||
CreateNoWindow = true,
|
||
WorkingDirectory = Path.GetDirectoryName(exe) ?? AppContext.BaseDirectory,
|
||
RedirectStandardError = true,
|
||
RedirectStandardOutput = true,
|
||
};
|
||
|
||
var process = new Process { StartInfo = info, EnableRaisingEvents = true };
|
||
process.Exited += (_, _) => OnWorkerGone($"进程退出(code={(TryGetExitCode(process)?.ToString() ?? "?")})");
|
||
process.ErrorDataReceived += (_, e) =>
|
||
{
|
||
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stderr] {e.Data}");
|
||
};
|
||
process.OutputDataReceived += (_, e) =>
|
||
{
|
||
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stdout] {e.Data}");
|
||
};
|
||
|
||
process.Start();
|
||
// 必须有读事件订阅,否则重定向缓冲填满会阻塞子进程
|
||
process.BeginErrorReadLine();
|
||
process.BeginOutputReadLine();
|
||
|
||
_process = process;
|
||
AppLog.Write($"[hik-worker] 已拉起(pid={process.Id})");
|
||
}
|
||
|
||
private static int? TryGetExitCode(Process process)
|
||
{
|
||
try
|
||
{
|
||
return process.ExitCode;
|
||
}
|
||
catch
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private async Task<NamedPipeClientStream> ConnectPipeAsync(CancellationToken ct)
|
||
{
|
||
Exception? last = null;
|
||
|
||
// worker 从启动到管道可用通常 < 500ms;给到 6 秒足够覆盖冷启动与磁盘慢的情况
|
||
for (var attempt = 0; attempt < 30; attempt++)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
|
||
if (_process is { HasExited: true })
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"取流进程启动后立即退出(code={TryGetExitCode(_process)?.ToString() ?? "?"}),"
|
||
+ "多半是缺 HCNetSDK 或 native 目录未复制到服务目录");
|
||
}
|
||
|
||
var pipe = new NamedPipeClientStream(".", HikWorkerPipe.PipeName,
|
||
PipeDirection.InOut, PipeOptions.Asynchronous);
|
||
try
|
||
{
|
||
await pipe.ConnectAsync(200, ct).ConfigureAwait(false);
|
||
return pipe;
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
pipe.Dispose();
|
||
throw;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
pipe.Dispose();
|
||
last = ex;
|
||
await Task.Delay(200, ct).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
throw new TimeoutException($"连接取流进程超时:{last?.Message}");
|
||
}
|
||
|
||
/// <summary>读循环:把响应按 Id 交回对应请求;断开即视为 worker 已死</summary>
|
||
private async Task ReadLoopAsync(Stream pipe, CancellationToken ct)
|
||
{
|
||
var reason = "连接已关闭";
|
||
try
|
||
{
|
||
while (!ct.IsCancellationRequested)
|
||
{
|
||
var response = await HikWorkerPipe.ReadAsync<HikWorkerResponse>(pipe, ct).ConfigureAwait(false);
|
||
if (response is null) break;
|
||
|
||
if (_pending.TryRemove(response.Id, out var completion))
|
||
{
|
||
completion.TrySetResult(response);
|
||
}
|
||
}
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
return;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
reason = ex.Message;
|
||
}
|
||
|
||
if (_disposing) return;
|
||
OnWorkerGone(reason);
|
||
}
|
||
|
||
private void OnWorkerGone(string reason)
|
||
{
|
||
if (_disposing) return;
|
||
|
||
CleanupConnection();
|
||
KillProcess();
|
||
RecordFailure(reason);
|
||
}
|
||
|
||
/// <summary>记一次失败并判断是否熔断(进程退出和启动失败走同一套阈值)</summary>
|
||
private void RecordFailure(string reason)
|
||
{
|
||
lock (_lifecycleGate)
|
||
{
|
||
if (_disposing) return;
|
||
|
||
LastError = reason;
|
||
|
||
var now = DateTime.UtcNow;
|
||
_crashes.Enqueue(now);
|
||
while (_crashes.Count > 0 && now - _crashes.Peek() > CrashWindow) _crashes.Dequeue();
|
||
|
||
if (_crashes.Count >= CrashLimit)
|
||
{
|
||
_blockedUntil = now.Add(BlockedDuration);
|
||
AppLog.Write($"[hik-worker] {CrashWindow.TotalMinutes:0} 分钟内失败 {_crashes.Count} 次 → "
|
||
+ $"熔断 {BlockedDuration.TotalMinutes:0} 分钟(扫码不受影响):{reason}");
|
||
}
|
||
else
|
||
{
|
||
AppLog.Write($"[hik-worker] 失败({reason}),下次任务到来时重试");
|
||
}
|
||
}
|
||
|
||
StateChanged?.Invoke();
|
||
}
|
||
|
||
private void CleanupConnection()
|
||
{
|
||
// 断开连接前先把挂起的请求全部失败掉,避免调用方一直等
|
||
foreach (var (_, completion) in _pending)
|
||
{
|
||
completion.TrySetException(new IOException("取流进程已退出"));
|
||
}
|
||
_pending.Clear();
|
||
|
||
try
|
||
{
|
||
_readCts?.Cancel();
|
||
}
|
||
catch
|
||
{
|
||
// 忽略
|
||
}
|
||
|
||
_readCts?.Dispose();
|
||
_readCts = null;
|
||
_readLoop = null;
|
||
|
||
try
|
||
{
|
||
_pipe?.Dispose();
|
||
}
|
||
catch
|
||
{
|
||
// 忽略
|
||
}
|
||
|
||
_pipe = null;
|
||
}
|
||
|
||
private void KillProcess()
|
||
{
|
||
var process = _process;
|
||
_process = null;
|
||
if (process is null) return;
|
||
|
||
try
|
||
{
|
||
if (!process.HasExited)
|
||
{
|
||
process.Kill(entireProcessTree: true);
|
||
process.WaitForExit(3000);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
AppLog.Write($"[hik-worker] 结束进程失败:{ex.Message}");
|
||
}
|
||
finally
|
||
{
|
||
process.Dispose();
|
||
}
|
||
}
|
||
|
||
public async ValueTask DisposeAsync()
|
||
{
|
||
lock (_lifecycleGate)
|
||
{
|
||
if (_disposing) return;
|
||
_disposing = true;
|
||
}
|
||
|
||
// 先请求优雅退出(让 worker 有机会收尾),失败再强杀
|
||
if (IsConnected)
|
||
{
|
||
try
|
||
{
|
||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
|
||
await SendAsync(HikWorkerOps.Shutdown, null, timeout.Token).ConfigureAwait(false);
|
||
}
|
||
catch
|
||
{
|
||
// 忽略:下面还会 Kill
|
||
}
|
||
}
|
||
|
||
CleanupConnection();
|
||
KillProcess();
|
||
|
||
_connectLock.Dispose();
|
||
_writeLock.Dispose();
|
||
}
|
||
}
|