packServer/desktop/Services/Agent/HikWorkerSupervisor.cs
liufumingren 46e4f12802 desktop: 取流子进程隔离 + 局域网询问应答 + 跨网下载
把海康取流从扫码服务主进程拆到 --hik-worker 子进程,原生 SDK 崩溃
不再连带扫码收码;新增局域网询问-应答与跨网下载通道。

隔离与守护
- 新增 HikWorker(子进程):独占 HCNetSDK 与 NVR 会话池
- 新增 HikWorkerSupervisor(主进程):懒启动、管道通信、
  5 分钟内失败 3 次熔断 10 分钟
- 管道协议为 [4 字节长度][JSON],进程退出即知,不占端口
- ServiceInstaller 复制 native(HCNetSDK/ffmpeg),先删后拷,
  避免 SDK 升级残留旧 DLL;安装时放行 TCP 17311 / UDP 17310

局域网询问-应答(不是主动公告)
- UDP 17310 收询问,只回同 subjectId;带了 nvrHosts 时先实测可达,
  连不上就不应答,让查询机继续走本机
- 应答带 TCP 17311,查询机一次连接即可判定是否同 LAN

跨网下载
- TCP 17311 提供 /health、/probe、/download、/download/{id}
- 除 /health 外均要求 X-Edge-Subject 且与配置主体一致
- 成片在传输完成后立即删除,另有 done 10 分钟 / failed 15 分钟
  TTL 兜底,避免现场磁盘被临时文件吃满

桌面端选路
- DownloadAsync 先探 NVR 8000(600ms):通则沿用原有本机直连,
  不通才广播找取流服务,取回后落到本机录像目录
- DownloadByPlanAsync 拆分:不查库、不需要用户令牌,
  供只有设备令牌的服务端复用

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-10-21 22:48:09 +08:00

381 lines
12 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Pipes;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace YuncangTrace.LocalService.Services;
/// <summary>
/// 主进程这一侧的 worker 管理:拉起、连接、转发任务、崩溃熔断。
///
/// 设计取舍:
/// - **懒启动**:第一个取流任务来才拉起 worker,空闲时零开销;
/// - **进程级隔离**:worker 崩了只影响取流,扫码/上报不受影响(这是它存在的唯一理由);
/// - **熔断**:如果某个视频一解码就崩,没有熔断会陷入"重启—崩溃"死循环把 CPU 打满。
/// </summary>
public sealed class HikWorkerSupervisor : IAsyncDisposable
{
/// <summary>熔断窗口与阈值(窗口内崩够次数就停自动重启)</summary>
private static readonly TimeSpan CrashWindow = TimeSpan.FromMinutes(5);
private const int CrashLimit = 3;
private static readonly TimeSpan BlockedDuration = TimeSpan.FromMinutes(10);
private readonly object _lifecycleGate = new();
private readonly SemaphoreSlim _connectLock = new(1, 1);
private readonly SemaphoreSlim _writeLock = new(1, 1);
private readonly ConcurrentDictionary<string, TaskCompletionSource<HikWorkerResponse>> _pending = new();
private readonly Queue<DateTime> _crashes = new();
private Process? _process;
private NamedPipeClientStream? _pipe;
private CancellationTokenSource? _readCts;
private Task? _readLoop;
private bool _disposing;
private DateTime _blockedUntil = DateTime.MinValue;
/// <summary>worker 状态变化(供 agent-status.json 更新)</summary>
public event Action? StateChanged;
public bool IsConnected => _pipe?.IsConnected == true;
public bool IsBlocked => DateTime.UtcNow < _blockedUntil;
public DateTime? BlockedUntil => IsBlocked ? _blockedUntil : null;
public int CrashCount
{
get { lock (_lifecycleGate) return _crashes.Count; }
}
public string? LastError { get; private set; }
/// <summary>自检:SDK / ffmpeg 是否就位(失败不影响扫码)</summary>
public async Task<JsonElement?> ProbeAsync(CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Probe, null, ct).ConfigureAwait(false);
/// <summary>派发一份下载计划</summary>
/// <param name="payload">包含 code / streams / credentials / scanTime / rootDir</param>
public async Task<JsonElement?> DownloadAsync(object payload, CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Download, payload, ct).ConfigureAwait(false);
private async Task<JsonElement?> SendAsync(string op, object? payload, CancellationToken ct)
{
await EnsureConnectedAsync(ct).ConfigureAwait(false);
var pipe = _pipe ?? throw new InvalidOperationException("hik-worker 未连接");
var request = HikWorkerRequest.Create(op, payload);
var completion = new TaskCompletionSource<HikWorkerResponse>(TaskCreationOptions.RunContinuationsAsynchronously);
_pending[request.Id] = completion;
try
{
await _writeLock.WaitAsync(ct).ConfigureAwait(false);
try
{
await HikWorkerPipe.WriteAsync(pipe, request, ct).ConfigureAwait(false);
}
finally
{
_writeLock.Release();
}
var response = await completion.Task.WaitAsync(ct).ConfigureAwait(false);
if (!response.Ok) throw new InvalidOperationException(response.Error ?? "hik-worker 执行失败");
return response.Data;
}
finally
{
_pending.TryRemove(request.Id, out _);
}
}
/// <summary>确保 worker 已拉起且管道已连接(失败会抛出可直接展示的原因)</summary>
private async Task EnsureConnectedAsync(CancellationToken ct)
{
if (IsConnected) return;
if (IsBlocked)
{
throw new InvalidOperationException(
$"取流服务连续崩溃已暂停({_blockedUntil:HH:mm:ss} 后自动重试):{LastError}");
}
await _connectLock.WaitAsync(ct).ConfigureAwait(false);
try
{
if (IsConnected) return;
if (IsBlocked) throw new InvalidOperationException($"取流服务连续崩溃已暂停:{LastError}");
StartProcess();
var pipe = await ConnectPipeAsync(ct).ConfigureAwait(false);
_pipe = pipe;
_readCts = new CancellationTokenSource();
_readLoop = Task.Run(() => ReadLoopAsync(pipe, _readCts.Token), CancellationToken.None);
AppLog.Write("[hik-worker] 已连接");
StateChanged?.Invoke();
}
catch (Exception ex)
{
KillProcess();
RecordFailure(ex.Message);
throw;
}
finally
{
_connectLock.Release();
}
}
private void StartProcess()
{
var existing = _process;
if (existing is not null && !existing.HasExited) return;
existing?.Dispose();
_process = null;
var exe = Environment.ProcessPath
?? throw new InvalidOperationException("取不到当前程序路径,无法启动取流进程");
var info = new ProcessStartInfo(exe, "--hik-worker")
{
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = Path.GetDirectoryName(exe) ?? AppContext.BaseDirectory,
RedirectStandardError = true,
RedirectStandardOutput = true,
};
var process = new Process { StartInfo = info, EnableRaisingEvents = true };
process.Exited += (_, _) => OnWorkerGone($"进程退出(code={(TryGetExitCode(process)?.ToString() ?? "?")})");
process.ErrorDataReceived += (_, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stderr] {e.Data}");
};
process.OutputDataReceived += (_, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stdout] {e.Data}");
};
process.Start();
// 必须有读事件订阅,否则重定向缓冲填满会阻塞子进程
process.BeginErrorReadLine();
process.BeginOutputReadLine();
_process = process;
AppLog.Write($"[hik-worker] 已拉起(pid={process.Id})");
}
private static int? TryGetExitCode(Process process)
{
try
{
return process.ExitCode;
}
catch
{
return null;
}
}
private async Task<NamedPipeClientStream> ConnectPipeAsync(CancellationToken ct)
{
Exception? last = null;
// worker 从启动到管道可用通常 < 500ms;给到 6 秒足够覆盖冷启动与磁盘慢的情况
for (var attempt = 0; attempt < 30; attempt++)
{
ct.ThrowIfCancellationRequested();
if (_process is { HasExited: true })
{
throw new InvalidOperationException(
$"取流进程启动后立即退出(code={TryGetExitCode(_process)?.ToString() ?? "?"}),"
+ "多半是缺 HCNetSDK 或 native 目录未复制到服务目录");
}
var pipe = new NamedPipeClientStream(".", HikWorkerPipe.PipeName,
PipeDirection.InOut, PipeOptions.Asynchronous);
try
{
await pipe.ConnectAsync(200, ct).ConfigureAwait(false);
return pipe;
}
catch (OperationCanceledException)
{
pipe.Dispose();
throw;
}
catch (Exception ex)
{
pipe.Dispose();
last = ex;
await Task.Delay(200, ct).ConfigureAwait(false);
}
}
throw new TimeoutException($"连接取流进程超时:{last?.Message}");
}
/// <summary>读循环:把响应按 Id 交回对应请求;断开即视为 worker 已死</summary>
private async Task ReadLoopAsync(Stream pipe, CancellationToken ct)
{
var reason = "连接已关闭";
try
{
while (!ct.IsCancellationRequested)
{
var response = await HikWorkerPipe.ReadAsync<HikWorkerResponse>(pipe, ct).ConfigureAwait(false);
if (response is null) break;
if (_pending.TryRemove(response.Id, out var completion))
{
completion.TrySetResult(response);
}
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
reason = ex.Message;
}
if (_disposing) return;
OnWorkerGone(reason);
}
private void OnWorkerGone(string reason)
{
if (_disposing) return;
CleanupConnection();
KillProcess();
RecordFailure(reason);
}
/// <summary>记一次失败并判断是否熔断(进程退出和启动失败走同一套阈值)</summary>
private void RecordFailure(string reason)
{
lock (_lifecycleGate)
{
if (_disposing) return;
LastError = reason;
var now = DateTime.UtcNow;
_crashes.Enqueue(now);
while (_crashes.Count > 0 && now - _crashes.Peek() > CrashWindow) _crashes.Dequeue();
if (_crashes.Count >= CrashLimit)
{
_blockedUntil = now.Add(BlockedDuration);
AppLog.Write($"[hik-worker] {CrashWindow.TotalMinutes:0} 分钟内失败 {_crashes.Count} 次 → "
+ $"熔断 {BlockedDuration.TotalMinutes:0} 分钟(扫码不受影响):{reason}");
}
else
{
AppLog.Write($"[hik-worker] 失败({reason}),下次任务到来时重试");
}
}
StateChanged?.Invoke();
}
private void CleanupConnection()
{
// 断开连接前先把挂起的请求全部失败掉,避免调用方一直等
foreach (var (_, completion) in _pending)
{
completion.TrySetException(new IOException("取流进程已退出"));
}
_pending.Clear();
try
{
_readCts?.Cancel();
}
catch
{
// 忽略
}
_readCts?.Dispose();
_readCts = null;
_readLoop = null;
try
{
_pipe?.Dispose();
}
catch
{
// 忽略
}
_pipe = null;
}
private void KillProcess()
{
var process = _process;
_process = null;
if (process is null) return;
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
process.WaitForExit(3000);
}
}
catch (Exception ex)
{
AppLog.Write($"[hik-worker] 结束进程失败:{ex.Message}");
}
finally
{
process.Dispose();
}
}
public async ValueTask DisposeAsync()
{
lock (_lifecycleGate)
{
if (_disposing) return;
_disposing = true;
}
// 先请求优雅退出(让 worker 有机会收尾),失败再强杀
if (IsConnected)
{
try
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
await SendAsync(HikWorkerOps.Shutdown, null, timeout.Token).ConfigureAwait(false);
}
catch
{
// 忽略:下面还会 Kill
}
}
CleanupConnection();
KillProcess();
_connectLock.Dispose();
_writeLock.Dispose();
}
}