把海康取流从扫码服务主进程拆到 --hik-worker 子进程,原生 SDK 崩溃
不再连带扫码收码;新增局域网询问-应答与跨网下载通道。
隔离与守护
- 新增 HikWorker(子进程):独占 HCNetSDK 与 NVR 会话池
- 新增 HikWorkerSupervisor(主进程):懒启动、管道通信、
5 分钟内失败 3 次熔断 10 分钟
- 管道协议为 [4 字节长度][JSON],进程退出即知,不占端口
- ServiceInstaller 复制 native(HCNetSDK/ffmpeg),先删后拷,
避免 SDK 升级残留旧 DLL;安装时放行 TCP 17311 / UDP 17310
局域网询问-应答(不是主动公告)
- UDP 17310 收询问,只回同 subjectId;带了 nvrHosts 时先实测可达,
连不上就不应答,让查询机继续走本机
- 应答带 TCP 17311,查询机一次连接即可判定是否同 LAN
跨网下载
- TCP 17311 提供 /health、/probe、/download、/download/{id}
- 除 /health 外均要求 X-Edge-Subject 且与配置主体一致
- 成片在传输完成后立即删除,另有 done 10 分钟 / failed 15 分钟
TTL 兜底,避免现场磁盘被临时文件吃满
桌面端选路
- DownloadAsync 先探 NVR 8000(600ms):通则沿用原有本机直连,
不通才广播找取流服务,取回后落到本机录像目录
- DownloadByPlanAsync 拆分:不查库、不需要用户令牌,
供只有设备令牌的服务端复用
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
381 lines
12 KiB
C#
381 lines
12 KiB
C#
using System;
|
||
using System.Collections.Concurrent;
|
||
using System.Collections.Generic;
|
||
using System.Diagnostics;
|
||
using System.IO;
|
||
using System.IO.Pipes;
|
||
using System.Text.Json;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
|
||
namespace YuncangTrace.LocalService.Services;
|
||
|
||
/// <summary>
|
||
/// 主进程这一侧的 worker 管理:拉起、连接、转发任务、崩溃熔断。
|
||
///
|
||
/// 设计取舍:
|
||
/// - **懒启动**:第一个取流任务来才拉起 worker,空闲时零开销;
|
||
/// - **进程级隔离**:worker 崩了只影响取流,扫码/上报不受影响(这是它存在的唯一理由);
|
||
/// - **熔断**:如果某个视频一解码就崩,没有熔断会陷入"重启—崩溃"死循环把 CPU 打满。
|
||
/// </summary>
|
||
public sealed class HikWorkerSupervisor : IAsyncDisposable
|
||
{
|
||
/// <summary>熔断窗口与阈值(窗口内崩够次数就停自动重启)</summary>
|
||
private static readonly TimeSpan CrashWindow = TimeSpan.FromMinutes(5);
|
||
private const int CrashLimit = 3;
|
||
private static readonly TimeSpan BlockedDuration = TimeSpan.FromMinutes(10);
|
||
|
||
private readonly object _lifecycleGate = new();
|
||
private readonly SemaphoreSlim _connectLock = new(1, 1);
|
||
private readonly SemaphoreSlim _writeLock = new(1, 1);
|
||
private readonly ConcurrentDictionary<string, TaskCompletionSource<HikWorkerResponse>> _pending = new();
|
||
private readonly Queue<DateTime> _crashes = new();
|
||
|
||
private Process? _process;
|
||
private NamedPipeClientStream? _pipe;
|
||
private CancellationTokenSource? _readCts;
|
||
private Task? _readLoop;
|
||
private bool _disposing;
|
||
private DateTime _blockedUntil = DateTime.MinValue;
|
||
|
||
/// <summary>worker 状态变化(供 agent-status.json 更新)</summary>
|
||
public event Action? StateChanged;
|
||
|
||
public bool IsConnected => _pipe?.IsConnected == true;
|
||
|
||
public bool IsBlocked => DateTime.UtcNow < _blockedUntil;
|
||
|
||
public DateTime? BlockedUntil => IsBlocked ? _blockedUntil : null;
|
||
|
||
public int CrashCount
|
||
{
|
||
get { lock (_lifecycleGate) return _crashes.Count; }
|
||
}
|
||
|
||
public string? LastError { get; private set; }
|
||
|
||
/// <summary>自检:SDK / ffmpeg 是否就位(失败不影响扫码)</summary>
|
||
public async Task<JsonElement?> ProbeAsync(CancellationToken ct = default)
|
||
=> await SendAsync(HikWorkerOps.Probe, null, ct).ConfigureAwait(false);
|
||
|
||
/// <summary>派发一份下载计划</summary>
|
||
/// <param name="payload">包含 code / streams / credentials / scanTime / rootDir</param>
|
||
public async Task<JsonElement?> DownloadAsync(object payload, CancellationToken ct = default)
|
||
=> await SendAsync(HikWorkerOps.Download, payload, ct).ConfigureAwait(false);
|
||
|
||
private async Task<JsonElement?> SendAsync(string op, object? payload, CancellationToken ct)
|
||
{
|
||
await EnsureConnectedAsync(ct).ConfigureAwait(false);
|
||
|
||
var pipe = _pipe ?? throw new InvalidOperationException("hik-worker 未连接");
|
||
var request = HikWorkerRequest.Create(op, payload);
|
||
var completion = new TaskCompletionSource<HikWorkerResponse>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||
_pending[request.Id] = completion;
|
||
|
||
try
|
||
{
|
||
await _writeLock.WaitAsync(ct).ConfigureAwait(false);
|
||
try
|
||
{
|
||
await HikWorkerPipe.WriteAsync(pipe, request, ct).ConfigureAwait(false);
|
||
}
|
||
finally
|
||
{
|
||
_writeLock.Release();
|
||
}
|
||
|
||
var response = await completion.Task.WaitAsync(ct).ConfigureAwait(false);
|
||
if (!response.Ok) throw new InvalidOperationException(response.Error ?? "hik-worker 执行失败");
|
||
return response.Data;
|
||
}
|
||
finally
|
||
{
|
||
_pending.TryRemove(request.Id, out _);
|
||
}
|
||
}
|
||
|
||
/// <summary>确保 worker 已拉起且管道已连接(失败会抛出可直接展示的原因)</summary>
|
||
private async Task EnsureConnectedAsync(CancellationToken ct)
|
||
{
|
||
if (IsConnected) return;
|
||
|
||
if (IsBlocked)
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"取流服务连续崩溃已暂停({_blockedUntil:HH:mm:ss} 后自动重试):{LastError}");
|
||
}
|
||
|
||
await _connectLock.WaitAsync(ct).ConfigureAwait(false);
|
||
try
|
||
{
|
||
if (IsConnected) return;
|
||
if (IsBlocked) throw new InvalidOperationException($"取流服务连续崩溃已暂停:{LastError}");
|
||
|
||
StartProcess();
|
||
var pipe = await ConnectPipeAsync(ct).ConfigureAwait(false);
|
||
|
||
_pipe = pipe;
|
||
_readCts = new CancellationTokenSource();
|
||
_readLoop = Task.Run(() => ReadLoopAsync(pipe, _readCts.Token), CancellationToken.None);
|
||
|
||
AppLog.Write("[hik-worker] 已连接");
|
||
StateChanged?.Invoke();
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
KillProcess();
|
||
RecordFailure(ex.Message);
|
||
throw;
|
||
}
|
||
finally
|
||
{
|
||
_connectLock.Release();
|
||
}
|
||
}
|
||
|
||
private void StartProcess()
|
||
{
|
||
var existing = _process;
|
||
if (existing is not null && !existing.HasExited) return;
|
||
|
||
existing?.Dispose();
|
||
_process = null;
|
||
|
||
var exe = Environment.ProcessPath
|
||
?? throw new InvalidOperationException("取不到当前程序路径,无法启动取流进程");
|
||
|
||
var info = new ProcessStartInfo(exe, "--hik-worker")
|
||
{
|
||
UseShellExecute = false,
|
||
CreateNoWindow = true,
|
||
WorkingDirectory = Path.GetDirectoryName(exe) ?? AppContext.BaseDirectory,
|
||
RedirectStandardError = true,
|
||
RedirectStandardOutput = true,
|
||
};
|
||
|
||
var process = new Process { StartInfo = info, EnableRaisingEvents = true };
|
||
process.Exited += (_, _) => OnWorkerGone($"进程退出(code={(TryGetExitCode(process)?.ToString() ?? "?")})");
|
||
process.ErrorDataReceived += (_, e) =>
|
||
{
|
||
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stderr] {e.Data}");
|
||
};
|
||
process.OutputDataReceived += (_, e) =>
|
||
{
|
||
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stdout] {e.Data}");
|
||
};
|
||
|
||
process.Start();
|
||
// 必须有读事件订阅,否则重定向缓冲填满会阻塞子进程
|
||
process.BeginErrorReadLine();
|
||
process.BeginOutputReadLine();
|
||
|
||
_process = process;
|
||
AppLog.Write($"[hik-worker] 已拉起(pid={process.Id})");
|
||
}
|
||
|
||
private static int? TryGetExitCode(Process process)
|
||
{
|
||
try
|
||
{
|
||
return process.ExitCode;
|
||
}
|
||
catch
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private async Task<NamedPipeClientStream> ConnectPipeAsync(CancellationToken ct)
|
||
{
|
||
Exception? last = null;
|
||
|
||
// worker 从启动到管道可用通常 < 500ms;给到 6 秒足够覆盖冷启动与磁盘慢的情况
|
||
for (var attempt = 0; attempt < 30; attempt++)
|
||
{
|
||
ct.ThrowIfCancellationRequested();
|
||
|
||
if (_process is { HasExited: true })
|
||
{
|
||
throw new InvalidOperationException(
|
||
$"取流进程启动后立即退出(code={TryGetExitCode(_process)?.ToString() ?? "?"}),"
|
||
+ "多半是缺 HCNetSDK 或 native 目录未复制到服务目录");
|
||
}
|
||
|
||
var pipe = new NamedPipeClientStream(".", HikWorkerPipe.PipeName,
|
||
PipeDirection.InOut, PipeOptions.Asynchronous);
|
||
try
|
||
{
|
||
await pipe.ConnectAsync(200, ct).ConfigureAwait(false);
|
||
return pipe;
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
pipe.Dispose();
|
||
throw;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
pipe.Dispose();
|
||
last = ex;
|
||
await Task.Delay(200, ct).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
throw new TimeoutException($"连接取流进程超时:{last?.Message}");
|
||
}
|
||
|
||
/// <summary>读循环:把响应按 Id 交回对应请求;断开即视为 worker 已死</summary>
|
||
private async Task ReadLoopAsync(Stream pipe, CancellationToken ct)
|
||
{
|
||
var reason = "连接已关闭";
|
||
try
|
||
{
|
||
while (!ct.IsCancellationRequested)
|
||
{
|
||
var response = await HikWorkerPipe.ReadAsync<HikWorkerResponse>(pipe, ct).ConfigureAwait(false);
|
||
if (response is null) break;
|
||
|
||
if (_pending.TryRemove(response.Id, out var completion))
|
||
{
|
||
completion.TrySetResult(response);
|
||
}
|
||
}
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
return;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
reason = ex.Message;
|
||
}
|
||
|
||
if (_disposing) return;
|
||
OnWorkerGone(reason);
|
||
}
|
||
|
||
private void OnWorkerGone(string reason)
|
||
{
|
||
if (_disposing) return;
|
||
|
||
CleanupConnection();
|
||
KillProcess();
|
||
RecordFailure(reason);
|
||
}
|
||
|
||
/// <summary>记一次失败并判断是否熔断(进程退出和启动失败走同一套阈值)</summary>
|
||
private void RecordFailure(string reason)
|
||
{
|
||
lock (_lifecycleGate)
|
||
{
|
||
if (_disposing) return;
|
||
|
||
LastError = reason;
|
||
|
||
var now = DateTime.UtcNow;
|
||
_crashes.Enqueue(now);
|
||
while (_crashes.Count > 0 && now - _crashes.Peek() > CrashWindow) _crashes.Dequeue();
|
||
|
||
if (_crashes.Count >= CrashLimit)
|
||
{
|
||
_blockedUntil = now.Add(BlockedDuration);
|
||
AppLog.Write($"[hik-worker] {CrashWindow.TotalMinutes:0} 分钟内失败 {_crashes.Count} 次 → "
|
||
+ $"熔断 {BlockedDuration.TotalMinutes:0} 分钟(扫码不受影响):{reason}");
|
||
}
|
||
else
|
||
{
|
||
AppLog.Write($"[hik-worker] 失败({reason}),下次任务到来时重试");
|
||
}
|
||
}
|
||
|
||
StateChanged?.Invoke();
|
||
}
|
||
|
||
private void CleanupConnection()
|
||
{
|
||
// 断开连接前先把挂起的请求全部失败掉,避免调用方一直等
|
||
foreach (var (_, completion) in _pending)
|
||
{
|
||
completion.TrySetException(new IOException("取流进程已退出"));
|
||
}
|
||
_pending.Clear();
|
||
|
||
try
|
||
{
|
||
_readCts?.Cancel();
|
||
}
|
||
catch
|
||
{
|
||
// 忽略
|
||
}
|
||
|
||
_readCts?.Dispose();
|
||
_readCts = null;
|
||
_readLoop = null;
|
||
|
||
try
|
||
{
|
||
_pipe?.Dispose();
|
||
}
|
||
catch
|
||
{
|
||
// 忽略
|
||
}
|
||
|
||
_pipe = null;
|
||
}
|
||
|
||
private void KillProcess()
|
||
{
|
||
var process = _process;
|
||
_process = null;
|
||
if (process is null) return;
|
||
|
||
try
|
||
{
|
||
if (!process.HasExited)
|
||
{
|
||
process.Kill(entireProcessTree: true);
|
||
process.WaitForExit(3000);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
AppLog.Write($"[hik-worker] 结束进程失败:{ex.Message}");
|
||
}
|
||
finally
|
||
{
|
||
process.Dispose();
|
||
}
|
||
}
|
||
|
||
public async ValueTask DisposeAsync()
|
||
{
|
||
lock (_lifecycleGate)
|
||
{
|
||
if (_disposing) return;
|
||
_disposing = true;
|
||
}
|
||
|
||
// 先请求优雅退出(让 worker 有机会收尾),失败再强杀
|
||
if (IsConnected)
|
||
{
|
||
try
|
||
{
|
||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
|
||
await SendAsync(HikWorkerOps.Shutdown, null, timeout.Token).ConfigureAwait(false);
|
||
}
|
||
catch
|
||
{
|
||
// 忽略:下面还会 Kill
|
||
}
|
||
}
|
||
|
||
CleanupConnection();
|
||
KillProcess();
|
||
|
||
_connectLock.Dispose();
|
||
_writeLock.Dispose();
|
||
}
|
||
}
|