using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.IO.Pipes; using System.Text.Json; using System.Threading; using System.Threading.Tasks; namespace YuncangTrace.LocalService.Services; /// /// 主进程这一侧的 worker 管理:拉起、连接、转发任务、崩溃熔断。 /// /// 设计取舍: /// - **懒启动**:第一个取流任务来才拉起 worker,空闲时零开销; /// - **进程级隔离**:worker 崩了只影响取流,扫码/上报不受影响(这是它存在的唯一理由); /// - **熔断**:如果某个视频一解码就崩,没有熔断会陷入"重启—崩溃"死循环把 CPU 打满。 /// public sealed class HikWorkerSupervisor : IAsyncDisposable { /// 熔断窗口与阈值(窗口内崩够次数就停自动重启) private static readonly TimeSpan CrashWindow = TimeSpan.FromMinutes(5); private const int CrashLimit = 3; private static readonly TimeSpan BlockedDuration = TimeSpan.FromMinutes(10); private readonly object _lifecycleGate = new(); private readonly SemaphoreSlim _connectLock = new(1, 1); private readonly SemaphoreSlim _writeLock = new(1, 1); private readonly ConcurrentDictionary> _pending = new(); private readonly Queue _crashes = new(); private Process? _process; private NamedPipeClientStream? _pipe; private CancellationTokenSource? _readCts; private Task? _readLoop; private bool _disposing; private DateTime _blockedUntil = DateTime.MinValue; /// worker 状态变化(供 agent-status.json 更新) public event Action? StateChanged; public bool IsConnected => _pipe?.IsConnected == true; public bool IsBlocked => DateTime.UtcNow < _blockedUntil; public DateTime? BlockedUntil => IsBlocked ? _blockedUntil : null; public int CrashCount { get { lock (_lifecycleGate) return _crashes.Count; } } public string? LastError { get; private set; } /// 自检:SDK / ffmpeg 是否就位(失败不影响扫码) public async Task ProbeAsync(CancellationToken ct = default) => await SendAsync(HikWorkerOps.Probe, null, ct).ConfigureAwait(false); /// 派发一份下载计划 /// 包含 code / streams / credentials / scanTime / rootDir public async Task DownloadAsync(object payload, CancellationToken ct = default) => await SendAsync(HikWorkerOps.Download, payload, ct).ConfigureAwait(false); /// 校验某时段有没有录像(真拉一小段) public async Task CheckAsync(object payload, CancellationToken ct = default) => await SendAsync(HikWorkerOps.Check, payload, ct).ConfigureAwait(false); private async Task SendAsync(string op, object? payload, CancellationToken ct) { await EnsureConnectedAsync(ct).ConfigureAwait(false); var pipe = _pipe ?? throw new InvalidOperationException("hik-worker 未连接"); var request = HikWorkerRequest.Create(op, payload); var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _pending[request.Id] = completion; try { await _writeLock.WaitAsync(ct).ConfigureAwait(false); try { await HikWorkerPipe.WriteAsync(pipe, request, ct).ConfigureAwait(false); } finally { _writeLock.Release(); } var response = await completion.Task.WaitAsync(ct).ConfigureAwait(false); if (!response.Ok) throw new InvalidOperationException(response.Error ?? "hik-worker 执行失败"); return response.Data; } finally { _pending.TryRemove(request.Id, out _); } } /// 确保 worker 已拉起且管道已连接(失败会抛出可直接展示的原因) private async Task EnsureConnectedAsync(CancellationToken ct) { if (IsConnected) return; if (IsBlocked) { throw new InvalidOperationException( $"取流服务连续崩溃已暂停({_blockedUntil:HH:mm:ss} 后自动重试):{LastError}"); } await _connectLock.WaitAsync(ct).ConfigureAwait(false); try { if (IsConnected) return; if (IsBlocked) throw new InvalidOperationException($"取流服务连续崩溃已暂停:{LastError}"); StartProcess(); var pipe = await ConnectPipeAsync(ct).ConfigureAwait(false); _pipe = pipe; _readCts = new CancellationTokenSource(); _readLoop = Task.Run(() => ReadLoopAsync(pipe, _readCts.Token), CancellationToken.None); AppLog.Write("[hik-worker] 已连接"); StateChanged?.Invoke(); } catch (Exception ex) { KillProcess(); RecordFailure(ex.Message); throw; } finally { _connectLock.Release(); } } private void StartProcess() { var existing = _process; if (existing is not null && !existing.HasExited) return; existing?.Dispose(); _process = null; var exe = Environment.ProcessPath ?? throw new InvalidOperationException("取不到当前程序路径,无法启动取流进程"); var info = new ProcessStartInfo(exe, "--hik-worker") { UseShellExecute = false, CreateNoWindow = true, WorkingDirectory = Path.GetDirectoryName(exe) ?? AppContext.BaseDirectory, RedirectStandardError = true, RedirectStandardOutput = true, }; var process = new Process { StartInfo = info, EnableRaisingEvents = true }; process.Exited += (_, _) => OnWorkerGone($"进程退出(code={(TryGetExitCode(process)?.ToString() ?? "?")})"); process.ErrorDataReceived += (_, e) => { if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stderr] {e.Data}"); }; process.OutputDataReceived += (_, e) => { if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stdout] {e.Data}"); }; process.Start(); // 必须有读事件订阅,否则重定向缓冲填满会阻塞子进程 process.BeginErrorReadLine(); process.BeginOutputReadLine(); _process = process; AppLog.Write($"[hik-worker] 已拉起(pid={process.Id})"); } private static int? TryGetExitCode(Process process) { try { return process.ExitCode; } catch { return null; } } private async Task ConnectPipeAsync(CancellationToken ct) { Exception? last = null; // worker 从启动到管道可用通常 < 500ms;给到 6 秒足够覆盖冷启动与磁盘慢的情况 for (var attempt = 0; attempt < 30; attempt++) { ct.ThrowIfCancellationRequested(); if (_process is { HasExited: true }) { throw new InvalidOperationException( $"取流进程启动后立即退出(code={TryGetExitCode(_process)?.ToString() ?? "?"})," + "多半是缺 HCNetSDK 或 native 目录未复制到服务目录"); } var pipe = new NamedPipeClientStream(".", HikWorkerPipe.PipeName, PipeDirection.InOut, PipeOptions.Asynchronous); try { await pipe.ConnectAsync(200, ct).ConfigureAwait(false); return pipe; } catch (OperationCanceledException) { pipe.Dispose(); throw; } catch (Exception ex) { pipe.Dispose(); last = ex; await Task.Delay(200, ct).ConfigureAwait(false); } } throw new TimeoutException($"连接取流进程超时:{last?.Message}"); } /// 读循环:把响应按 Id 交回对应请求;断开即视为 worker 已死 private async Task ReadLoopAsync(Stream pipe, CancellationToken ct) { var reason = "连接已关闭"; try { while (!ct.IsCancellationRequested) { var response = await HikWorkerPipe.ReadAsync(pipe, ct).ConfigureAwait(false); if (response is null) break; if (_pending.TryRemove(response.Id, out var completion)) { completion.TrySetResult(response); } } } catch (OperationCanceledException) { return; } catch (Exception ex) { reason = ex.Message; } if (_disposing) return; OnWorkerGone(reason); } private void OnWorkerGone(string reason) { if (_disposing) return; CleanupConnection(); KillProcess(); RecordFailure(reason); } /// 记一次失败并判断是否熔断(进程退出和启动失败走同一套阈值) private void RecordFailure(string reason) { lock (_lifecycleGate) { if (_disposing) return; LastError = reason; var now = DateTime.UtcNow; _crashes.Enqueue(now); while (_crashes.Count > 0 && now - _crashes.Peek() > CrashWindow) _crashes.Dequeue(); if (_crashes.Count >= CrashLimit) { _blockedUntil = now.Add(BlockedDuration); AppLog.Write($"[hik-worker] {CrashWindow.TotalMinutes:0} 分钟内失败 {_crashes.Count} 次 → " + $"熔断 {BlockedDuration.TotalMinutes:0} 分钟(扫码不受影响):{reason}"); } else { AppLog.Write($"[hik-worker] 失败({reason}),下次任务到来时重试"); } } StateChanged?.Invoke(); } private void CleanupConnection() { // 断开连接前先把挂起的请求全部失败掉,避免调用方一直等 foreach (var (_, completion) in _pending) { completion.TrySetException(new IOException("取流进程已退出")); } _pending.Clear(); try { _readCts?.Cancel(); } catch { // 忽略 } _readCts?.Dispose(); _readCts = null; _readLoop = null; try { _pipe?.Dispose(); } catch { // 忽略 } _pipe = null; } private void KillProcess() { var process = _process; _process = null; if (process is null) return; try { if (!process.HasExited) { process.Kill(entireProcessTree: true); process.WaitForExit(3000); } } catch (Exception ex) { AppLog.Write($"[hik-worker] 结束进程失败:{ex.Message}"); } finally { process.Dispose(); } } public async ValueTask DisposeAsync() { lock (_lifecycleGate) { if (_disposing) return; _disposing = true; } // 先请求优雅退出(让 worker 有机会收尾),失败再强杀 if (IsConnected) { try { using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2)); await SendAsync(HikWorkerOps.Shutdown, null, timeout.Token).ConfigureAwait(false); } catch { // 忽略:下面还会 Kill } } CleanupConnection(); KillProcess(); _connectLock.Dispose(); _writeLock.Dispose(); } }