using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Pipes;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace YuncangTrace.LocalService.Services;
///
/// 主进程这一侧的 worker 管理:拉起、连接、转发任务、崩溃熔断。
///
/// 设计取舍:
/// - **懒启动**:第一个取流任务来才拉起 worker,空闲时零开销;
/// - **进程级隔离**:worker 崩了只影响取流,扫码/上报不受影响(这是它存在的唯一理由);
/// - **熔断**:如果某个视频一解码就崩,没有熔断会陷入"重启—崩溃"死循环把 CPU 打满。
///
public sealed class HikWorkerSupervisor : IAsyncDisposable
{
/// 熔断窗口与阈值(窗口内崩够次数就停自动重启)
private static readonly TimeSpan CrashWindow = TimeSpan.FromMinutes(5);
private const int CrashLimit = 3;
private static readonly TimeSpan BlockedDuration = TimeSpan.FromMinutes(10);
private readonly object _lifecycleGate = new();
private readonly SemaphoreSlim _connectLock = new(1, 1);
private readonly SemaphoreSlim _writeLock = new(1, 1);
private readonly ConcurrentDictionary> _pending = new();
private readonly Queue _crashes = new();
private Process? _process;
private NamedPipeClientStream? _pipe;
private CancellationTokenSource? _readCts;
private Task? _readLoop;
private bool _disposing;
private DateTime _blockedUntil = DateTime.MinValue;
/// worker 状态变化(供 agent-status.json 更新)
public event Action? StateChanged;
public bool IsConnected => _pipe?.IsConnected == true;
public bool IsBlocked => DateTime.UtcNow < _blockedUntil;
public DateTime? BlockedUntil => IsBlocked ? _blockedUntil : null;
public int CrashCount
{
get { lock (_lifecycleGate) return _crashes.Count; }
}
public string? LastError { get; private set; }
/// 自检:SDK / ffmpeg 是否就位(失败不影响扫码)
public async Task ProbeAsync(CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Probe, null, ct).ConfigureAwait(false);
/// 派发一份下载计划
/// 包含 code / streams / credentials / scanTime / rootDir
public async Task DownloadAsync(object payload, CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Download, payload, ct).ConfigureAwait(false);
/// 校验某时段有没有录像(真拉一小段)
public async Task CheckAsync(object payload, CancellationToken ct = default)
=> await SendAsync(HikWorkerOps.Check, payload, ct).ConfigureAwait(false);
private async Task SendAsync(string op, object? payload, CancellationToken ct)
{
await EnsureConnectedAsync(ct).ConfigureAwait(false);
var pipe = _pipe ?? throw new InvalidOperationException("hik-worker 未连接");
var request = HikWorkerRequest.Create(op, payload);
var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
_pending[request.Id] = completion;
try
{
await _writeLock.WaitAsync(ct).ConfigureAwait(false);
try
{
await HikWorkerPipe.WriteAsync(pipe, request, ct).ConfigureAwait(false);
}
finally
{
_writeLock.Release();
}
var response = await completion.Task.WaitAsync(ct).ConfigureAwait(false);
if (!response.Ok) throw new InvalidOperationException(response.Error ?? "hik-worker 执行失败");
return response.Data;
}
finally
{
_pending.TryRemove(request.Id, out _);
}
}
/// 确保 worker 已拉起且管道已连接(失败会抛出可直接展示的原因)
private async Task EnsureConnectedAsync(CancellationToken ct)
{
if (IsConnected) return;
if (IsBlocked)
{
throw new InvalidOperationException(
$"取流服务连续崩溃已暂停({_blockedUntil:HH:mm:ss} 后自动重试):{LastError}");
}
await _connectLock.WaitAsync(ct).ConfigureAwait(false);
try
{
if (IsConnected) return;
if (IsBlocked) throw new InvalidOperationException($"取流服务连续崩溃已暂停:{LastError}");
StartProcess();
var pipe = await ConnectPipeAsync(ct).ConfigureAwait(false);
_pipe = pipe;
_readCts = new CancellationTokenSource();
_readLoop = Task.Run(() => ReadLoopAsync(pipe, _readCts.Token), CancellationToken.None);
AppLog.Write("[hik-worker] 已连接");
StateChanged?.Invoke();
}
catch (Exception ex)
{
KillProcess();
RecordFailure(ex.Message);
throw;
}
finally
{
_connectLock.Release();
}
}
private void StartProcess()
{
var existing = _process;
if (existing is not null && !existing.HasExited) return;
existing?.Dispose();
_process = null;
var exe = Environment.ProcessPath
?? throw new InvalidOperationException("取不到当前程序路径,无法启动取流进程");
var info = new ProcessStartInfo(exe, "--hik-worker")
{
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = Path.GetDirectoryName(exe) ?? AppContext.BaseDirectory,
RedirectStandardError = true,
RedirectStandardOutput = true,
};
var process = new Process { StartInfo = info, EnableRaisingEvents = true };
process.Exited += (_, _) => OnWorkerGone($"进程退出(code={(TryGetExitCode(process)?.ToString() ?? "?")})");
process.ErrorDataReceived += (_, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stderr] {e.Data}");
};
process.OutputDataReceived += (_, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data)) AppLog.Write($"[hik-worker:stdout] {e.Data}");
};
process.Start();
// 必须有读事件订阅,否则重定向缓冲填满会阻塞子进程
process.BeginErrorReadLine();
process.BeginOutputReadLine();
_process = process;
AppLog.Write($"[hik-worker] 已拉起(pid={process.Id})");
}
private static int? TryGetExitCode(Process process)
{
try
{
return process.ExitCode;
}
catch
{
return null;
}
}
private async Task ConnectPipeAsync(CancellationToken ct)
{
Exception? last = null;
// worker 从启动到管道可用通常 < 500ms;给到 6 秒足够覆盖冷启动与磁盘慢的情况
for (var attempt = 0; attempt < 30; attempt++)
{
ct.ThrowIfCancellationRequested();
if (_process is { HasExited: true })
{
throw new InvalidOperationException(
$"取流进程启动后立即退出(code={TryGetExitCode(_process)?.ToString() ?? "?"}),"
+ "多半是缺 HCNetSDK 或 native 目录未复制到服务目录");
}
var pipe = new NamedPipeClientStream(".", HikWorkerPipe.PipeName,
PipeDirection.InOut, PipeOptions.Asynchronous);
try
{
await pipe.ConnectAsync(200, ct).ConfigureAwait(false);
return pipe;
}
catch (OperationCanceledException)
{
pipe.Dispose();
throw;
}
catch (Exception ex)
{
pipe.Dispose();
last = ex;
await Task.Delay(200, ct).ConfigureAwait(false);
}
}
throw new TimeoutException($"连接取流进程超时:{last?.Message}");
}
/// 读循环:把响应按 Id 交回对应请求;断开即视为 worker 已死
private async Task ReadLoopAsync(Stream pipe, CancellationToken ct)
{
var reason = "连接已关闭";
try
{
while (!ct.IsCancellationRequested)
{
var response = await HikWorkerPipe.ReadAsync(pipe, ct).ConfigureAwait(false);
if (response is null) break;
if (_pending.TryRemove(response.Id, out var completion))
{
completion.TrySetResult(response);
}
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
reason = ex.Message;
}
if (_disposing) return;
OnWorkerGone(reason);
}
private void OnWorkerGone(string reason)
{
if (_disposing) return;
CleanupConnection();
KillProcess();
RecordFailure(reason);
}
/// 记一次失败并判断是否熔断(进程退出和启动失败走同一套阈值)
private void RecordFailure(string reason)
{
lock (_lifecycleGate)
{
if (_disposing) return;
LastError = reason;
var now = DateTime.UtcNow;
_crashes.Enqueue(now);
while (_crashes.Count > 0 && now - _crashes.Peek() > CrashWindow) _crashes.Dequeue();
if (_crashes.Count >= CrashLimit)
{
_blockedUntil = now.Add(BlockedDuration);
AppLog.Write($"[hik-worker] {CrashWindow.TotalMinutes:0} 分钟内失败 {_crashes.Count} 次 → "
+ $"熔断 {BlockedDuration.TotalMinutes:0} 分钟(扫码不受影响):{reason}");
}
else
{
AppLog.Write($"[hik-worker] 失败({reason}),下次任务到来时重试");
}
}
StateChanged?.Invoke();
}
private void CleanupConnection()
{
// 断开连接前先把挂起的请求全部失败掉,避免调用方一直等
foreach (var (_, completion) in _pending)
{
completion.TrySetException(new IOException("取流进程已退出"));
}
_pending.Clear();
try
{
_readCts?.Cancel();
}
catch
{
// 忽略
}
_readCts?.Dispose();
_readCts = null;
_readLoop = null;
try
{
_pipe?.Dispose();
}
catch
{
// 忽略
}
_pipe = null;
}
private void KillProcess()
{
var process = _process;
_process = null;
if (process is null) return;
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
process.WaitForExit(3000);
}
}
catch (Exception ex)
{
AppLog.Write($"[hik-worker] 结束进程失败:{ex.Message}");
}
finally
{
process.Dispose();
}
}
public async ValueTask DisposeAsync()
{
lock (_lifecycleGate)
{
if (_disposing) return;
_disposing = true;
}
// 先请求优雅退出(让 worker 有机会收尾),失败再强杀
if (IsConnected)
{
try
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
await SendAsync(HikWorkerOps.Shutdown, null, timeout.Token).ConfigureAwait(false);
}
catch
{
// 忽略:下面还会 Kill
}
}
CleanupConnection();
KillProcess();
_connectLock.Dispose();
_writeLock.Dispose();
}
}