WMS_MAIN/script/deploy-watchdog.cjs
liufumingren 0918f2d572 feat(deploy): 证书同步改为单片段 include + ver 命名 + 声明式自愈与差集下线
- 站点内容外置为单一片段(upstreamInclude,运维维护):生成的 server 块 = 证书骨架 +
  一行 include <片段>,撤掉此前的 customDir 多片段方案;并强制 include 必须是绝对路径
  且文件已存在(nginx 解析相对 include 的基准是 nginx prefix,不是脚本 cwd,写相对路径
  会让 nginx -t 失败并回滚整轮变更),不满足直接 exit 1(dry-run 下仅告警)
- conf 命名加版本位 <domain>-<ver>.conf:轮转只改配置的 ver,旧命名自动清理并 reload
- 声明式对账自愈:证书 / 私钥 / server 块缺失或被改坏,按库内内容原子重建后统一
  nginx -t + reload(内容一致时零写入、零 reload);失败态(sync_status=2)同样参与
  收敛并在校验通过后回到 1;nginx -t / reload 属全局失败,只落 last_sync_error 下轮
  重试,不再把行写成 2 卡死
- 域名下线改为配置差集驱动:以「已配置 + 已验证 + 证书就绪」的期望集合为准(同一仓库
  只取 id 最小的一条),目录中属于本系统命名规则但配置里已无该域名的 conf / 证书直接
  删除并 reload;不再消费 retired_domain(该列仅作应用侧审计留痕)
- 证书未配置的行不写 conf 也不作为文件保留依据(遗留产物由差集回收,避免 conf 引用
  不存在的证书导致 nginx -t 全局失败),行状态回写 sync_status=0 待同步;证书非空但
  非法、域名形态非法才记行级失败(2)
- cert-sync 部署命令瘦身为 install / uninstall;新增 watchdog 部署命令(precheck /
  install / status / dry-run / run),安装路径统一到 script/;deploy-lib 保护生产运行时
  配置不被发布覆盖;README 补「发布与运维」章节(目录硬约定 / 定时任务 / 命令清单)
2026-09-28 01:02:23 +08:00

293 lines
13 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

'use strict';
/**
* qimen 接口服务看门狗(keep-alive)的安装与运维工具。
*
* 与 script/deploy-cert-sync.cjs 同源思路,但对象完全不同:
* - 安装对象:script/qimen-watchdog.sh(仓库源)→ <部署根>/script/qimen-watchdog.sh(服务器稳定副本)
* - 调度载体:/etc/cron.d/qimen-watchdog(系统级 cron.d)
* **刻意不放 root crontab**:宝塔面板会整体重写 root crontab,放那儿会被冲掉。
* - 日志:/www/wwwlogs/wms/qimen-watchdog.log
*
* 为什么副本放部署根下的 script/ 目录而不是 v1/v2:
* 与证书同步脚本同理——不受版本切换与发布清理影响,版本目录里没有它的位置;
* 同时与 warehouse-cert-sync.sh / warehouse-cert-sync.cjs 收口在同一目录,运维只需看一处。
* install 会把历史位置(部署根)的同名副本一次性挪成 script/<name>.legacy-<时间戳> 备份。
*
* 设计要点:
* 1. 幂等:可重复 install;cron.d 文件用「临时文件 + mv」原子覆盖,不会出现半截配置;
* 2. 备份:覆盖前把既有副本 cp -a 成 <path>.prev-<时间戳>,保留回滚余地;
* 3. 去重:若 root crontab 里存在历史 watchdog 行(早期手工装的),install 会删掉,
* 避免与 cron.d 条目双跑——看门狗双跑会并发 kill/start 同一实例,风险很高;
* 4. 校验三连:bash -n 语法校验 + sha256 与本地源比对(不一致直接失败)+
* 远端 --dry-run 冒烟(只探活、绝不启动/杀进程);
* 5. 不碰服务:install 不重启 qimen、不 reload nginx、不动任何数据库。
*
* 用法:
* node script/deploy-watchdog.cjs production precheck
* node script/deploy-watchdog.cjs production install
* node script/deploy-watchdog.cjs production status
* node script/deploy-watchdog.cjs production dry-run # 远端执行 --dry-run(只探活并报告)
* node script/deploy-watchdog.cjs production run # 远端真实保活(探活失败才会拉起服务)
*
* 退出码:0 成功;非 0 失败(参数非法 / 本地文件缺失 / sha 不一致 / 远端命令失败)。
*/
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const {
ROOT,
getNowStamp,
loadDeployConfig,
quoteForBash,
runScp,
runSsh,
} = require('./deploy-lib.cjs');
const COMMANDS = new Set([ 'precheck', 'install', 'status', 'dry-run', 'run' ]);
const COMMAND_HINT = 'precheck | install | status | dry-run | run';
/** 仓库源文件(脚本本体)。 */
const LOCAL_WATCHDOG = path.join(ROOT, 'script', 'qimen-watchdog.sh');
/** 服务器副本文件名(落在部署根的 script/ 目录下,与 cert-sync 脚本同目录)。 */
const REMOTE_FILENAME = 'qimen-watchdog.sh';
/** 服务器副本相对部署根的路径。 */
const REMOTE_RELATIVE = `script/${REMOTE_FILENAME}`;
/** cron.d 文件名 / 路径(名称不含点,避免被 cron 忽略)。 */
const CRON_D_NAME = 'qimen-watchdog';
const CRON_D_PATH = `/etc/cron.d/${CRON_D_NAME}`;
const LOG_PATH = '/www/wwwlogs/wms/qimen-watchdog.log';
const LOCK_PATH = '/var/lock/qimen-watchdog.lock';
const CRON_SCHEDULE = '*/5 * * * *';
const CRON_USER = 'root';
/**
* 解析命令行参数。
* @param {string[]} argv process.argv.slice(2)。
* @return {{envName: string, command: string}} 解析结果。
* @throws {Error} 缺少/非法命令时抛出。
*/
function parseArgs(argv) {
const options = { envName: 'production', command: '' };
const tokens = [ ...argv ];
if (tokens.length === 0) {
throw new Error(`missing command: ${COMMAND_HINT}`);
}
if (!COMMANDS.has(tokens[0])) {
options.envName = tokens[0];
tokens.shift();
}
if (tokens.length === 0 || !COMMANDS.has(tokens[0])) {
throw new Error(`missing command: ${COMMAND_HINT}`);
}
options.command = tokens[0];
return options;
}
/**
* 计算本地文件 sha256(十六进制,小写)。
* @param {string} filePath 文件绝对路径。
* @return {string} sha256 摘要。
*/
function localSha256(filePath) {
return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
}
/**
* 目标服务器上的看门狗脚本绝对路径。
* @param {Object} config deploy 配置对象。
* @return {string} 绝对路径。
*/
function remoteWatchdogPath(config) {
return `${config.deployRoot}/${REMOTE_RELATIVE}`;
}
/**
* 生成 /etc/cron.d/qimen-watchdog 的完整内容。
* @param {string} watchdogPath 服务器上的看门狗脚本绝对路径。
* @return {string} cron.d 文件内容(含首部注释与 env 声明)。
*/
function buildCronFileContent(watchdogPath) {
return [
'# qimen 接口服务看门狗(保活)',
'# 由 npm run deploy:watchdog:install 维护,请勿手工编辑(会被下次安装覆盖)。',
'# 仓库源:script/qimen-watchdog.sh;服务器稳定副本:<部署根>/script/qimen-watchdog.sh',
'# 放在 /etc/cron.d 而非 root crontab:宝塔面板会整体重写 root crontab,会把它冲掉。',
'# 手动验证:bash <部署根>/script/qimen-watchdog.sh --dry-run',
'SHELL=/bin/bash',
'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
`${CRON_SCHEDULE} ${CRON_USER} ${watchdogPath} >> ${LOG_PATH} 2>&1`,
'',
].join('\n');
}
/**
* 只读预检:确认目标机具备安装条件。
* @param {Object} config deploy 配置对象。
* @return {void}
*/
function runPrecheck(config) {
const deployRoot = config.deployRoot;
const remoteScript = [
'set -uo pipefail',
`DEPLOY_ROOT=${quoteForBash(deployRoot)}`,
`TARGET="$DEPLOY_ROOT/${REMOTE_FILENAME}"`,
'echo "[WHOAMI] $(id -un)"',
'echo "[UNAME] $(uname -a)"',
'echo "[DEPLOY_ROOT_LS]"; ls -la "$DEPLOY_ROOT" | awk \'{print $1, $3, $4, $9}\'',
'echo "[CURRENT_MD] $(cat "$DEPLOY_ROOT/current.md" 2>/dev/null | tr -d "[:space:]" || echo MISSING)"',
'echo "[UPSTREAM_CONF]"; cat "$DEPLOY_ROOT/nginx-upstream.conf" 2>/dev/null || echo "(missing)"',
'echo "[WATCHDOG_FILE]"; if [ -f "$TARGET" ]; then ls -l --time-style=full-iso "$TARGET"; sha256sum "$TARGET"; else echo "MISSING"; fi',
'echo "[CRON_D]"; if [ -f ' + quoteForBash(CRON_D_PATH) + ' ]; then cat ' + quoteForBash(CRON_D_PATH) + '; else echo "MISSING"; fi',
'echo "[CRON_SERVICE]"; systemctl is-active cron 2>/dev/null || (service cron status 2>/dev/null | head -1) || echo unknown',
'echo "[ROOT_CRONTAB_MATCH]"; crontab -l 2>/dev/null | grep -n "qimen-watchdog" || echo NONE',
'echo "[LOG]"; ls -l ' + quoteForBash(LOG_PATH) + ' 2>/dev/null || echo "log missing"; tail -5 ' + quoteForBash(LOG_PATH) + ' 2>/dev/null',
'echo "[LOCK]"; ls -l ' + quoteForBash(LOCK_PATH) + ' 2>/dev/null || echo "lock not created yet"',
'echo "[INSTANCES]"; ps -eo pid=,etime=,args= | grep "[e]gg-scripts start" | head -6 || true',
'echo "[PORTS]"; ss -lntp 2>/dev/null | grep -E ":6061|:6062" || echo "no 6061/6062 listener"',
].join('\n');
console.log('[PRECHECK] watchdog remote env check (read-only, no service start/kill)');
console.log(String(runSsh(config, remoteScript, { stdio: 'pipe' }) || '').trim());
}
/**
* 安装:备份旧副本 → 上传脚本 → 写 cron.d → 清理历史 root crontab 行 → 语法/哈希/冒烟校验。
* @param {Object} config deploy 配置对象。
* @return {void}
*/
function runInstall(config) {
const deployRoot = config.deployRoot;
if (!fs.existsSync(LOCAL_WATCHDOG)) {
throw new Error(`local file missing: ${LOCAL_WATCHDOG} (script/qimen-watchdog.sh)`);
}
const localSha = localSha256(LOCAL_WATCHDOG);
const target = remoteWatchdogPath(config);
console.log(`[INSTALL] local source: ${LOCAL_WATCHDOG} sha256=${localSha}`);
const remoteDirScript = [
'set -euo pipefail',
`DEPLOY_ROOT=${quoteForBash(deployRoot)}`,
`TARGET="$DEPLOY_ROOT/${REMOTE_RELATIVE}"`,
`STAMP=${quoteForBash(getNowStamp())}`,
'mkdir -p "$DEPLOY_ROOT/script" /www/wwwlogs/wms',
'# 历史位置(部署根)一次性迁移:旧副本挪进 script/ 留作回滚备份,避免根目录残留孤儿',
'LEGACY="$DEPLOY_ROOT/qimen-watchdog.sh"',
'if [ -f "$LEGACY" ]; then mv "$LEGACY" "$DEPLOY_ROOT/script/qimen-watchdog.sh.legacy-$STAMP" && echo "LEGACY_MOVED $LEGACY -> $DEPLOY_ROOT/script/qimen-watchdog.sh.legacy-$STAMP"; fi',
'if [ -f "$TARGET" ]; then cp -a "$TARGET" "$TARGET.prev-$STAMP" && echo "BACKED_UP $TARGET -> $TARGET.prev-$STAMP"; else echo "NO_EXISTING_COPY"; fi',
`touch ${quoteForBash(LOG_PATH)} && chmod 644 ${quoteForBash(LOG_PATH)} && echo "LOG_FILE_OK"`,
].join('\n');
console.log('[INSTALL] remote dir prep + backup of existing copy');
console.log(String(runSsh(config, remoteDirScript, { stdio: 'pipe' }) || '').trim());
runScp(config, LOCAL_WATCHDOG, target);
console.log(`[INSTALL] uploaded -> ${target}`);
const finalizeScript = [
'set -euo pipefail',
`TARGET=${quoteForBash(target)}`,
`CRON_D=${quoteForBash(CRON_D_PATH)}`,
'chmod 755 "$TARGET" && echo "CHMOD_OK"',
'bash -n "$TARGET" && echo "WATCHDOG_SYNTAX_OK"',
`LOCAL_SHA=${quoteForBash(localSha)}`,
'REMOTE_SHA_LINE=$(sha256sum "$TARGET")',
'REMOTE_SHA=${REMOTE_SHA_LINE%% *}',
'if [ "$REMOTE_SHA" = "$LOCAL_SHA" ]; then echo "SHA_MATCH $REMOTE_SHA"; else echo "SHA_MISMATCH local=$LOCAL_SHA remote=$REMOTE_SHA"; exit 1; fi',
'TMP=$(mktemp /tmp/qimen-watchdog.cron.XXXXXX)',
'cat > "$TMP" <<\'QIMEN_WATCHDOG_CRON_EOF\'',
buildCronFileContent(target).replace(/\n+$/, ''),
'QIMEN_WATCHDOG_CRON_EOF',
'chown root:root "$TMP" && chmod 644 "$TMP"',
'mv "$TMP" "$CRON_D" && echo "CRON_D_WRITTEN $CRON_D"',
'echo "[CRON_D_CONTENT]"',
'cat "$CRON_D"',
'if crontab -l 2>/dev/null | grep -q "qimen-watchdog"; then',
' crontab -l 2>/dev/null | grep -v "qimen-watchdog" | crontab -',
' echo "ROOT_CRONTAB_DEDUPED (legacy watchdog line removed)"',
'else',
' echo "ROOT_CRONTAB_CLEAN"',
'fi',
'echo "[CRON_SERVICE] $(systemctl is-active cron 2>/dev/null || echo unknown)"',
'echo "[SMOKE] bash $TARGET --dry-run (probe only, never starts/kills)"',
'bash "$TARGET" --dry-run 2>&1 | tail -15 || true',
].join('\n');
console.log('[INSTALL] write cron.d + verify (syntax / sha256 / dry-run smoke)');
console.log(String(runSsh(config, finalizeScript, { stdio: 'pipe' }) || '').trim());
}
/**
* 状态:只读查看副本、cron.d、日志、锁、探活结果,并与本地源比对哈希。
* @param {Object} config deploy 配置对象。
* @return {void}
*/
function runStatus(config) {
const target = remoteWatchdogPath(config);
const localSha = fs.existsSync(LOCAL_WATCHDOG) ? localSha256(LOCAL_WATCHDOG) : '(local file missing)';
const remoteScript = [
'set -uo pipefail',
`TARGET=${quoteForBash(target)}`,
`CRON_D=${quoteForBash(CRON_D_PATH)}`,
'echo "[WATCHDOG_FILE]"; if [ -f "$TARGET" ]; then ls -l --time-style=full-iso "$TARGET"; else echo "MISSING"; fi',
'echo "[SHA_REMOTE]"; sha256sum "$TARGET" 2>/dev/null || echo "(missing)"',
'echo "[CRON_D]"; cat "$CRON_D" 2>/dev/null || echo "MISSING"',
'echo "[CRON_SERVICE]"; systemctl is-active cron 2>/dev/null || echo unknown',
'echo "[ROOT_CRONTAB_MATCH]"; crontab -l 2>/dev/null | grep -n "qimen-watchdog" || echo NONE',
'echo "[LOCK]"; ls -l ' + quoteForBash(LOCK_PATH) + ' 2>/dev/null || echo "lock not created yet"',
'echo "[LOG_TAIL]"; tail -10 ' + quoteForBash(LOG_PATH) + ' 2>/dev/null || echo "log missing"',
'echo "[INSTANCES]"; ps -eo pid=,etime=,args= | grep "[e]gg-scripts start" | head -6 || true',
'echo "[PORTS]"; ss -lntp 2>/dev/null | grep -E ":6061|:6062" || echo "no 6061/6062 listener"',
'echo "[PROBE_DRY_RUN]"; bash "$TARGET" --dry-run 2>&1 | tail -12 || true',
].join('\n');
console.log('[STATUS] watchdog (read-only; probe uses --dry-run)');
console.log(`[STATUS] local source sha256 = ${localSha}`);
console.log(String(runSsh(config, remoteScript, { stdio: 'pipe' }) || '').trim());
}
/**
* 远端执行看门狗脚本(转发参数)。
* @param {Object} config deploy 配置对象。
* @param {string[]} extraArgs 追加参数(如 --dry-run)。
* @return {string} 远端输出。
*/
function runRemoteWatchdog(config, extraArgs = []) {
const script = `bash ${quoteForBash(remoteWatchdogPath(config))}${extraArgs.map(arg => ` ${arg}`).join('')}`;
return String(runSsh(config, script) || '');
}
function main() {
const options = parseArgs(process.argv.slice(2));
const config = loadDeployConfig(options.envName);
if (options.command === 'precheck') {
runPrecheck(config);
return;
}
if (options.command === 'install') {
runInstall(config);
return;
}
if (options.command === 'status') {
runStatus(config);
return;
}
if (options.command === 'dry-run') {
console.log('[DRY-RUN] remote watchdog --dry-run (probe only, never starts/kills)');
console.log(runRemoteWatchdog(config, [ '--dry-run' ]));
return;
}
if (options.command === 'run') {
console.log('[RUN] remote watchdog real execution (starts the instance only when probe fails)');
console.log(runRemoteWatchdog(config, []));
return;
}
throw new Error(`unsupported command: ${options.command}`);
}
main();