- 站点内容外置为单一片段(upstreamInclude,运维维护):生成的 server 块 = 证书骨架 + 一行 include <片段>,撤掉此前的 customDir 多片段方案;并强制 include 必须是绝对路径 且文件已存在(nginx 解析相对 include 的基准是 nginx prefix,不是脚本 cwd,写相对路径 会让 nginx -t 失败并回滚整轮变更),不满足直接 exit 1(dry-run 下仅告警) - conf 命名加版本位 <domain>-<ver>.conf:轮转只改配置的 ver,旧命名自动清理并 reload - 声明式对账自愈:证书 / 私钥 / server 块缺失或被改坏,按库内内容原子重建后统一 nginx -t + reload(内容一致时零写入、零 reload);失败态(sync_status=2)同样参与 收敛并在校验通过后回到 1;nginx -t / reload 属全局失败,只落 last_sync_error 下轮 重试,不再把行写成 2 卡死 - 域名下线改为配置差集驱动:以「已配置 + 已验证 + 证书就绪」的期望集合为准(同一仓库 只取 id 最小的一条),目录中属于本系统命名规则但配置里已无该域名的 conf / 证书直接 删除并 reload;不再消费 retired_domain(该列仅作应用侧审计留痕) - 证书未配置的行不写 conf 也不作为文件保留依据(遗留产物由差集回收,避免 conf 引用 不存在的证书导致 nginx -t 全局失败),行状态回写 sync_status=0 待同步;证书非空但 非法、域名形态非法才记行级失败(2) - cert-sync 部署命令瘦身为 install / uninstall;新增 watchdog 部署命令(precheck / install / status / dry-run / run),安装路径统一到 script/;deploy-lib 保护生产运行时 配置不被发布覆盖;README 补「发布与运维」章节(目录硬约定 / 定时任务 / 命令清单)
293 lines
13 KiB
JavaScript
293 lines
13 KiB
JavaScript
'use strict';
|
||
|
||
/**
|
||
* qimen 接口服务看门狗(keep-alive)的安装与运维工具。
|
||
*
|
||
* 与 script/deploy-cert-sync.cjs 同源思路,但对象完全不同:
|
||
* - 安装对象:script/qimen-watchdog.sh(仓库源)→ <部署根>/script/qimen-watchdog.sh(服务器稳定副本)
|
||
* - 调度载体:/etc/cron.d/qimen-watchdog(系统级 cron.d)
|
||
* **刻意不放 root crontab**:宝塔面板会整体重写 root crontab,放那儿会被冲掉。
|
||
* - 日志:/www/wwwlogs/wms/qimen-watchdog.log
|
||
*
|
||
* 为什么副本放部署根下的 script/ 目录而不是 v1/v2:
|
||
* 与证书同步脚本同理——不受版本切换与发布清理影响,版本目录里没有它的位置;
|
||
* 同时与 warehouse-cert-sync.sh / warehouse-cert-sync.cjs 收口在同一目录,运维只需看一处。
|
||
* install 会把历史位置(部署根)的同名副本一次性挪成 script/<name>.legacy-<时间戳> 备份。
|
||
*
|
||
* 设计要点:
|
||
* 1. 幂等:可重复 install;cron.d 文件用「临时文件 + mv」原子覆盖,不会出现半截配置;
|
||
* 2. 备份:覆盖前把既有副本 cp -a 成 <path>.prev-<时间戳>,保留回滚余地;
|
||
* 3. 去重:若 root crontab 里存在历史 watchdog 行(早期手工装的),install 会删掉,
|
||
* 避免与 cron.d 条目双跑——看门狗双跑会并发 kill/start 同一实例,风险很高;
|
||
* 4. 校验三连:bash -n 语法校验 + sha256 与本地源比对(不一致直接失败)+
|
||
* 远端 --dry-run 冒烟(只探活、绝不启动/杀进程);
|
||
* 5. 不碰服务:install 不重启 qimen、不 reload nginx、不动任何数据库。
|
||
*
|
||
* 用法:
|
||
* node script/deploy-watchdog.cjs production precheck
|
||
* node script/deploy-watchdog.cjs production install
|
||
* node script/deploy-watchdog.cjs production status
|
||
* node script/deploy-watchdog.cjs production dry-run # 远端执行 --dry-run(只探活并报告)
|
||
* node script/deploy-watchdog.cjs production run # 远端真实保活(探活失败才会拉起服务)
|
||
*
|
||
* 退出码:0 成功;非 0 失败(参数非法 / 本地文件缺失 / sha 不一致 / 远端命令失败)。
|
||
*/
|
||
|
||
const fs = require('node:fs');
|
||
const path = require('node:path');
|
||
const crypto = require('node:crypto');
|
||
const {
|
||
ROOT,
|
||
getNowStamp,
|
||
loadDeployConfig,
|
||
quoteForBash,
|
||
runScp,
|
||
runSsh,
|
||
} = require('./deploy-lib.cjs');
|
||
|
||
const COMMANDS = new Set([ 'precheck', 'install', 'status', 'dry-run', 'run' ]);
|
||
const COMMAND_HINT = 'precheck | install | status | dry-run | run';
|
||
|
||
/** 仓库源文件(脚本本体)。 */
|
||
const LOCAL_WATCHDOG = path.join(ROOT, 'script', 'qimen-watchdog.sh');
|
||
|
||
/** 服务器副本文件名(落在部署根的 script/ 目录下,与 cert-sync 脚本同目录)。 */
|
||
const REMOTE_FILENAME = 'qimen-watchdog.sh';
|
||
/** 服务器副本相对部署根的路径。 */
|
||
const REMOTE_RELATIVE = `script/${REMOTE_FILENAME}`;
|
||
/** cron.d 文件名 / 路径(名称不含点,避免被 cron 忽略)。 */
|
||
const CRON_D_NAME = 'qimen-watchdog';
|
||
const CRON_D_PATH = `/etc/cron.d/${CRON_D_NAME}`;
|
||
const LOG_PATH = '/www/wwwlogs/wms/qimen-watchdog.log';
|
||
const LOCK_PATH = '/var/lock/qimen-watchdog.lock';
|
||
const CRON_SCHEDULE = '*/5 * * * *';
|
||
const CRON_USER = 'root';
|
||
|
||
/**
|
||
* 解析命令行参数。
|
||
* @param {string[]} argv process.argv.slice(2)。
|
||
* @return {{envName: string, command: string}} 解析结果。
|
||
* @throws {Error} 缺少/非法命令时抛出。
|
||
*/
|
||
function parseArgs(argv) {
|
||
const options = { envName: 'production', command: '' };
|
||
const tokens = [ ...argv ];
|
||
if (tokens.length === 0) {
|
||
throw new Error(`missing command: ${COMMAND_HINT}`);
|
||
}
|
||
if (!COMMANDS.has(tokens[0])) {
|
||
options.envName = tokens[0];
|
||
tokens.shift();
|
||
}
|
||
if (tokens.length === 0 || !COMMANDS.has(tokens[0])) {
|
||
throw new Error(`missing command: ${COMMAND_HINT}`);
|
||
}
|
||
options.command = tokens[0];
|
||
return options;
|
||
}
|
||
|
||
/**
|
||
* 计算本地文件 sha256(十六进制,小写)。
|
||
* @param {string} filePath 文件绝对路径。
|
||
* @return {string} sha256 摘要。
|
||
*/
|
||
function localSha256(filePath) {
|
||
return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
|
||
}
|
||
|
||
/**
|
||
* 目标服务器上的看门狗脚本绝对路径。
|
||
* @param {Object} config deploy 配置对象。
|
||
* @return {string} 绝对路径。
|
||
*/
|
||
function remoteWatchdogPath(config) {
|
||
return `${config.deployRoot}/${REMOTE_RELATIVE}`;
|
||
}
|
||
|
||
/**
|
||
* 生成 /etc/cron.d/qimen-watchdog 的完整内容。
|
||
* @param {string} watchdogPath 服务器上的看门狗脚本绝对路径。
|
||
* @return {string} cron.d 文件内容(含首部注释与 env 声明)。
|
||
*/
|
||
function buildCronFileContent(watchdogPath) {
|
||
return [
|
||
'# qimen 接口服务看门狗(保活)',
|
||
'# 由 npm run deploy:watchdog:install 维护,请勿手工编辑(会被下次安装覆盖)。',
|
||
'# 仓库源:script/qimen-watchdog.sh;服务器稳定副本:<部署根>/script/qimen-watchdog.sh',
|
||
'# 放在 /etc/cron.d 而非 root crontab:宝塔面板会整体重写 root crontab,会把它冲掉。',
|
||
'# 手动验证:bash <部署根>/script/qimen-watchdog.sh --dry-run',
|
||
'SHELL=/bin/bash',
|
||
'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
|
||
`${CRON_SCHEDULE} ${CRON_USER} ${watchdogPath} >> ${LOG_PATH} 2>&1`,
|
||
'',
|
||
].join('\n');
|
||
}
|
||
|
||
/**
|
||
* 只读预检:确认目标机具备安装条件。
|
||
* @param {Object} config deploy 配置对象。
|
||
* @return {void}
|
||
*/
|
||
function runPrecheck(config) {
|
||
const deployRoot = config.deployRoot;
|
||
const remoteScript = [
|
||
'set -uo pipefail',
|
||
`DEPLOY_ROOT=${quoteForBash(deployRoot)}`,
|
||
`TARGET="$DEPLOY_ROOT/${REMOTE_FILENAME}"`,
|
||
'echo "[WHOAMI] $(id -un)"',
|
||
'echo "[UNAME] $(uname -a)"',
|
||
'echo "[DEPLOY_ROOT_LS]"; ls -la "$DEPLOY_ROOT" | awk \'{print $1, $3, $4, $9}\'',
|
||
'echo "[CURRENT_MD] $(cat "$DEPLOY_ROOT/current.md" 2>/dev/null | tr -d "[:space:]" || echo MISSING)"',
|
||
'echo "[UPSTREAM_CONF]"; cat "$DEPLOY_ROOT/nginx-upstream.conf" 2>/dev/null || echo "(missing)"',
|
||
'echo "[WATCHDOG_FILE]"; if [ -f "$TARGET" ]; then ls -l --time-style=full-iso "$TARGET"; sha256sum "$TARGET"; else echo "MISSING"; fi',
|
||
'echo "[CRON_D]"; if [ -f ' + quoteForBash(CRON_D_PATH) + ' ]; then cat ' + quoteForBash(CRON_D_PATH) + '; else echo "MISSING"; fi',
|
||
'echo "[CRON_SERVICE]"; systemctl is-active cron 2>/dev/null || (service cron status 2>/dev/null | head -1) || echo unknown',
|
||
'echo "[ROOT_CRONTAB_MATCH]"; crontab -l 2>/dev/null | grep -n "qimen-watchdog" || echo NONE',
|
||
'echo "[LOG]"; ls -l ' + quoteForBash(LOG_PATH) + ' 2>/dev/null || echo "log missing"; tail -5 ' + quoteForBash(LOG_PATH) + ' 2>/dev/null',
|
||
'echo "[LOCK]"; ls -l ' + quoteForBash(LOCK_PATH) + ' 2>/dev/null || echo "lock not created yet"',
|
||
'echo "[INSTANCES]"; ps -eo pid=,etime=,args= | grep "[e]gg-scripts start" | head -6 || true',
|
||
'echo "[PORTS]"; ss -lntp 2>/dev/null | grep -E ":6061|:6062" || echo "no 6061/6062 listener"',
|
||
].join('\n');
|
||
|
||
console.log('[PRECHECK] watchdog remote env check (read-only, no service start/kill)');
|
||
console.log(String(runSsh(config, remoteScript, { stdio: 'pipe' }) || '').trim());
|
||
}
|
||
|
||
/**
|
||
* 安装:备份旧副本 → 上传脚本 → 写 cron.d → 清理历史 root crontab 行 → 语法/哈希/冒烟校验。
|
||
* @param {Object} config deploy 配置对象。
|
||
* @return {void}
|
||
*/
|
||
function runInstall(config) {
|
||
const deployRoot = config.deployRoot;
|
||
if (!fs.existsSync(LOCAL_WATCHDOG)) {
|
||
throw new Error(`local file missing: ${LOCAL_WATCHDOG} (script/qimen-watchdog.sh)`);
|
||
}
|
||
const localSha = localSha256(LOCAL_WATCHDOG);
|
||
const target = remoteWatchdogPath(config);
|
||
|
||
console.log(`[INSTALL] local source: ${LOCAL_WATCHDOG} sha256=${localSha}`);
|
||
|
||
const remoteDirScript = [
|
||
'set -euo pipefail',
|
||
`DEPLOY_ROOT=${quoteForBash(deployRoot)}`,
|
||
`TARGET="$DEPLOY_ROOT/${REMOTE_RELATIVE}"`,
|
||
`STAMP=${quoteForBash(getNowStamp())}`,
|
||
'mkdir -p "$DEPLOY_ROOT/script" /www/wwwlogs/wms',
|
||
'# 历史位置(部署根)一次性迁移:旧副本挪进 script/ 留作回滚备份,避免根目录残留孤儿',
|
||
'LEGACY="$DEPLOY_ROOT/qimen-watchdog.sh"',
|
||
'if [ -f "$LEGACY" ]; then mv "$LEGACY" "$DEPLOY_ROOT/script/qimen-watchdog.sh.legacy-$STAMP" && echo "LEGACY_MOVED $LEGACY -> $DEPLOY_ROOT/script/qimen-watchdog.sh.legacy-$STAMP"; fi',
|
||
'if [ -f "$TARGET" ]; then cp -a "$TARGET" "$TARGET.prev-$STAMP" && echo "BACKED_UP $TARGET -> $TARGET.prev-$STAMP"; else echo "NO_EXISTING_COPY"; fi',
|
||
`touch ${quoteForBash(LOG_PATH)} && chmod 644 ${quoteForBash(LOG_PATH)} && echo "LOG_FILE_OK"`,
|
||
].join('\n');
|
||
console.log('[INSTALL] remote dir prep + backup of existing copy');
|
||
console.log(String(runSsh(config, remoteDirScript, { stdio: 'pipe' }) || '').trim());
|
||
|
||
runScp(config, LOCAL_WATCHDOG, target);
|
||
console.log(`[INSTALL] uploaded -> ${target}`);
|
||
|
||
const finalizeScript = [
|
||
'set -euo pipefail',
|
||
`TARGET=${quoteForBash(target)}`,
|
||
`CRON_D=${quoteForBash(CRON_D_PATH)}`,
|
||
'chmod 755 "$TARGET" && echo "CHMOD_OK"',
|
||
'bash -n "$TARGET" && echo "WATCHDOG_SYNTAX_OK"',
|
||
`LOCAL_SHA=${quoteForBash(localSha)}`,
|
||
'REMOTE_SHA_LINE=$(sha256sum "$TARGET")',
|
||
'REMOTE_SHA=${REMOTE_SHA_LINE%% *}',
|
||
'if [ "$REMOTE_SHA" = "$LOCAL_SHA" ]; then echo "SHA_MATCH $REMOTE_SHA"; else echo "SHA_MISMATCH local=$LOCAL_SHA remote=$REMOTE_SHA"; exit 1; fi',
|
||
'TMP=$(mktemp /tmp/qimen-watchdog.cron.XXXXXX)',
|
||
'cat > "$TMP" <<\'QIMEN_WATCHDOG_CRON_EOF\'',
|
||
buildCronFileContent(target).replace(/\n+$/, ''),
|
||
'QIMEN_WATCHDOG_CRON_EOF',
|
||
'chown root:root "$TMP" && chmod 644 "$TMP"',
|
||
'mv "$TMP" "$CRON_D" && echo "CRON_D_WRITTEN $CRON_D"',
|
||
'echo "[CRON_D_CONTENT]"',
|
||
'cat "$CRON_D"',
|
||
'if crontab -l 2>/dev/null | grep -q "qimen-watchdog"; then',
|
||
' crontab -l 2>/dev/null | grep -v "qimen-watchdog" | crontab -',
|
||
' echo "ROOT_CRONTAB_DEDUPED (legacy watchdog line removed)"',
|
||
'else',
|
||
' echo "ROOT_CRONTAB_CLEAN"',
|
||
'fi',
|
||
'echo "[CRON_SERVICE] $(systemctl is-active cron 2>/dev/null || echo unknown)"',
|
||
'echo "[SMOKE] bash $TARGET --dry-run (probe only, never starts/kills)"',
|
||
'bash "$TARGET" --dry-run 2>&1 | tail -15 || true',
|
||
].join('\n');
|
||
|
||
console.log('[INSTALL] write cron.d + verify (syntax / sha256 / dry-run smoke)');
|
||
console.log(String(runSsh(config, finalizeScript, { stdio: 'pipe' }) || '').trim());
|
||
}
|
||
|
||
/**
|
||
* 状态:只读查看副本、cron.d、日志、锁、探活结果,并与本地源比对哈希。
|
||
* @param {Object} config deploy 配置对象。
|
||
* @return {void}
|
||
*/
|
||
function runStatus(config) {
|
||
const target = remoteWatchdogPath(config);
|
||
const localSha = fs.existsSync(LOCAL_WATCHDOG) ? localSha256(LOCAL_WATCHDOG) : '(local file missing)';
|
||
|
||
const remoteScript = [
|
||
'set -uo pipefail',
|
||
`TARGET=${quoteForBash(target)}`,
|
||
`CRON_D=${quoteForBash(CRON_D_PATH)}`,
|
||
'echo "[WATCHDOG_FILE]"; if [ -f "$TARGET" ]; then ls -l --time-style=full-iso "$TARGET"; else echo "MISSING"; fi',
|
||
'echo "[SHA_REMOTE]"; sha256sum "$TARGET" 2>/dev/null || echo "(missing)"',
|
||
'echo "[CRON_D]"; cat "$CRON_D" 2>/dev/null || echo "MISSING"',
|
||
'echo "[CRON_SERVICE]"; systemctl is-active cron 2>/dev/null || echo unknown',
|
||
'echo "[ROOT_CRONTAB_MATCH]"; crontab -l 2>/dev/null | grep -n "qimen-watchdog" || echo NONE',
|
||
'echo "[LOCK]"; ls -l ' + quoteForBash(LOCK_PATH) + ' 2>/dev/null || echo "lock not created yet"',
|
||
'echo "[LOG_TAIL]"; tail -10 ' + quoteForBash(LOG_PATH) + ' 2>/dev/null || echo "log missing"',
|
||
'echo "[INSTANCES]"; ps -eo pid=,etime=,args= | grep "[e]gg-scripts start" | head -6 || true',
|
||
'echo "[PORTS]"; ss -lntp 2>/dev/null | grep -E ":6061|:6062" || echo "no 6061/6062 listener"',
|
||
'echo "[PROBE_DRY_RUN]"; bash "$TARGET" --dry-run 2>&1 | tail -12 || true',
|
||
].join('\n');
|
||
|
||
console.log('[STATUS] watchdog (read-only; probe uses --dry-run)');
|
||
console.log(`[STATUS] local source sha256 = ${localSha}`);
|
||
console.log(String(runSsh(config, remoteScript, { stdio: 'pipe' }) || '').trim());
|
||
}
|
||
|
||
/**
|
||
* 远端执行看门狗脚本(转发参数)。
|
||
* @param {Object} config deploy 配置对象。
|
||
* @param {string[]} extraArgs 追加参数(如 --dry-run)。
|
||
* @return {string} 远端输出。
|
||
*/
|
||
function runRemoteWatchdog(config, extraArgs = []) {
|
||
const script = `bash ${quoteForBash(remoteWatchdogPath(config))}${extraArgs.map(arg => ` ${arg}`).join('')}`;
|
||
return String(runSsh(config, script) || '');
|
||
}
|
||
|
||
function main() {
|
||
const options = parseArgs(process.argv.slice(2));
|
||
const config = loadDeployConfig(options.envName);
|
||
|
||
if (options.command === 'precheck') {
|
||
runPrecheck(config);
|
||
return;
|
||
}
|
||
if (options.command === 'install') {
|
||
runInstall(config);
|
||
return;
|
||
}
|
||
if (options.command === 'status') {
|
||
runStatus(config);
|
||
return;
|
||
}
|
||
if (options.command === 'dry-run') {
|
||
console.log('[DRY-RUN] remote watchdog --dry-run (probe only, never starts/kills)');
|
||
console.log(runRemoteWatchdog(config, [ '--dry-run' ]));
|
||
return;
|
||
}
|
||
if (options.command === 'run') {
|
||
console.log('[RUN] remote watchdog real execution (starts the instance only when probe fails)');
|
||
console.log(runRemoteWatchdog(config, []));
|
||
return;
|
||
}
|
||
throw new Error(`unsupported command: ${options.command}`);
|
||
}
|
||
|
||
main();
|