feat(watchdog): 检测主机丢失安装时的地址与时钟未经 NTP 同步

cmd/felis/watchdog.go

@@ -28,7 +28,8 @@ const proxyFor = 3 * time.Minute
 
 // cmdWatchdog runs one pass of the platform watchdog (internal/watchdog): it
 // checks the cluster, PostgreSQL, the game proxy, the database backups and the
-// host, prints every finding, and mails the platform owners what came due.
+// host (disks, memory, its address and clock), prints every finding, and mails
+// the platform owners what came due.
 // deploy/bootstrap.sh runs it every two minutes from felis-watchdog.timer.
 func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
@@ -39,6 +40,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
 	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
 	proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
+	nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`)
 	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")
 	offsiteStatus := fs.String("offsite-status", offsite.DefaultStatusFile, "the record `felis offsite sync` leaves, checked when [offsite] is configured")
 	toolsStatus := fs.String("build-tools-status", defaultBuildToolsStatus, "the record `felis mirror-build-tools` leaves, checked when builds scan against the registry's DB copy")
@@ -112,6 +114,12 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	}
 	report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...)
 	add(watchdog.MemoryFinding("/proc/meminfo"))
+	if *nodeIP != "" {
+		if held, err := watchdog.HostAddresses(); err == nil {
+			add(watchdog.AddressFinding(*nodeIP, held))
+		}
+	}
+	add(watchdog.ClockFinding(watchdog.ClockStatus()))
 
 	if len(report.Findings) == 0 {
 		fmt.Fprintln(stdout, "felis watchdog: every check passed")

internal/watchdog/clock_linux.go

@@ -0,0 +1,15 @@
+//go:build linux
+
+package watchdog
+
+import "syscall"
+
+// ClockStatus reads the kernel's clock status word. Modes stays zero, so the
+// call only reads and needs no privilege.
+func ClockStatus() (int32, bool) {
+	var t syscall.Timex
+	if _, err := syscall.Adjtimex(&t); err != nil {
+		return 0, false
+	}
+	return t.Status, true
+}

internal/watchdog/clock_other.go

@@ -0,0 +1,8 @@
+//go:build !linux
+
+package watchdog
+
+// ClockStatus has no adjtimex to read outside Linux; the clock check is skipped.
+func ClockStatus() (int32, bool) {
+	return 0, false
+}

internal/watchdog/host.go

@@ -0,0 +1,86 @@
+package watchdog
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"time"
+)
+
+const (
+	// addressFor is short: nothing reaches the database or the panel while the
+	// address is gone, so a DHCP renewal that lands on a new lease is an outage.
+	addressFor = 5 * time.Minute
+	// clockFor leaves room for an NTP daemon that was just enabled (by the
+	// installer, or after a reboot) to reach its first synchronization.
+	clockFor = 30 * time.Minute
+
+	// staUnsync is STA_UNSYNC from <linux/timex.h>. The kernel holds it set while
+	// no NTP daemon disciplines the clock; chronyd and systemd-timesyncd clear it
+	// once they have synchronized. It is what `timedatectl` prints as "System
+	// clock synchronized: no".
+	staUnsync = 0x0040
+)
+
+// AddressFinding reports that this host no longer holds want, the node address
+// the installer wrote into the database connection string, pg_hba, the network
+// policies, the panel certificate and (by default) the nip.io root domain. held
+// is every address on the host's interfaces. An empty or unparsable want skips
+// the check.
+func AddressFinding(want string, held []net.IP) *Finding {
+	ip := net.ParseIP(want)
+	if ip == nil {
+		return nil
+	}
+	var now []string
+	for _, h := range held {
+		if h.Equal(ip) {
+			return nil
+		}
+		if !h.IsLoopback() && !h.IsLinkLocalUnicast() {
+			now = append(now, h.String())
+		}
+	}
+	current := strings.Join(now, ", ")
+	if current == "" {
+		current = "无 / none"
+	}
+	return &Finding{
+		Key: "host-address", Severity: Critical, For: addressFor,
+		Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：数据库连接、pg_hba、网络策略和面板证书仍指向旧地址",
+			want, current),
+		SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the database connection, pg_hba, the network policies and the panel certificate still point at it",
+			want, current),
+		Hint: "give the host its old address back (a DHCP reservation or a static address); docs/troubleshooting.md §13c",
+	}
+}
+
+// HostAddresses lists the addresses on this host's interfaces.
+func HostAddresses() ([]net.IP, error) {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return nil, err
+	}
+	out := make([]net.IP, 0, len(addrs))
+	for _, a := range addrs {
+		if n, ok := a.(*net.IPNet); ok {
+			out = append(out, n.IP)
+		}
+	}
+	return out, nil
+}
+
+// ClockFinding reports a clock no NTP daemon has synchronized, from the kernel's
+// adjtimex status word. ok is false where the status cannot be read (not Linux),
+// which skips the check.
+func ClockFinding(status int32, ok bool) *Finding {
+	if !ok || status&staUnsync == 0 {
+		return nil
+	}
+	return &Finding{
+		Key: "clock", Severity: Warning, For: clockFor,
+		Summary:   "系统时钟没有经 NTP 同步：登录验证码与会话的过期、异地备份上传（S3 拒收偏差超过 15 分钟的请求）和证书校验都依赖准确时间",
+		SummaryEN: "the system clock is not synchronized by NTP: sign-in code and session expiry, off-site uploads (S3 refuses requests more than 15 minutes off) and certificate checks all depend on it",
+		Hint:      "timedatectl; sudo timedatectl set-ntp true (docs/troubleshooting.md §13c)",
+	}
+}

internal/watchdog/host_test.go

@@ -0,0 +1,55 @@
+package watchdog
+
+import (
+	"net"
+	"strings"
+	"testing"
+)
+
+func TestAddressFinding(t *testing.T) {
+	loop := net.ParseIP("127.0.0.1")
+	if f := AddressFinding("10.211.55.6", []net.IP{loop, net.IPv4(10, 211, 55, 6)}); f != nil {
+		t.Fatalf("still held: got %+v", f)
+	}
+	// The 4-byte form an interface can report is the same address.
+	if f := AddressFinding("10.211.55.6", []net.IP{{10, 211, 55, 6}}); f != nil {
+		t.Fatalf("4-byte form: got %+v", f)
+	}
+	if f := AddressFinding("", []net.IP{loop}); f != nil {
+		t.Fatalf("no recorded address: got %+v", f)
+	}
+
+	f := AddressFinding("10.211.55.6", []net.IP{loop, net.ParseIP("10.211.55.9"), net.ParseIP("fe80::1c1a:2bff:fe3c:4d5e")})
+	if f == nil {
+		t.Fatal("moved address: no finding")
+	}
+	if f.Key != "host-address" || f.Severity != Critical {
+		t.Fatalf("moved address: key %q severity %v", f.Key, f.Severity)
+	}
+	if !strings.Contains(f.SummaryEN, "no longer holds 10.211.55.6") || !strings.Contains(f.SummaryEN, "(it has: 10.211.55.9)") {
+		t.Fatalf("moved address: summary %q", f.SummaryEN)
+	}
+
+	f = AddressFinding("10.211.55.6", []net.IP{loop})
+	if f == nil || !strings.Contains(f.Summary, "（现在是：无 / none）") {
+		t.Fatalf("no address at all: got %+v", f)
+	}
+}
+
+func TestClockFinding(t *testing.T) {
+	// Status words as <linux/timex.h> spells them: STA_PLL 0x0001, STA_UNSYNC
+	// 0x0040, STA_NANO 0x2000. An unsynced kernel reports 0x0041 with a daemon
+	// that has not locked yet, 0x0040 with none; chronyd synced leaves 0x2001.
+	if f := ClockFinding(0x2001, true); f != nil {
+		t.Fatalf("synchronized: got %+v", f)
+	}
+	for _, st := range []int32{0x0040, 0x0041} {
+		f := ClockFinding(st, true)
+		if f == nil || f.Key != "clock" || f.Severity != Warning {
+			t.Fatalf("status %#x: got %+v", st, f)
+		}
+	}
+	if f := ClockFinding(0x0040, false); f != nil {
+		t.Fatalf("unreadable status: got %+v", f)
+	}
+}