diff --git a/cmd/felis/watchdog.go b/cmd/felis/watchdog.go index 59426cf..39e1719 100644 --- a/cmd/felis/watchdog.go +++ b/cmd/felis/watchdog.go @@ -28,7 +28,8 @@ const proxyFor = 3 * time.Minute // cmdWatchdog runs one pass of the platform watchdog (internal/watchdog): it // checks the cluster, PostgreSQL, the game proxy, the database backups and the -// host, prints every finding, and mails the platform owners what came due. +// host (disks, memory, its address and clock), prints every finding, and mails +// the platform owners what came due. // deploy/bootstrap.sh runs it every two minutes from felis-watchdog.timer. func cmdWatchdog(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("watchdog", flag.ContinueOnError) @@ -39,6 +40,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`) diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space") proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`) + nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`) controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane") offsiteStatus := fs.String("offsite-status", offsite.DefaultStatusFile, "the record `felis offsite sync` leaves, checked when [offsite] is configured") toolsStatus := fs.String("build-tools-status", defaultBuildToolsStatus, "the record `felis mirror-build-tools` leaves, checked when builds scan against the registry's DB copy") @@ -112,6 +114,12 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { } report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...) add(watchdog.MemoryFinding("/proc/meminfo")) + if *nodeIP != "" { + if held, err := watchdog.HostAddresses(); err == nil { + add(watchdog.AddressFinding(*nodeIP, held)) + } + } + add(watchdog.ClockFinding(watchdog.ClockStatus())) if len(report.Findings) == 0 { fmt.Fprintln(stdout, "felis watchdog: every check passed") diff --git a/internal/watchdog/clock_linux.go b/internal/watchdog/clock_linux.go new file mode 100644 index 0000000..41d4256 --- /dev/null +++ b/internal/watchdog/clock_linux.go @@ -0,0 +1,15 @@ +//go:build linux + +package watchdog + +import "syscall" + +// ClockStatus reads the kernel's clock status word. Modes stays zero, so the +// call only reads and needs no privilege. +func ClockStatus() (int32, bool) { + var t syscall.Timex + if _, err := syscall.Adjtimex(&t); err != nil { + return 0, false + } + return t.Status, true +} diff --git a/internal/watchdog/clock_other.go b/internal/watchdog/clock_other.go new file mode 100644 index 0000000..5394d94 --- /dev/null +++ b/internal/watchdog/clock_other.go @@ -0,0 +1,8 @@ +//go:build !linux + +package watchdog + +// ClockStatus has no adjtimex to read outside Linux; the clock check is skipped. +func ClockStatus() (int32, bool) { + return 0, false +} diff --git a/internal/watchdog/host.go b/internal/watchdog/host.go new file mode 100644 index 0000000..4cd83a3 --- /dev/null +++ b/internal/watchdog/host.go @@ -0,0 +1,86 @@ +package watchdog + +import ( + "fmt" + "net" + "strings" + "time" +) + +const ( + // addressFor is short: nothing reaches the database or the panel while the + // address is gone, so a DHCP renewal that lands on a new lease is an outage. + addressFor = 5 * time.Minute + // clockFor leaves room for an NTP daemon that was just enabled (by the + // installer, or after a reboot) to reach its first synchronization. + clockFor = 30 * time.Minute + + // staUnsync is STA_UNSYNC from . The kernel holds it set while + // no NTP daemon disciplines the clock; chronyd and systemd-timesyncd clear it + // once they have synchronized. It is what `timedatectl` prints as "System + // clock synchronized: no". + staUnsync = 0x0040 +) + +// AddressFinding reports that this host no longer holds want, the node address +// the installer wrote into the database connection string, pg_hba, the network +// policies, the panel certificate and (by default) the nip.io root domain. held +// is every address on the host's interfaces. An empty or unparsable want skips +// the check. +func AddressFinding(want string, held []net.IP) *Finding { + ip := net.ParseIP(want) + if ip == nil { + return nil + } + var now []string + for _, h := range held { + if h.Equal(ip) { + return nil + } + if !h.IsLoopback() && !h.IsLinkLocalUnicast() { + now = append(now, h.String()) + } + } + current := strings.Join(now, ", ") + if current == "" { + current = "无 / none" + } + return &Finding{ + Key: "host-address", Severity: Critical, For: addressFor, + Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s(现在是:%s):数据库连接、pg_hba、网络策略和面板证书仍指向旧地址", + want, current), + SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the database connection, pg_hba, the network policies and the panel certificate still point at it", + want, current), + Hint: "give the host its old address back (a DHCP reservation or a static address); docs/troubleshooting.md §13c", + } +} + +// HostAddresses lists the addresses on this host's interfaces. +func HostAddresses() ([]net.IP, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return nil, err + } + out := make([]net.IP, 0, len(addrs)) + for _, a := range addrs { + if n, ok := a.(*net.IPNet); ok { + out = append(out, n.IP) + } + } + return out, nil +} + +// ClockFinding reports a clock no NTP daemon has synchronized, from the kernel's +// adjtimex status word. ok is false where the status cannot be read (not Linux), +// which skips the check. +func ClockFinding(status int32, ok bool) *Finding { + if !ok || status&staUnsync == 0 { + return nil + } + return &Finding{ + Key: "clock", Severity: Warning, For: clockFor, + Summary: "系统时钟没有经 NTP 同步:登录验证码与会话的过期、异地备份上传(S3 拒收偏差超过 15 分钟的请求)和证书校验都依赖准确时间", + SummaryEN: "the system clock is not synchronized by NTP: sign-in code and session expiry, off-site uploads (S3 refuses requests more than 15 minutes off) and certificate checks all depend on it", + Hint: "timedatectl; sudo timedatectl set-ntp true (docs/troubleshooting.md §13c)", + } +} diff --git a/internal/watchdog/host_test.go b/internal/watchdog/host_test.go new file mode 100644 index 0000000..a94541b --- /dev/null +++ b/internal/watchdog/host_test.go @@ -0,0 +1,55 @@ +package watchdog + +import ( + "net" + "strings" + "testing" +) + +func TestAddressFinding(t *testing.T) { + loop := net.ParseIP("127.0.0.1") + if f := AddressFinding("10.211.55.6", []net.IP{loop, net.IPv4(10, 211, 55, 6)}); f != nil { + t.Fatalf("still held: got %+v", f) + } + // The 4-byte form an interface can report is the same address. + if f := AddressFinding("10.211.55.6", []net.IP{{10, 211, 55, 6}}); f != nil { + t.Fatalf("4-byte form: got %+v", f) + } + if f := AddressFinding("", []net.IP{loop}); f != nil { + t.Fatalf("no recorded address: got %+v", f) + } + + f := AddressFinding("10.211.55.6", []net.IP{loop, net.ParseIP("10.211.55.9"), net.ParseIP("fe80::1c1a:2bff:fe3c:4d5e")}) + if f == nil { + t.Fatal("moved address: no finding") + } + if f.Key != "host-address" || f.Severity != Critical { + t.Fatalf("moved address: key %q severity %v", f.Key, f.Severity) + } + if !strings.Contains(f.SummaryEN, "no longer holds 10.211.55.6") || !strings.Contains(f.SummaryEN, "(it has: 10.211.55.9)") { + t.Fatalf("moved address: summary %q", f.SummaryEN) + } + + f = AddressFinding("10.211.55.6", []net.IP{loop}) + if f == nil || !strings.Contains(f.Summary, "(现在是:无 / none)") { + t.Fatalf("no address at all: got %+v", f) + } +} + +func TestClockFinding(t *testing.T) { + // Status words as spells them: STA_PLL 0x0001, STA_UNSYNC + // 0x0040, STA_NANO 0x2000. An unsynced kernel reports 0x0041 with a daemon + // that has not locked yet, 0x0040 with none; chronyd synced leaves 0x2001. + if f := ClockFinding(0x2001, true); f != nil { + t.Fatalf("synchronized: got %+v", f) + } + for _, st := range []int32{0x0040, 0x0041} { + f := ClockFinding(st, true) + if f == nil || f.Key != "clock" || f.Severity != Warning { + t.Fatalf("status %#x: got %+v", st, f) + } + } + if f := ClockFinding(0x0040, false); f != nil { + t.Fatalf("unreadable status: got %+v", f) + } +}