feat(watchdog): 检测主机丢失安装时的地址与时钟未经 NTP 同步

// cmdWatchdog runs one pass of the platform watchdog (internal/watchdog): it

// checks the cluster, PostgreSQL, the game proxy, the database backups and the

// host, prints every finding, and mails the platform owners what came due.

// host (disks, memory, its address and clock), prints every finding, and mails

// the platform owners what came due.

// deploy/bootstrap.sh runs it every two minutes from felis-watchdog.timer.

func cmdWatchdog(args []string, stdout, stderr io.Writer) int {

	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)

	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)

	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/postgresql,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")

	proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)

	nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`)

	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")

	offsiteStatus := fs.String("offsite-status", offsite.DefaultStatusFile, "the record `felis offsite sync` leaves, checked when [offsite] is configured")

	toolsStatus := fs.String("build-tools-status", defaultBuildToolsStatus, "the record `felis mirror-build-tools` leaves, checked when builds scan against the registry's DB copy")

	}

	report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...)

	add(watchdog.MemoryFinding("/proc/meminfo"))

	if *nodeIP != "" {

		if held, err := watchdog.HostAddresses(); err == nil {

			add(watchdog.AddressFinding(*nodeIP, held))

		}

	}

	add(watchdog.ClockFinding(watchdog.ClockStatus()))

	if len(report.Findings) == 0 {

		fmt.Fprintln(stdout, "felis watchdog: every check passed")

//go:build linux

package watchdog

import "syscall"

// ClockStatus reads the kernel's clock status word. Modes stays zero, so the

// call only reads and needs no privilege.

func ClockStatus() (int32, bool) {

	var t syscall.Timex

	if _, err := syscall.Adjtimex(&t); err != nil {

		return 0, false

	}

	return t.Status, true

}

//go:build !linux

package watchdog

// ClockStatus has no adjtimex to read outside Linux; the clock check is skipped.

func ClockStatus() (int32, bool) {

	return 0, false

}

package watchdog

import (

	"fmt"

	"net"

	"strings"

	"time"

)

const (

	// addressFor is short: nothing reaches the database or the panel while the

	// address is gone, so a DHCP renewal that lands on a new lease is an outage.

	addressFor = 5 * time.Minute

	// clockFor leaves room for an NTP daemon that was just enabled (by the

	// installer, or after a reboot) to reach its first synchronization.

	clockFor = 30 * time.Minute

	// staUnsync is STA_UNSYNC from <linux/timex.h>. The kernel holds it set while

	// no NTP daemon disciplines the clock; chronyd and systemd-timesyncd clear it

	// once they have synchronized. It is what `timedatectl` prints as "System

	// clock synchronized: no".

	staUnsync = 0x0040

)

// AddressFinding reports that this host no longer holds want, the node address

// the installer wrote into the database connection string, pg_hba, the network

// policies, the panel certificate and (by default) the nip.io root domain. held

// is every address on the host's interfaces. An empty or unparsable want skips

// the check.

func AddressFinding(want string, held []net.IP) *Finding {

	ip := net.ParseIP(want)

	if ip == nil {

		return nil

	}

	var now []string

	for _, h := range held {

		if h.Equal(ip) {

			return nil

		}

		if !h.IsLoopback() && !h.IsLinkLocalUnicast() {

			now = append(now, h.String())

		}

	}

	current := strings.Join(now, ", ")

	if current == "" {

		current = "无 / none"

	}

	return &Finding{

		Key: "host-address", Severity: Critical, For: addressFor,

		Summary: fmt.Sprintf("本机已不再持有安装时的地址 %s（现在是：%s）：数据库连接、pg_hba、网络策略和面板证书仍指向旧地址",

			want, current),

		SummaryEN: fmt.Sprintf("this host no longer holds %s, the address the install was made on (it has: %s): the database connection, pg_hba, the network policies and the panel certificate still point at it",

			want, current),

		Hint: "give the host its old address back (a DHCP reservation or a static address); docs/troubleshooting.md §13c",

	}

}

// HostAddresses lists the addresses on this host's interfaces.

func HostAddresses() ([]net.IP, error) {

	addrs, err := net.InterfaceAddrs()

	if err != nil {

		return nil, err

	}

	out := make([]net.IP, 0, len(addrs))

	for _, a := range addrs {

		if n, ok := a.(*net.IPNet); ok {

			out = append(out, n.IP)

		}

	}

	return out, nil

}

// ClockFinding reports a clock no NTP daemon has synchronized, from the kernel's

// adjtimex status word. ok is false where the status cannot be read (not Linux),

// which skips the check.

func ClockFinding(status int32, ok bool) *Finding {

	if !ok || status&staUnsync == 0 {

		return nil

	}

	return &Finding{

		Key: "clock", Severity: Warning, For: clockFor,

		Summary:   "系统时钟没有经 NTP 同步：登录验证码与会话的过期、异地备份上传（S3 拒收偏差超过 15 分钟的请求）和证书校验都依赖准确时间",

		SummaryEN: "the system clock is not synchronized by NTP: sign-in code and session expiry, off-site uploads (S3 refuses requests more than 15 minutes off) and certificate checks all depend on it",

		Hint:      "timedatectl; sudo timedatectl set-ntp true (docs/troubleshooting.md §13c)",

	}

}

package watchdog

import (

	"net"

	"strings"

	"testing"

)

func TestAddressFinding(t *testing.T) {

	loop := net.ParseIP("127.0.0.1")

	if f := AddressFinding("10.211.55.6", []net.IP{loop, net.IPv4(10, 211, 55, 6)}); f != nil {

		t.Fatalf("still held: got %+v", f)

	}

	// The 4-byte form an interface can report is the same address.

	if f := AddressFinding("10.211.55.6", []net.IP{{10, 211, 55, 6}}); f != nil {

		t.Fatalf("4-byte form: got %+v", f)

	}

	if f := AddressFinding("", []net.IP{loop}); f != nil {

		t.Fatalf("no recorded address: got %+v", f)

	}

	f := AddressFinding("10.211.55.6", []net.IP{loop, net.ParseIP("10.211.55.9"), net.ParseIP("fe80::1c1a:2bff:fe3c:4d5e")})

	if f == nil {

		t.Fatal("moved address: no finding")

	}

	if f.Key != "host-address" || f.Severity != Critical {

		t.Fatalf("moved address: key %q severity %v", f.Key, f.Severity)

	}

	if !strings.Contains(f.SummaryEN, "no longer holds 10.211.55.6") || !strings.Contains(f.SummaryEN, "(it has: 10.211.55.9)") {

		t.Fatalf("moved address: summary %q", f.SummaryEN)

	}

	f = AddressFinding("10.211.55.6", []net.IP{loop})

	if f == nil || !strings.Contains(f.Summary, "（现在是：无 / none）") {

		t.Fatalf("no address at all: got %+v", f)

	}

}

func TestClockFinding(t *testing.T) {

	// Status words as <linux/timex.h> spells them: STA_PLL 0x0001, STA_UNSYNC

	// 0x0040, STA_NANO 0x2000. An unsynced kernel reports 0x0041 with a daemon

	// that has not locked yet, 0x0040 with none; chronyd synced leaves 0x2001.

	if f := ClockFinding(0x2001, true); f != nil {

		t.Fatalf("synchronized: got %+v", f)

	}

	for _, st := range []int32{0x0040, 0x0041} {

		f := ClockFinding(st, true)

		if f == nil || f.Key != "clock" || f.Severity != Warning {

			t.Fatalf("status %#x: got %+v", st, f)

		}

	}

	if f := ClockFinding(0x0040, false); f != nil {

		t.Fatalf("unreadable status: got %+v", f)

	}

}