fix(watchdog): k3s 证书 30 天内到期告警、7 天内转严重，文档说明重启续期与 rotate

cmd/felis/watchdog.go

@@ -42,6 +42,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")
 	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)
 	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")
+	certDirs := fs.String("k3s-cert-dirs", strings.Join(watchdog.K3sCertDirs, ","), `k3s certificate directories whose *.crt files must not be near expiry ("" skips the check)`)
 	proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)
 	nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`)
 	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")
@@ -137,6 +138,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int {
 	}
 	report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...)
 	add(watchdog.MemoryFinding("/proc/meminfo"))
+	add(watchdog.CertFinding(splitList(*certDirs), now))
 	if *nodeIP != "" {
 		if held, err := watchdog.HostAddresses(); err == nil {
 			add(watchdog.AddressFinding(*nodeIP, held))

docs/troubleshooting.md

@@ -1509,6 +1509,55 @@ boot before a reboot.
 
 ---
 
+## 13d. k3s certificates expire after a year
+
+k3s issues its own client and serving certificates (the API server's, the
+kubelet's, the admin kubeconfig's, etcd's) for 365 days and its CA certificates
+for ten years. It renews the client and serving certificates only as it starts:
+each start reissues, from the same keys, every one that has expired or is within
+120 days of expiring. A host that reboots or takes a k3s upgrade
+(`FELIS_UPGRADE_DEPS=1`, docs/operations.md §4) inside that window renews them
+unnoticed. A host that runs a year without restarting k3s loses its API server
+and its node the day they lapse, and the panel can no longer start, stop or back
+up servers. No restart renews a CA certificate; that takes `k3s certificate
+rotate-ca` and the procedure in the k3s documentation (Certificate Management).
+
+The watchdog reads the `*.crt` files under `/var/lib/rancher/k3s/server/tls`
+(and its `etcd`, `kube-controller-manager` and `kube-scheduler` directories) and
+`/var/lib/rancher/k3s/agent`, the set `k3s certificate check` reads, and never
+the keys beside them. It reports the certificate that expires first as
+`k3s-certs`: a warning 30 days before, critical in the last 7 days and once it
+has lapsed, and the hint says whether a restart renews it (a CA it does not).
+`felis watchdog -k3s-cert-dirs ""` turns the check off. [GO-TESTED: `TestCertFinding`]
+[VM-VERIFIED: against the real certificates of a v1.36.4+k3s1 install, with the
+clock moved ahead]
+
+Check and renew:
+
+```sh
+sudo k3s certificate check --output table   # every certificate, its expiry and residual time
+sudo systemctl restart k3s                  # reissues the ones within 120 days of expiry
+sudo k3s certificate check --output table   # the renewed ones show about a year again
+```
+
+A restart stops k3s alone. The unit's `KillMode=process` leaves the containers
+running, so game servers, PostgreSQL and the control plane keep serving while
+the API server comes back within a minute. [VM-VERIFIED: every container ID and
+restart count unchanged across `systemctl restart k3s`] `k3s-killall.sh` stops
+every container along with k3s; keep it out of this.
+
+To renew ahead of the 120-day window (to line it up with a maintenance slot),
+reissue every client and serving certificate at once:
+
+```sh
+sudo systemctl stop k3s && sudo k3s certificate rotate && sudo systemctl start k3s
+```
+
+The panel's own certificate (`/etc/felis/panel-tls.crt`, which the installer
+self-signs for 825 days) is a different certificate, outside this check.
+
+---
+
 ## 14. Health alerts, and metrics for diagnosis (spec §23)
 
 Every full install runs `felis watchdog` from `felis-watchdog.timer`, which
@@ -1537,6 +1586,7 @@ Every two minutes the host checks:
 | Host memory available below 10% | 15 min | warning |
 | The host no longer holds the address the install was made on (§13c) | 5 min | critical |
 | The system clock is not synchronized by NTP (§13c) | 30 min | warning |
+| A k3s certificate expires within 30 days (within 7 days, or lapsed: critical) (§13d) | at once | warning |
 | The watchdog's own runs keep failing (`watchdog/run`, below) | 10 min | critical |
 | The watchdog's state file did not parse and was moved aside (`watchdog/state`, below) | at once, once | warning |
 

internal/watchdog/host.go

@@ -1,8 +1,12 @@
 package watchdog
 
 import (
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -14,6 +18,13 @@ const (
 	// clockFor leaves room for an NTP daemon that was just enabled (by the
 	// installer, or after a reboot) to reach its first synchronization.
 	clockFor = 30 * time.Minute
+	// certFor is zero: an expiry date does not heal itself while it waits.
+	certFor = 0
+
+	// certWarnWithin gives a month to find a moment to restart k3s;
+	// certCriticalWithin is the last week.
+	certWarnWithin     = 30 * 24 * time.Hour
+	certCriticalWithin = 7 * 24 * time.Hour
 
 	// staUnsync is STA_UNSYNC from <linux/timex.h>. The kernel holds it set while
 	// no NTP daemon disciplines the clock; chronyd and systemd-timesyncd clear it
@@ -84,3 +95,87 @@ func ClockFinding(status int32, ok bool) *Finding {
 		Hint:      "timedatectl; sudo timedatectl set-ntp true (docs/troubleshooting.md §13c)",
 	}
 }
+
+// K3sCertDirs are where k3s keeps the certificates it issues itself, the set
+// `k3s certificate check` reads: the API server's serving and client
+// certificates, the component and admin client certificates, etcd's, and the
+// agent's (kubelet, kube-proxy). temporary-certs is left out: the API server
+// makes its loopback certificate there on each start, for a hundred years.
+var K3sCertDirs = []string{
+	"/var/lib/rancher/k3s/server/tls",
+	"/var/lib/rancher/k3s/server/tls/etcd",
+	"/var/lib/rancher/k3s/server/tls/kube-controller-manager",
+	"/var/lib/rancher/k3s/server/tls/kube-scheduler",
+	"/var/lib/rancher/k3s/agent",
+}
+
+// CertFinding reports the certificate in the *.crt files under dirs that
+// expires first, once it is within certWarnWithin of expiring (certCriticalWithin,
+// or past it: critical). k3s issues its client and serving certificates for a
+// year and renews the ones close to expiry only as it starts, so a host that
+// runs a year without restarting k3s loses its API server and its kubelet the
+// day they lapse. Its CA certificates last ten years and no restart renews
+// them. Only the public .crt files are read, never the keys beside them; a
+// directory that does not exist (a host without k3s) reports nothing, and a
+// file that does not parse is skipped.
+func CertFinding(dirs []string, now time.Time) *Finding {
+	var soonest *x509.Certificate
+	var file string
+	for _, dir := range dirs {
+		// A directory or file that cannot be read reads as empty.
+		entries, _ := os.ReadDir(dir)
+		for _, e := range entries {
+			if !strings.HasSuffix(e.Name(), ".crt") {
+				continue
+			}
+			path := filepath.Join(dir, e.Name())
+			data, _ := os.ReadFile(path)
+			// k3s writes a leaf followed by the CA that signed it.
+			for {
+				var block *pem.Block
+				if block, data = pem.Decode(data); block == nil {
+					break
+				}
+				c, err := x509.ParseCertificate(block.Bytes)
+				if err != nil {
+					continue
+				}
+				if soonest == nil || c.NotAfter.Before(soonest.NotAfter) {
+					soonest, file = c, path
+				}
+			}
+		}
+	}
+	if soonest == nil {
+		return nil
+	}
+	left := soonest.NotAfter.Sub(now)
+	if left >= certWarnWithin {
+		return nil
+	}
+	sev := Warning
+	if left < certCriticalWithin {
+		sev = Critical
+	}
+	when := soonest.NotAfter.UTC().Format("2006-01-02 15:04 UTC")
+	f := &Finding{Key: "k3s-certs", Severity: sev, For: certFor}
+	// x509 holds a certificate valid through the instant of NotAfter.
+	if left < 0 {
+		f.Summary = fmt.Sprintf("k3s 证书 %s（%s）已于 %s 过期：k3s 组件之间无法再认证，集群 API、节点和面板对服务器的管理都会中断",
+			file, soonest.Subject.CommonName, when)
+		f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expired at %s: the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working",
+			file, soonest.Subject.CommonName, when)
+	} else {
+		days := int(left / (24 * time.Hour))
+		f.Summary = fmt.Sprintf("k3s 证书 %s（%s）将于 %s 过期（还剩 %d 天）：过期后 k3s 组件之间无法认证，集群 API、节点和面板对服务器的管理都会中断",
+			file, soonest.Subject.CommonName, when, days)
+		f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expires at %s (%d days left): once it does, the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working",
+			file, soonest.Subject.CommonName, when, days)
+	}
+	if soonest.IsCA {
+		f.Hint = "a k3s CA certificate, which no restart renews: rotate it with `k3s certificate rotate-ca` (docs/troubleshooting.md §13d)"
+	} else {
+		f.Hint = "sudo systemctl restart k3s: k3s renews its certificates near expiry as it starts, and running game servers keep running; check with sudo k3s certificate check (docs/troubleshooting.md §13d)"
+	}
+	return f
+}

internal/watchdog/host_test.go

@@ -1,9 +1,20 @@
 package watchdog
 
 import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestAddressFinding(t *testing.T) {
@@ -53,3 +64,123 @@ func TestClockFinding(t *testing.T) {
 		t.Fatalf("unreadable status: got %+v", f)
 	}
 }
+
+// certPEM makes a self-signed certificate that expires at notAfter. Only the
+// dates, the name and the CA bit matter to CertFinding.
+func certPEM(t *testing.T, cn string, notAfter time.Time, ca bool) []byte {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             notAfter.Add(-365 * 24 * time.Hour),
+		NotAfter:              notAfter,
+		IsCA:                  ca,
+		BasicConstraintsValid: true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+}
+
+func writeCert(t *testing.T, path string, pems ...[]byte) {
+	t.Helper()
+	if err := os.WriteFile(path, bytes.Join(pems, nil), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCertFinding(t *testing.T) {
+	now := time.Date(2027, 8, 1, 12, 0, 0, 0, time.UTC)
+	day := 24 * time.Hour
+	ca := certPEM(t, "k3s-client-ca@1790445013", now.Add(3650*day), true)
+
+	// A fresh install: leaves for a year, the CA for ten. Nothing to say, and
+	// a missing directory (a host without k3s) is no finding either.
+	fresh := t.TempDir()
+	writeCert(t, filepath.Join(fresh, "client-admin.crt"), certPEM(t, "system:admin", now.Add(300*day), false), ca)
+	if f := CertFinding([]string{fresh, filepath.Join(fresh, "missing")}, now); f != nil {
+		t.Fatalf("fresh certificates: got %+v", f)
+	}
+	if f := CertFinding(nil, now); f != nil {
+		t.Fatalf("no directories: got %+v", f)
+	}
+
+	// The soonest certificate wins, across directories and within a chained
+	// file; a key file, garbage and a directory named like a certificate are
+	// passed over.
+	server, agent := t.TempDir(), t.TempDir()
+	writeCert(t, filepath.Join(server, "serving-kube-apiserver.crt"), certPEM(t, "kube-apiserver", now.Add(25*day), false), ca)
+	writeCert(t, filepath.Join(agent, "client-kubelet.crt"), ca, certPEM(t, "system:node:felis", now.Add(12*day+time.Hour), false))
+	writeCert(t, filepath.Join(agent, "client-kubelet.key"), certPEM(t, "in a key file", now.Add(time.Hour), false))
+	writeCert(t, filepath.Join(agent, "garbage.crt"), []byte("-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----\n"))
+	if err := os.Mkdir(filepath.Join(agent, "old.crt"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	f := CertFinding([]string{server, agent}, now)
+	if f == nil {
+		t.Fatal("certificate 12 days from expiry: no finding")
+	}
+	if f.Key != "k3s-certs" || f.Severity != Warning || f.For != 0 {
+		t.Fatalf("12 days: key %q severity %v for %v", f.Key, f.Severity, f.For)
+	}
+	want := "the k3s certificate " + filepath.Join(agent, "client-kubelet.crt") + " (system:node:felis) expires at 2027-08-13 13:00 UTC (12 days left)"
+	if !strings.HasPrefix(f.SummaryEN, want+": ") {
+		t.Fatalf("12 days: summary %q, want it to start %q", f.SummaryEN, want)
+	}
+	if !strings.Contains(f.Summary, "（还剩 12 天）") {
+		t.Fatalf("12 days: 中文摘要 %q", f.Summary)
+	}
+	if !strings.HasPrefix(f.Hint, "sudo systemctl restart k3s") {
+		t.Fatalf("12 days: hint %q", f.Hint)
+	}
+
+	// The thresholds: a month out is still quiet, the last week is critical,
+	// and a lapsed certificate says so.
+	edge := t.TempDir()
+	for _, tc := range []struct {
+		left time.Duration
+		sev  Severity
+	}{
+		{30 * day, ""},
+		{30*day - time.Second, Warning},
+		{7 * day, Warning},
+		{7*day - time.Second, Critical},
+		{-time.Hour, Critical},
+	} {
+		writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now.Add(tc.left), false))
+		f := CertFinding([]string{edge}, now)
+		var got Severity
+		if f != nil {
+			got = f.Severity
+		}
+		if got != tc.sev {
+			t.Fatalf("%v left: severity %q, want %q", tc.left, got, tc.sev)
+		}
+	}
+	f = CertFinding([]string{edge}, now)
+	if !strings.Contains(f.SummaryEN, "(system:kube-scheduler) expired at 2027-08-01 11:00 UTC: ") || strings.Contains(f.SummaryEN, "days left") {
+		t.Fatalf("expired: summary %q", f.SummaryEN)
+	}
+	if !strings.Contains(f.Summary, "已于 2027-08-01 11:00 UTC 过期") {
+		t.Fatalf("expired: 中文摘要 %q", f.Summary)
+	}
+
+	writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now, false))
+	if f := CertFinding([]string{edge}, now); f == nil || !strings.Contains(f.SummaryEN, "expires at 2027-08-01 12:00 UTC (0 days left)") {
+		t.Fatalf("expiring this instant: got %+v", f)
+	}
+
+	// A CA near its end needs a rotation, which a restart does not do.
+	old := t.TempDir()
+	writeCert(t, filepath.Join(old, "server-ca.crt"), certPEM(t, "k3s-server-ca@1", now.Add(20*day), true))
+	f = CertFinding([]string{old}, now)
+	if f == nil || !strings.Contains(f.Hint, "k3s certificate rotate-ca") || strings.Contains(f.Hint, "systemctl restart") {
+		t.Fatalf("CA near expiry: got %+v", f)
+	}
+}