fix(watchdog): k3s 证书 30 天内到期告警、7 天内转严重，文档说明重启续期与 rotate

2026-09-27 10:09:20 +08:00
1 parent 9d0253a75b
commit d112a43c65
@@ -1,8 +1,12 @@
 package watchdog

 import (
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -14,6 +18,13 @@ const (
 	// clockFor leaves room for an NTP daemon that was just enabled (by the
 	// installer, or after a reboot) to reach its first synchronization.
 	clockFor = 30 * time.Minute
+	// certFor is zero: an expiry date does not heal itself while it waits.
+	certFor = 0
+
+	// certWarnWithin gives a month to find a moment to restart k3s;
+	// certCriticalWithin is the last week.
+	certWarnWithin     = 30 * 24 * time.Hour
+	certCriticalWithin = 7 * 24 * time.Hour

 	// staUnsync is STA_UNSYNC from <linux/timex.h>. The kernel holds it set while
 	// no NTP daemon disciplines the clock; chronyd and systemd-timesyncd clear it
@@ -84,3 +95,87 @@ func ClockFinding(status int32, ok bool) *Finding {
 		Hint:      "timedatectl; sudo timedatectl set-ntp true (docs/troubleshooting.md §13c)",
 	}
 }
+
+// K3sCertDirs are where k3s keeps the certificates it issues itself, the set
+// `k3s certificate check` reads: the API server's serving and client
+// certificates, the component and admin client certificates, etcd's, and the
+// agent's (kubelet, kube-proxy). temporary-certs is left out: the API server
+// makes its loopback certificate there on each start, for a hundred years.
+var K3sCertDirs = []string{
+	"/var/lib/rancher/k3s/server/tls",
+	"/var/lib/rancher/k3s/server/tls/etcd",
+	"/var/lib/rancher/k3s/server/tls/kube-controller-manager",
+	"/var/lib/rancher/k3s/server/tls/kube-scheduler",
+	"/var/lib/rancher/k3s/agent",
+}
+
+// CertFinding reports the certificate in the *.crt files under dirs that
+// expires first, once it is within certWarnWithin of expiring (certCriticalWithin,
+// or past it: critical). k3s issues its client and serving certificates for a
+// year and renews the ones close to expiry only as it starts, so a host that
+// runs a year without restarting k3s loses its API server and its kubelet the
+// day they lapse. Its CA certificates last ten years and no restart renews
+// them. Only the public .crt files are read, never the keys beside them; a
+// directory that does not exist (a host without k3s) reports nothing, and a
+// file that does not parse is skipped.
+func CertFinding(dirs []string, now time.Time) *Finding {
+	var soonest *x509.Certificate
+	var file string
+	for _, dir := range dirs {
+		// A directory or file that cannot be read reads as empty.
+		entries, _ := os.ReadDir(dir)
+		for _, e := range entries {
+			if !strings.HasSuffix(e.Name(), ".crt") {
+				continue
+			}
+			path := filepath.Join(dir, e.Name())
+			data, _ := os.ReadFile(path)
+			// k3s writes a leaf followed by the CA that signed it.
+			for {
+				var block *pem.Block
+				if block, data = pem.Decode(data); block == nil {
+					break
+				}
+				c, err := x509.ParseCertificate(block.Bytes)
+				if err != nil {
+					continue
+				}
+				if soonest == nil || c.NotAfter.Before(soonest.NotAfter) {
+					soonest, file = c, path
+				}
+			}
+		}
+	}
+	if soonest == nil {
+		return nil
+	}
+	left := soonest.NotAfter.Sub(now)
+	if left >= certWarnWithin {
+		return nil
+	}
+	sev := Warning
+	if left < certCriticalWithin {
+		sev = Critical
+	}
+	when := soonest.NotAfter.UTC().Format("2006-01-02 15:04 UTC")
+	f := &Finding{Key: "k3s-certs", Severity: sev, For: certFor}
+	// x509 holds a certificate valid through the instant of NotAfter.
+	if left < 0 {
+		f.Summary = fmt.Sprintf("k3s 证书 %s（%s）已于 %s 过期：k3s 组件之间无法再认证，集群 API、节点和面板对服务器的管理都会中断",
+			file, soonest.Subject.CommonName, when)
+		f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expired at %s: the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working",
+			file, soonest.Subject.CommonName, when)
+	} else {
+		days := int(left / (24 * time.Hour))
+		f.Summary = fmt.Sprintf("k3s 证书 %s（%s）将于 %s 过期（还剩 %d 天）：过期后 k3s 组件之间无法认证，集群 API、节点和面板对服务器的管理都会中断",
+			file, soonest.Subject.CommonName, when, days)
+		f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expires at %s (%d days left): once it does, the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working",
+			file, soonest.Subject.CommonName, when, days)
+	}
+	if soonest.IsCA {
+		f.Hint = "a k3s CA certificate, which no restart renews: rotate it with `k3s certificate rotate-ca` (docs/troubleshooting.md §13d)"
+	} else {
+		f.Hint = "sudo systemctl restart k3s: k3s renews its certificates near expiry as it starts, and running game servers keep running; check with sudo k3s certificate check (docs/troubleshooting.md §13d)"
+	}
+	return f
+}
@@ -1,9 +1,20 @@
 package watchdog

 import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 )

 func TestAddressFinding(t *testing.T) {
@@ -53,3 +64,123 @@ func TestClockFinding(t *testing.T) {
 		t.Fatalf("unreadable status: got %+v", f)
 	}
 }
+
+// certPEM makes a self-signed certificate that expires at notAfter. Only the
+// dates, the name and the CA bit matter to CertFinding.
+func certPEM(t *testing.T, cn string, notAfter time.Time, ca bool) []byte {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             notAfter.Add(-365 * 24 * time.Hour),
+		NotAfter:              notAfter,
+		IsCA:                  ca,
+		BasicConstraintsValid: true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+}
+
+func writeCert(t *testing.T, path string, pems ...[]byte) {
+	t.Helper()
+	if err := os.WriteFile(path, bytes.Join(pems, nil), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCertFinding(t *testing.T) {
+	now := time.Date(2027, 8, 1, 12, 0, 0, 0, time.UTC)
+	day := 24 * time.Hour
+	ca := certPEM(t, "k3s-client-ca@1790445013", now.Add(3650*day), true)
+
+	// A fresh install: leaves for a year, the CA for ten. Nothing to say, and
+	// a missing directory (a host without k3s) is no finding either.
+	fresh := t.TempDir()
+	writeCert(t, filepath.Join(fresh, "client-admin.crt"), certPEM(t, "system:admin", now.Add(300*day), false), ca)
+	if f := CertFinding([]string{fresh, filepath.Join(fresh, "missing")}, now); f != nil {
+		t.Fatalf("fresh certificates: got %+v", f)
+	}
+	if f := CertFinding(nil, now); f != nil {
+		t.Fatalf("no directories: got %+v", f)
+	}
+
+	// The soonest certificate wins, across directories and within a chained
+	// file; a key file, garbage and a directory named like a certificate are
+	// passed over.
+	server, agent := t.TempDir(), t.TempDir()
+	writeCert(t, filepath.Join(server, "serving-kube-apiserver.crt"), certPEM(t, "kube-apiserver", now.Add(25*day), false), ca)
+	writeCert(t, filepath.Join(agent, "client-kubelet.crt"), ca, certPEM(t, "system:node:felis", now.Add(12*day+time.Hour), false))
+	writeCert(t, filepath.Join(agent, "client-kubelet.key"), certPEM(t, "in a key file", now.Add(time.Hour), false))
+	writeCert(t, filepath.Join(agent, "garbage.crt"), []byte("-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----\n"))
+	if err := os.Mkdir(filepath.Join(agent, "old.crt"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	f := CertFinding([]string{server, agent}, now)
+	if f == nil {
+		t.Fatal("certificate 12 days from expiry: no finding")
+	}
+	if f.Key != "k3s-certs" || f.Severity != Warning || f.For != 0 {
+		t.Fatalf("12 days: key %q severity %v for %v", f.Key, f.Severity, f.For)
+	}
+	want := "the k3s certificate " + filepath.Join(agent, "client-kubelet.crt") + " (system:node:felis) expires at 2027-08-13 13:00 UTC (12 days left)"
+	if !strings.HasPrefix(f.SummaryEN, want+": ") {
+		t.Fatalf("12 days: summary %q, want it to start %q", f.SummaryEN, want)
+	}
+	if !strings.Contains(f.Summary, "（还剩 12 天）") {
+		t.Fatalf("12 days: 中文摘要 %q", f.Summary)
+	}
+	if !strings.HasPrefix(f.Hint, "sudo systemctl restart k3s") {
+		t.Fatalf("12 days: hint %q", f.Hint)
+	}
+
+	// The thresholds: a month out is still quiet, the last week is critical,
+	// and a lapsed certificate says so.
+	edge := t.TempDir()
+	for _, tc := range []struct {
+		left time.Duration
+		sev  Severity
+	}{
+		{30 * day, ""},
+		{30*day - time.Second, Warning},
+		{7 * day, Warning},
+		{7*day - time.Second, Critical},
+		{-time.Hour, Critical},
+	} {
+		writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now.Add(tc.left), false))
+		f := CertFinding([]string{edge}, now)
+		var got Severity
+		if f != nil {
+			got = f.Severity
+		}
+		if got != tc.sev {
+			t.Fatalf("%v left: severity %q, want %q", tc.left, got, tc.sev)
+		}
+	}
+	f = CertFinding([]string{edge}, now)
+	if !strings.Contains(f.SummaryEN, "(system:kube-scheduler) expired at 2027-08-01 11:00 UTC: ") || strings.Contains(f.SummaryEN, "days left") {
+		t.Fatalf("expired: summary %q", f.SummaryEN)
+	}
+	if !strings.Contains(f.Summary, "已于 2027-08-01 11:00 UTC 过期") {
+		t.Fatalf("expired: 中文摘要 %q", f.Summary)
+	}
+
+	writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now, false))
+	if f := CertFinding([]string{edge}, now); f == nil || !strings.Contains(f.SummaryEN, "expires at 2027-08-01 12:00 UTC (0 days left)") {
+		t.Fatalf("expiring this instant: got %+v", f)
+	}
+
+	// A CA near its end needs a rotation, which a restart does not do.
+	old := t.TempDir()
+	writeCert(t, filepath.Join(old, "server-ca.crt"), certPEM(t, "k3s-server-ca@1", now.Add(20*day), true))
+	f = CertFinding([]string{old}, now)
+	if f == nil || !strings.Contains(f.Hint, "k3s certificate rotate-ca") || strings.Contains(f.Hint, "systemctl restart") {
+		t.Fatalf("CA near expiry: got %+v", f)
+	}
+}