diff --git a/cmd/felis/watchdog.go b/cmd/felis/watchdog.go index 27dd054..8487b30 100644 --- a/cmd/felis/watchdog.go +++ b/cmd/felis/watchdog.go @@ -42,6 +42,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose") backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`) diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space") + certDirs := fs.String("k3s-cert-dirs", strings.Join(watchdog.K3sCertDirs, ","), `k3s certificate directories whose *.crt files must not be near expiry ("" skips the check)`) proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`) nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`) controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane") @@ -137,6 +138,7 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { } report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...) add(watchdog.MemoryFinding("/proc/meminfo")) + add(watchdog.CertFinding(splitList(*certDirs), now)) if *nodeIP != "" { if held, err := watchdog.HostAddresses(); err == nil { add(watchdog.AddressFinding(*nodeIP, held)) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c104f75..6bab44f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1509,6 +1509,55 @@ boot before a reboot. --- +## 13d. k3s certificates expire after a year + +k3s issues its own client and serving certificates (the API server's, the +kubelet's, the admin kubeconfig's, etcd's) for 365 days and its CA certificates +for ten years. It renews the client and serving certificates only as it starts: +each start reissues, from the same keys, every one that has expired or is within +120 days of expiring. A host that reboots or takes a k3s upgrade +(`FELIS_UPGRADE_DEPS=1`, docs/operations.md §4) inside that window renews them +unnoticed. A host that runs a year without restarting k3s loses its API server +and its node the day they lapse, and the panel can no longer start, stop or back +up servers. No restart renews a CA certificate; that takes `k3s certificate +rotate-ca` and the procedure in the k3s documentation (Certificate Management). + +The watchdog reads the `*.crt` files under `/var/lib/rancher/k3s/server/tls` +(and its `etcd`, `kube-controller-manager` and `kube-scheduler` directories) and +`/var/lib/rancher/k3s/agent`, the set `k3s certificate check` reads, and never +the keys beside them. It reports the certificate that expires first as +`k3s-certs`: a warning 30 days before, critical in the last 7 days and once it +has lapsed, and the hint says whether a restart renews it (a CA it does not). +`felis watchdog -k3s-cert-dirs ""` turns the check off. [GO-TESTED: `TestCertFinding`] +[VM-VERIFIED: against the real certificates of a v1.36.4+k3s1 install, with the +clock moved ahead] + +Check and renew: + +```sh +sudo k3s certificate check --output table # every certificate, its expiry and residual time +sudo systemctl restart k3s # reissues the ones within 120 days of expiry +sudo k3s certificate check --output table # the renewed ones show about a year again +``` + +A restart stops k3s alone. The unit's `KillMode=process` leaves the containers +running, so game servers, PostgreSQL and the control plane keep serving while +the API server comes back within a minute. [VM-VERIFIED: every container ID and +restart count unchanged across `systemctl restart k3s`] `k3s-killall.sh` stops +every container along with k3s; keep it out of this. + +To renew ahead of the 120-day window (to line it up with a maintenance slot), +reissue every client and serving certificate at once: + +```sh +sudo systemctl stop k3s && sudo k3s certificate rotate && sudo systemctl start k3s +``` + +The panel's own certificate (`/etc/felis/panel-tls.crt`, which the installer +self-signs for 825 days) is a different certificate, outside this check. + +--- + ## 14. Health alerts, and metrics for diagnosis (spec §23) Every full install runs `felis watchdog` from `felis-watchdog.timer`, which @@ -1537,6 +1586,7 @@ Every two minutes the host checks: | Host memory available below 10% | 15 min | warning | | The host no longer holds the address the install was made on (§13c) | 5 min | critical | | The system clock is not synchronized by NTP (§13c) | 30 min | warning | +| A k3s certificate expires within 30 days (within 7 days, or lapsed: critical) (§13d) | at once | warning | | The watchdog's own runs keep failing (`watchdog/run`, below) | 10 min | critical | | The watchdog's state file did not parse and was moved aside (`watchdog/state`, below) | at once, once | warning | diff --git a/internal/watchdog/host.go b/internal/watchdog/host.go index 35e0ecc..a74f6fb 100644 --- a/internal/watchdog/host.go +++ b/internal/watchdog/host.go @@ -1,8 +1,12 @@ package watchdog import ( + "crypto/x509" + "encoding/pem" "fmt" "net" + "os" + "path/filepath" "strings" "time" ) @@ -14,6 +18,13 @@ const ( // clockFor leaves room for an NTP daemon that was just enabled (by the // installer, or after a reboot) to reach its first synchronization. clockFor = 30 * time.Minute + // certFor is zero: an expiry date does not heal itself while it waits. + certFor = 0 + + // certWarnWithin gives a month to find a moment to restart k3s; + // certCriticalWithin is the last week. + certWarnWithin = 30 * 24 * time.Hour + certCriticalWithin = 7 * 24 * time.Hour // staUnsync is STA_UNSYNC from . The kernel holds it set while // no NTP daemon disciplines the clock; chronyd and systemd-timesyncd clear it @@ -84,3 +95,87 @@ func ClockFinding(status int32, ok bool) *Finding { Hint: "timedatectl; sudo timedatectl set-ntp true (docs/troubleshooting.md §13c)", } } + +// K3sCertDirs are where k3s keeps the certificates it issues itself, the set +// `k3s certificate check` reads: the API server's serving and client +// certificates, the component and admin client certificates, etcd's, and the +// agent's (kubelet, kube-proxy). temporary-certs is left out: the API server +// makes its loopback certificate there on each start, for a hundred years. +var K3sCertDirs = []string{ + "/var/lib/rancher/k3s/server/tls", + "/var/lib/rancher/k3s/server/tls/etcd", + "/var/lib/rancher/k3s/server/tls/kube-controller-manager", + "/var/lib/rancher/k3s/server/tls/kube-scheduler", + "/var/lib/rancher/k3s/agent", +} + +// CertFinding reports the certificate in the *.crt files under dirs that +// expires first, once it is within certWarnWithin of expiring (certCriticalWithin, +// or past it: critical). k3s issues its client and serving certificates for a +// year and renews the ones close to expiry only as it starts, so a host that +// runs a year without restarting k3s loses its API server and its kubelet the +// day they lapse. Its CA certificates last ten years and no restart renews +// them. Only the public .crt files are read, never the keys beside them; a +// directory that does not exist (a host without k3s) reports nothing, and a +// file that does not parse is skipped. +func CertFinding(dirs []string, now time.Time) *Finding { + var soonest *x509.Certificate + var file string + for _, dir := range dirs { + // A directory or file that cannot be read reads as empty. + entries, _ := os.ReadDir(dir) + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".crt") { + continue + } + path := filepath.Join(dir, e.Name()) + data, _ := os.ReadFile(path) + // k3s writes a leaf followed by the CA that signed it. + for { + var block *pem.Block + if block, data = pem.Decode(data); block == nil { + break + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + if soonest == nil || c.NotAfter.Before(soonest.NotAfter) { + soonest, file = c, path + } + } + } + } + if soonest == nil { + return nil + } + left := soonest.NotAfter.Sub(now) + if left >= certWarnWithin { + return nil + } + sev := Warning + if left < certCriticalWithin { + sev = Critical + } + when := soonest.NotAfter.UTC().Format("2006-01-02 15:04 UTC") + f := &Finding{Key: "k3s-certs", Severity: sev, For: certFor} + // x509 holds a certificate valid through the instant of NotAfter. + if left < 0 { + f.Summary = fmt.Sprintf("k3s 证书 %s（%s）已于 %s 过期：k3s 组件之间无法再认证，集群 API、节点和面板对服务器的管理都会中断", + file, soonest.Subject.CommonName, when) + f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expired at %s: the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working", + file, soonest.Subject.CommonName, when) + } else { + days := int(left / (24 * time.Hour)) + f.Summary = fmt.Sprintf("k3s 证书 %s（%s）将于 %s 过期（还剩 %d 天）：过期后 k3s 组件之间无法认证，集群 API、节点和面板对服务器的管理都会中断", + file, soonest.Subject.CommonName, when, days) + f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expires at %s (%d days left): once it does, the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working", + file, soonest.Subject.CommonName, when, days) + } + if soonest.IsCA { + f.Hint = "a k3s CA certificate, which no restart renews: rotate it with `k3s certificate rotate-ca` (docs/troubleshooting.md §13d)" + } else { + f.Hint = "sudo systemctl restart k3s: k3s renews its certificates near expiry as it starts, and running game servers keep running; check with sudo k3s certificate check (docs/troubleshooting.md §13d)" + } + return f +} diff --git a/internal/watchdog/host_test.go b/internal/watchdog/host_test.go index a94541b..0780a2a 100644 --- a/internal/watchdog/host_test.go +++ b/internal/watchdog/host_test.go @@ -1,9 +1,20 @@ package watchdog import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" "net" + "os" + "path/filepath" "strings" "testing" + "time" ) func TestAddressFinding(t *testing.T) { @@ -53,3 +64,123 @@ func TestClockFinding(t *testing.T) { t.Fatalf("unreadable status: got %+v", f) } } + +// certPEM makes a self-signed certificate that expires at notAfter. Only the +// dates, the name and the CA bit matter to CertFinding. +func certPEM(t *testing.T, cn string, notAfter time.Time, ca bool) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: cn}, + NotBefore: notAfter.Add(-365 * 24 * time.Hour), + NotAfter: notAfter, + IsCA: ca, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func writeCert(t *testing.T, path string, pems ...[]byte) { + t.Helper() + if err := os.WriteFile(path, bytes.Join(pems, nil), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestCertFinding(t *testing.T) { + now := time.Date(2027, 8, 1, 12, 0, 0, 0, time.UTC) + day := 24 * time.Hour + ca := certPEM(t, "k3s-client-ca@1790445013", now.Add(3650*day), true) + + // A fresh install: leaves for a year, the CA for ten. Nothing to say, and + // a missing directory (a host without k3s) is no finding either. + fresh := t.TempDir() + writeCert(t, filepath.Join(fresh, "client-admin.crt"), certPEM(t, "system:admin", now.Add(300*day), false), ca) + if f := CertFinding([]string{fresh, filepath.Join(fresh, "missing")}, now); f != nil { + t.Fatalf("fresh certificates: got %+v", f) + } + if f := CertFinding(nil, now); f != nil { + t.Fatalf("no directories: got %+v", f) + } + + // The soonest certificate wins, across directories and within a chained + // file; a key file, garbage and a directory named like a certificate are + // passed over. + server, agent := t.TempDir(), t.TempDir() + writeCert(t, filepath.Join(server, "serving-kube-apiserver.crt"), certPEM(t, "kube-apiserver", now.Add(25*day), false), ca) + writeCert(t, filepath.Join(agent, "client-kubelet.crt"), ca, certPEM(t, "system:node:felis", now.Add(12*day+time.Hour), false)) + writeCert(t, filepath.Join(agent, "client-kubelet.key"), certPEM(t, "in a key file", now.Add(time.Hour), false)) + writeCert(t, filepath.Join(agent, "garbage.crt"), []byte("-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----\n")) + if err := os.Mkdir(filepath.Join(agent, "old.crt"), 0o755); err != nil { + t.Fatal(err) + } + f := CertFinding([]string{server, agent}, now) + if f == nil { + t.Fatal("certificate 12 days from expiry: no finding") + } + if f.Key != "k3s-certs" || f.Severity != Warning || f.For != 0 { + t.Fatalf("12 days: key %q severity %v for %v", f.Key, f.Severity, f.For) + } + want := "the k3s certificate " + filepath.Join(agent, "client-kubelet.crt") + " (system:node:felis) expires at 2027-08-13 13:00 UTC (12 days left)" + if !strings.HasPrefix(f.SummaryEN, want+": ") { + t.Fatalf("12 days: summary %q, want it to start %q", f.SummaryEN, want) + } + if !strings.Contains(f.Summary, "（还剩 12 天）") { + t.Fatalf("12 days: 中文摘要 %q", f.Summary) + } + if !strings.HasPrefix(f.Hint, "sudo systemctl restart k3s") { + t.Fatalf("12 days: hint %q", f.Hint) + } + + // The thresholds: a month out is still quiet, the last week is critical, + // and a lapsed certificate says so. + edge := t.TempDir() + for _, tc := range []struct { + left time.Duration + sev Severity + }{ + {30 * day, ""}, + {30*day - time.Second, Warning}, + {7 * day, Warning}, + {7*day - time.Second, Critical}, + {-time.Hour, Critical}, + } { + writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now.Add(tc.left), false)) + f := CertFinding([]string{edge}, now) + var got Severity + if f != nil { + got = f.Severity + } + if got != tc.sev { + t.Fatalf("%v left: severity %q, want %q", tc.left, got, tc.sev) + } + } + f = CertFinding([]string{edge}, now) + if !strings.Contains(f.SummaryEN, "(system:kube-scheduler) expired at 2027-08-01 11:00 UTC: ") || strings.Contains(f.SummaryEN, "days left") { + t.Fatalf("expired: summary %q", f.SummaryEN) + } + if !strings.Contains(f.Summary, "已于 2027-08-01 11:00 UTC 过期") { + t.Fatalf("expired: 中文摘要 %q", f.Summary) + } + + writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now, false)) + if f := CertFinding([]string{edge}, now); f == nil || !strings.Contains(f.SummaryEN, "expires at 2027-08-01 12:00 UTC (0 days left)") { + t.Fatalf("expiring this instant: got %+v", f) + } + + // A CA near its end needs a rotation, which a restart does not do. + old := t.TempDir() + writeCert(t, filepath.Join(old, "server-ca.crt"), certPEM(t, "k3s-server-ca@1", now.Add(20*day), true)) + f = CertFinding([]string{old}, now) + if f == nil || !strings.Contains(f.Hint, "k3s certificate rotate-ca") || strings.Contains(f.Hint, "systemctl restart") { + t.Fatalf("CA near expiry: got %+v", f) + } +}