fix(watchdog): k3s 证书 30 天内到期告警、7 天内转严重，文档说明重启续期与 rotate

	quietPath := fs.String("quiet-file", "/run/felis/watchdog-quiet-until", "Unix time before which nothing is mailed; the installer writes it while it restarts things on purpose")

	backupDir := fs.String("backup-dir", "/var/lib/felis/db-backups", `control-plane database backups to check for freshness ("" skips the check)`)

	diskPaths := fs.String("disk-paths", "/,/var/lib/rancher/k3s,/var/lib/felis", "comma-separated paths whose filesystems must keep free space")

	certDirs := fs.String("k3s-cert-dirs", strings.Join(watchdog.K3sCertDirs, ","), `k3s certificate directories whose *.crt files must not be near expiry ("" skips the check)`)

	proxyAddr := fs.String("proxy-addr", "", `game proxy address to dial, e.g. 127.0.0.1:25565 ("" skips the check)`)

	nodeIP := fs.String("node-ip", "", `the node address the install was made on, which must stay on this host ("" skips the check)`)

	controlNS := fs.String("control-namespace", platform.DefaultControlNamespace, "namespace of the control plane")

	}

	report.Findings = append(report.Findings, watchdog.DiskFindings(splitList(*diskPaths))...)

	add(watchdog.MemoryFinding("/proc/meminfo"))

	add(watchdog.CertFinding(splitList(*certDirs), now))

	if *nodeIP != "" {

		if held, err := watchdog.HostAddresses(); err == nil {

			add(watchdog.AddressFinding(*nodeIP, held))

---

## 13d. k3s certificates expire after a year

k3s issues its own client and serving certificates (the API server's, the

kubelet's, the admin kubeconfig's, etcd's) for 365 days and its CA certificates

for ten years. It renews the client and serving certificates only as it starts:

each start reissues, from the same keys, every one that has expired or is within

120 days of expiring. A host that reboots or takes a k3s upgrade

(`FELIS_UPGRADE_DEPS=1`, docs/operations.md §4) inside that window renews them

unnoticed. A host that runs a year without restarting k3s loses its API server

and its node the day they lapse, and the panel can no longer start, stop or back

up servers. No restart renews a CA certificate; that takes `k3s certificate

rotate-ca` and the procedure in the k3s documentation (Certificate Management).

The watchdog reads the `*.crt` files under `/var/lib/rancher/k3s/server/tls`

(and its `etcd`, `kube-controller-manager` and `kube-scheduler` directories) and

`/var/lib/rancher/k3s/agent`, the set `k3s certificate check` reads, and never

the keys beside them. It reports the certificate that expires first as

`k3s-certs`: a warning 30 days before, critical in the last 7 days and once it

has lapsed, and the hint says whether a restart renews it (a CA it does not).

`felis watchdog -k3s-cert-dirs ""` turns the check off. [GO-TESTED: `TestCertFinding`]

[VM-VERIFIED: against the real certificates of a v1.36.4+k3s1 install, with the

clock moved ahead]

Check and renew:

```sh

sudo k3s certificate check --output table   # every certificate, its expiry and residual time

sudo systemctl restart k3s                  # reissues the ones within 120 days of expiry

sudo k3s certificate check --output table   # the renewed ones show about a year again

```

A restart stops k3s alone. The unit's `KillMode=process` leaves the containers

running, so game servers, PostgreSQL and the control plane keep serving while

the API server comes back within a minute. [VM-VERIFIED: every container ID and

restart count unchanged across `systemctl restart k3s`] `k3s-killall.sh` stops

every container along with k3s; keep it out of this.

To renew ahead of the 120-day window (to line it up with a maintenance slot),

reissue every client and serving certificate at once:

```sh

sudo systemctl stop k3s && sudo k3s certificate rotate && sudo systemctl start k3s

```

The panel's own certificate (`/etc/felis/panel-tls.crt`, which the installer

self-signs for 825 days) is a different certificate, outside this check.

---

## 14. Health alerts, and metrics for diagnosis (spec §23)

Every full install runs `felis watchdog` from `felis-watchdog.timer`, which

| Host memory available below 10% | 15 min | warning |

| The host no longer holds the address the install was made on (§13c) | 5 min | critical |

| The system clock is not synchronized by NTP (§13c) | 30 min | warning |

| A k3s certificate expires within 30 days (within 7 days, or lapsed: critical) (§13d) | at once | warning |

| The watchdog's own runs keep failing (`watchdog/run`, below) | 10 min | critical |

| The watchdog's state file did not parse and was moved aside (`watchdog/state`, below) | at once, once | warning |

package watchdog

import (

	"crypto/x509"

	"encoding/pem"

	"fmt"

	"net"

	"os"

	"path/filepath"

	"strings"

	"time"

)

	// clockFor leaves room for an NTP daemon that was just enabled (by the

	// installer, or after a reboot) to reach its first synchronization.

	clockFor = 30 * time.Minute

	// certFor is zero: an expiry date does not heal itself while it waits.

	certFor = 0

	// certWarnWithin gives a month to find a moment to restart k3s;

	// certCriticalWithin is the last week.

	certWarnWithin     = 30 * 24 * time.Hour

	certCriticalWithin = 7 * 24 * time.Hour

	// staUnsync is STA_UNSYNC from <linux/timex.h>. The kernel holds it set while

	// no NTP daemon disciplines the clock; chronyd and systemd-timesyncd clear it

		Hint:      "timedatectl; sudo timedatectl set-ntp true (docs/troubleshooting.md §13c)",

	}

}

// K3sCertDirs are where k3s keeps the certificates it issues itself, the set

// `k3s certificate check` reads: the API server's serving and client

// certificates, the component and admin client certificates, etcd's, and the

// agent's (kubelet, kube-proxy). temporary-certs is left out: the API server

// makes its loopback certificate there on each start, for a hundred years.

var K3sCertDirs = []string{

	"/var/lib/rancher/k3s/server/tls",

	"/var/lib/rancher/k3s/server/tls/etcd",

	"/var/lib/rancher/k3s/server/tls/kube-controller-manager",

	"/var/lib/rancher/k3s/server/tls/kube-scheduler",

	"/var/lib/rancher/k3s/agent",

}

// CertFinding reports the certificate in the *.crt files under dirs that

// expires first, once it is within certWarnWithin of expiring (certCriticalWithin,

// or past it: critical). k3s issues its client and serving certificates for a

// year and renews the ones close to expiry only as it starts, so a host that

// runs a year without restarting k3s loses its API server and its kubelet the

// day they lapse. Its CA certificates last ten years and no restart renews

// them. Only the public .crt files are read, never the keys beside them; a

// directory that does not exist (a host without k3s) reports nothing, and a

// file that does not parse is skipped.

func CertFinding(dirs []string, now time.Time) *Finding {

	var soonest *x509.Certificate

	var file string

	for _, dir := range dirs {

		// A directory or file that cannot be read reads as empty.

		entries, _ := os.ReadDir(dir)

		for _, e := range entries {

			if !strings.HasSuffix(e.Name(), ".crt") {

				continue

			}

			path := filepath.Join(dir, e.Name())

			data, _ := os.ReadFile(path)

			// k3s writes a leaf followed by the CA that signed it.

			for {

				var block *pem.Block

				if block, data = pem.Decode(data); block == nil {

					break

				}

				c, err := x509.ParseCertificate(block.Bytes)

				if err != nil {

					continue

				}

				if soonest == nil || c.NotAfter.Before(soonest.NotAfter) {

					soonest, file = c, path

				}

			}

		}

	}

	if soonest == nil {

		return nil

	}

	left := soonest.NotAfter.Sub(now)

	if left >= certWarnWithin {

		return nil

	}

	sev := Warning

	if left < certCriticalWithin {

		sev = Critical

	}

	when := soonest.NotAfter.UTC().Format("2006-01-02 15:04 UTC")

	f := &Finding{Key: "k3s-certs", Severity: sev, For: certFor}

	// x509 holds a certificate valid through the instant of NotAfter.

	if left < 0 {

		f.Summary = fmt.Sprintf("k3s 证书 %s（%s）已于 %s 过期：k3s 组件之间无法再认证，集群 API、节点和面板对服务器的管理都会中断",

			file, soonest.Subject.CommonName, when)

		f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expired at %s: the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working",

			file, soonest.Subject.CommonName, when)

	} else {

		days := int(left / (24 * time.Hour))

		f.Summary = fmt.Sprintf("k3s 证书 %s（%s）将于 %s 过期（还剩 %d 天）：过期后 k3s 组件之间无法认证，集群 API、节点和面板对服务器的管理都会中断",

			file, soonest.Subject.CommonName, when, days)

		f.SummaryEN = fmt.Sprintf("the k3s certificate %s (%s) expires at %s (%d days left): once it does, the k3s components can no longer authenticate to each other, so the cluster API, the node and the panel's server management stop working",

			file, soonest.Subject.CommonName, when, days)

	}

	if soonest.IsCA {

		f.Hint = "a k3s CA certificate, which no restart renews: rotate it with `k3s certificate rotate-ca` (docs/troubleshooting.md §13d)"

	} else {

		f.Hint = "sudo systemctl restart k3s: k3s renews its certificates near expiry as it starts, and running game servers keep running; check with sudo k3s certificate check (docs/troubleshooting.md §13d)"

	}

	return f

}

package watchdog

import (

	"bytes"

	"crypto/ecdsa"

	"crypto/elliptic"

	"crypto/rand"

	"crypto/x509"

	"crypto/x509/pkix"

	"encoding/pem"

	"math/big"

	"net"

	"os"

	"path/filepath"

	"strings"

	"testing"

	"time"

)

func TestAddressFinding(t *testing.T) {

		t.Fatalf("unreadable status: got %+v", f)

	}

}

// certPEM makes a self-signed certificate that expires at notAfter. Only the

// dates, the name and the CA bit matter to CertFinding.

func certPEM(t *testing.T, cn string, notAfter time.Time, ca bool) []byte {

	t.Helper()

	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)

	if err != nil {

		t.Fatal(err)

	}

	tmpl := &x509.Certificate{

		SerialNumber:          big.NewInt(1),

		Subject:               pkix.Name{CommonName: cn},

		NotBefore:             notAfter.Add(-365 * 24 * time.Hour),

		NotAfter:              notAfter,

		IsCA:                  ca,

		BasicConstraintsValid: true,

	}

	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)

	if err != nil {

		t.Fatal(err)

	}

	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})

}

func writeCert(t *testing.T, path string, pems ...[]byte) {

	t.Helper()

	if err := os.WriteFile(path, bytes.Join(pems, nil), 0o644); err != nil {

		t.Fatal(err)

	}

}

func TestCertFinding(t *testing.T) {

	now := time.Date(2027, 8, 1, 12, 0, 0, 0, time.UTC)

	day := 24 * time.Hour

	ca := certPEM(t, "k3s-client-ca@1790445013", now.Add(3650*day), true)

	// A fresh install: leaves for a year, the CA for ten. Nothing to say, and

	// a missing directory (a host without k3s) is no finding either.

	fresh := t.TempDir()

	writeCert(t, filepath.Join(fresh, "client-admin.crt"), certPEM(t, "system:admin", now.Add(300*day), false), ca)

	if f := CertFinding([]string{fresh, filepath.Join(fresh, "missing")}, now); f != nil {

		t.Fatalf("fresh certificates: got %+v", f)

	}

	if f := CertFinding(nil, now); f != nil {

		t.Fatalf("no directories: got %+v", f)

	}

	// The soonest certificate wins, across directories and within a chained

	// file; a key file, garbage and a directory named like a certificate are

	// passed over.

	server, agent := t.TempDir(), t.TempDir()

	writeCert(t, filepath.Join(server, "serving-kube-apiserver.crt"), certPEM(t, "kube-apiserver", now.Add(25*day), false), ca)

	writeCert(t, filepath.Join(agent, "client-kubelet.crt"), ca, certPEM(t, "system:node:felis", now.Add(12*day+time.Hour), false))

	writeCert(t, filepath.Join(agent, "client-kubelet.key"), certPEM(t, "in a key file", now.Add(time.Hour), false))

	writeCert(t, filepath.Join(agent, "garbage.crt"), []byte("-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----\n"))

	if err := os.Mkdir(filepath.Join(agent, "old.crt"), 0o755); err != nil {

		t.Fatal(err)

	}

	f := CertFinding([]string{server, agent}, now)

	if f == nil {

		t.Fatal("certificate 12 days from expiry: no finding")

	}

	if f.Key != "k3s-certs" || f.Severity != Warning || f.For != 0 {

		t.Fatalf("12 days: key %q severity %v for %v", f.Key, f.Severity, f.For)

	}

	want := "the k3s certificate " + filepath.Join(agent, "client-kubelet.crt") + " (system:node:felis) expires at 2027-08-13 13:00 UTC (12 days left)"

	if !strings.HasPrefix(f.SummaryEN, want+": ") {

		t.Fatalf("12 days: summary %q, want it to start %q", f.SummaryEN, want)

	}

	if !strings.Contains(f.Summary, "（还剩 12 天）") {

		t.Fatalf("12 days: 中文摘要 %q", f.Summary)

	}

	if !strings.HasPrefix(f.Hint, "sudo systemctl restart k3s") {

		t.Fatalf("12 days: hint %q", f.Hint)

	}

	// The thresholds: a month out is still quiet, the last week is critical,

	// and a lapsed certificate says so.

	edge := t.TempDir()

	for _, tc := range []struct {

		left time.Duration

		sev  Severity

	}{

		{30 * day, ""},

		{30*day - time.Second, Warning},

		{7 * day, Warning},

		{7*day - time.Second, Critical},

		{-time.Hour, Critical},

	} {

		writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now.Add(tc.left), false))

		f := CertFinding([]string{edge}, now)

		var got Severity

		if f != nil {

			got = f.Severity

		}

		if got != tc.sev {

			t.Fatalf("%v left: severity %q, want %q", tc.left, got, tc.sev)

		}

	}

	f = CertFinding([]string{edge}, now)

	if !strings.Contains(f.SummaryEN, "(system:kube-scheduler) expired at 2027-08-01 11:00 UTC: ") || strings.Contains(f.SummaryEN, "days left") {

		t.Fatalf("expired: summary %q", f.SummaryEN)

	}

	if !strings.Contains(f.Summary, "已于 2027-08-01 11:00 UTC 过期") {

		t.Fatalf("expired: 中文摘要 %q", f.Summary)

	}

	writeCert(t, filepath.Join(edge, "client-scheduler.crt"), certPEM(t, "system:kube-scheduler", now, false))

	if f := CertFinding([]string{edge}, now); f == nil || !strings.Contains(f.SummaryEN, "expires at 2027-08-01 12:00 UTC (0 days left)") {

		t.Fatalf("expiring this instant: got %+v", f)

	}

	// A CA near its end needs a rotation, which a restart does not do.

	old := t.TempDir()

	writeCert(t, filepath.Join(old, "server-ca.crt"), certPEM(t, "k3s-server-ca@1", now.Add(20*day), true))

	f = CertFinding([]string{old}, now)

	if f == nil || !strings.Contains(f.Hint, "k3s certificate rotate-ca") || strings.Contains(f.Hint, "systemctl restart") {

		t.Fatalf("CA near expiry: got %+v", f)

	}

}