feat(reaper): 未配置世界目录时渲染只清理备份库存的 CronJob,默认安装也每日删除过期备份,安装器与清单生成器说明回收开关状态

This commit is contained in:
Lemon-miaow committed 2026-09-25 03:42:55 +08:00
1 parent fe15b56074
commit 0770a4d676
12 files changed
+300 -93

No files matched your search

+4 -3
View File
@@ -39,9 +39,10 @@ type Object interface {
// node-pressure eviction shield is the BUILT-IN system-cluster-critical
// PriorityClass the pod templates reference (workloads.go controlPlanePriorityName),
// not an object this bundle renders.
// The reaper CronJob is also part of Workloads, rendered only when the retention
// storage topology is supplied (WorldsHostPath + BackupPVC + ArchiveLocalPath —
// workloads.go documents the gate and the shape-asserted hostPath caveat). The
// The reaper CronJob is also part of Workloads: it reaps worlds when the full
// storage topology is supplied (WorldsHostPath + BackupPVC + ArchiveLocalPath)
// and only looks after the archive store when the worlds root is missing
// (workloads.go documents both gates and the shape-asserted hostPath caveat). The
// per-server StatefulSet is never a static manifest — the operator renders it at
// reconcile time (internal/operator).
func Objects(p Params) []Object {
+65 -37
View File
@@ -25,17 +25,20 @@ import (
// workloads_test.go evaluates those correspondences with the SAME selector
// machinery K8s uses, because no cluster runs here.
//
// The reaper CronJob (spec §18 three-clock retention) renders ONLY when the
// The reaper CronJob (spec §18 three-clock retention) reaps worlds ONLY when the
// storage topology is supplied — WorldsHostPath + BackupPVC + ArchiveLocalPath,
// gated by reaperEnabled. It is opt-in-when-configured rather than always-on
// gated by reaperEnabled. World reaping is opt-in-when-configured rather than always-on
// because archiving idle worlds means mounting where the worlds physically live,
// and the spec keeps that open (§18/§19: tarLocal-on-local-path is the starter,
// Longhorn/snapshot the documented evolution, and they do not share a mount
// model). The starter model — a node-local worlds-root, exposed through a static
// hostPath PV and mounted read-only — is the only one coherent with the operator's per-server
// ReadWriteOnce world PVCs (a shared RWX worlds mount would contradict them), so
// that is what renders; when the trio is absent no CronJob is emitted, which is
// the fail-safe choice for a workload that deletes PVCs. The reaper resolver
// that is what renders; when the trio is absent no world is ever reaped, which is
// the fail-safe choice for a workload that deletes PVCs. With only the archive
// store configured (BackupPVC + ArchiveLocalPath) the CronJob still renders, in
// a retention-only shape that expires, reads back and sweeps backups and never
// touches a world (retentionEnabled). The reaper resolver
// (cmd/felis/reaper.resolveWorldDir) finds worlds either as <WorldsHostPath>/<pvc>
// or in the stock local-path layout k3s writes under its storage root
// (<pv-name>_<ns>_<pvc-name>, read from the live PVC), so pointing
@@ -249,8 +252,11 @@ func Workloads(p Params) []Object {
if p.BackupPVC != "" {
objs = append(objs, backupPVC(p))
}
if reaperEnabled(p) {
switch {
case reaperEnabled(p):
objs = append(objs, worldsRootPV(p), worldsRootPVC(p), reaperCronJob(p))
case retentionEnabled(p):
objs = append(objs, reaperCronJob(p))
}
return objs
}
@@ -289,7 +295,16 @@ const controlPlanePriorityName = "system-cluster-critical"
// together so a partial configuration fails loudly rather than silently dropping
// retention here.
func reaperEnabled(p Params) bool {
return p.WorldsHostPath != "" && p.BackupPVC != "" && p.ArchiveLocalPath != ""
return p.WorldsHostPath != "" && retentionEnabled(p)
}
// retentionEnabled reports whether the archive store can be looked after: the
// backup PVC and the path it must be mounted at. Without a worlds root the
// same CronJob renders in its retention-only shape (see reaperCronJob), so
// backups past their expiry still leave the store on an install that never
// reaps worlds; without it they would pile up until the node's disk filled.
func retentionEnabled(p Params) bool {
return p.BackupPVC != "" && p.ArchiveLocalPath != ""
}
// APIDeployment renders the felis-api Deployment (spec §7). It runs as the
@@ -600,37 +615,26 @@ func OperatorDeployment(p Params) *appsv1.Deployment {
// pod loud if the worlds-root is absent, rather than silently creating an empty dir
// and archiving nothing.
//
// Pre-conditions are the caller's: reaperCronJob assumes reaperEnabled(p) — it
// dereferences WorldsHostPath / BackupPVC / ArchiveLocalPath without re-checking.
// Retention only. With no WorldsHostPath the same CronJob runs `felis reaper
// --retention-only`: it expires, reads back and sweeps the archive store and
// never looks at a server. It then mounts no worlds root, reads no SMTP
// password (it sends no warnings) and gets no service account token, since it
// never calls the K8s API. It keeps the reaper's root + DAC_OVERRIDE identity:
// the archives it reads back and deletes were written by that identity, and by
// the backup Jobs.
//
// Pre-conditions are the caller's: reaperCronJob assumes retentionEnabled(p) —
// it dereferences BackupPVC / ArchiveLocalPath without re-checking.
func reaperCronJob(p Params) *batchv1.CronJob {
p = p.withDefaults()
labels := controlPlanePodLabels(ComponentReaper)
container := corev1.Container{
Name: ComponentReaper,
Image: p.FelisImage,
Command: []string{felisBinaryPath, "reaper"},
Args: []string{
"--config", configFilePath,
"--worlds-root", worldsMountPath,
},
// The [smtp] relay password for pre-reap warning emails — same optional
// Secret felis-api reads. Namespace caveat: a secretKeyRef is
// namespace-local, so this resolves against the minecraft-ns felis-smtp
// mirror that the "configure email" screen refreshes (the felis-config
// mirror it also refreshes is what puts [smtp] in this pod's config).
// Absent Secret ⇒ empty env ⇒ the reaper logs suppressed warnings
// instead of stamping them (never a failed pod).
Env: []corev1.EnvVar{
{Name: SMTPPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: SMTPSecretName},
Key: SMTPSecretPasswordKey,
Optional: boolPtr(true),
}}},
},
Args: []string{"--config", configFilePath, "--retention-only"},
VolumeMounts: []corev1.VolumeMount{
{Name: configVolume, MountPath: configMountPath, ReadOnly: true},
{Name: worldsVolume, MountPath: worldsMountPath, ReadOnly: true},
{Name: backupVolume, MountPath: p.ArchiveLocalPath},
{Name: tmpVolume, MountPath: "/tmp"},
},
@@ -645,14 +649,6 @@ func reaperCronJob(p Params) *batchv1.CronJob {
Secret: &corev1.SecretVolumeSource{SecretName: configSecretName},
},
},
{
Name: worldsVolume,
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: worldsRootName(p), ReadOnly: true,
},
},
},
{
Name: backupVolume,
VolumeSource: corev1.VolumeSource{
@@ -662,6 +658,34 @@ func reaperCronJob(p Params) *batchv1.CronJob {
{Name: tmpVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
}
if p.WorldsHostPath != "" {
container.Args = []string{"--config", configFilePath, "--worlds-root", worldsMountPath}
// The [smtp] relay password for pre-reap warning emails — same optional
// Secret felis-api reads. Namespace caveat: a secretKeyRef is
// namespace-local, so this resolves against the minecraft-ns felis-smtp
// mirror that the "configure email" screen refreshes (the felis-config
// mirror it also refreshes is what puts [smtp] in this pod's config).
// Absent Secret ⇒ empty env ⇒ the reaper logs suppressed warnings
// instead of stamping them (never a failed pod).
container.Env = []corev1.EnvVar{
{Name: SMTPPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: SMTPSecretName},
Key: SMTPSecretPasswordKey,
Optional: boolPtr(true),
}}},
}
container.VolumeMounts = append(container.VolumeMounts,
corev1.VolumeMount{Name: worldsVolume, MountPath: worldsMountPath, ReadOnly: true})
volumes = append(volumes, corev1.Volume{
Name: worldsVolume,
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: worldsRootName(p), ReadOnly: true,
},
},
})
}
return &batchv1.CronJob{
TypeMeta: metav1.TypeMeta{APIVersion: "batch/v1", Kind: "CronJob"},
// The CronJob lives in the MINECRAFT namespace: a Pod can only mount PVCs
@@ -771,7 +795,8 @@ func worldsRootPVC(p Params) *corev1.PersistentVolumeClaim {
// reaperPodSpec is the reaper Job's pod template. It lives apart from the CronJob
// literal only so the optional node pin is one visible branch: with ReaperNode
// set the pod carries a kubernetes.io/hostname selector, keeping the reaper on
// the node that actually holds the worlds hostPath on a multi-node cluster.
// the node that actually holds the worlds hostPath on a multi-node cluster. The
// retention-only shape gets no service account token: it never calls the API.
func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume) corev1.PodSpec {
spec := corev1.PodSpec{
ServiceAccountName: SAReaper,
@@ -784,6 +809,9 @@ func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume
if p.ReaperNode != "" {
spec.NodeSelector = map[string]string{"kubernetes.io/hostname": p.ReaperNode}
}
if p.WorldsHostPath == "" {
spec.AutomountServiceAccountToken = boolPtr(false)
}
return spec
}
+63 -19
View File
@@ -2,6 +2,7 @@ package platform
import (
"fmt"
"reflect"
"strings"
"testing"
@@ -785,40 +786,83 @@ func reaperParams() Params {
return p
}
// TestReaperCronJob_Gating proves the reaper renders iff all three storage
// coordinates are present: an incomplete configuration must produce NO CronJob
// (the partial-flag mistake is rejected at the CLI; here the renderer fails safe).
// TestReaperCronJob_Gating proves the reaper reaps iff all three storage
// coordinates are present, and that the archive store alone (backup PVC + its
// mount path) still renders the CronJob in its retention-only shape, with no
// worlds-root pair: backups must expire on an install that never reaps worlds.
// Anything less renders none (the partial-flag mistake is rejected at the CLI;
// here the renderer fails safe).
func TestReaperCronJob_Gating(t *testing.T) {
cases := []struct {
name string
mutate func(p *Params)
want bool
name string
mutate func(p *Params)
reap, cron bool
}{
{"none", func(p *Params) {}, false},
{"worlds only", func(p *Params) { p.WorldsHostPath = "/w" }, false},
{"worlds+backup", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b" }, false},
{"worlds+archive", func(p *Params) { p.WorldsHostPath = "/w"; p.ArchiveLocalPath = "/a" }, false},
{"backup+archive (no worlds)", func(p *Params) { p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, false},
{"all three", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, true},
{"none", func(p *Params) {}, false, false},
{"worlds only", func(p *Params) { p.WorldsHostPath = "/w" }, false, false},
{"worlds+backup", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b" }, false, false},
{"worlds+archive", func(p *Params) { p.WorldsHostPath = "/w"; p.ArchiveLocalPath = "/a" }, false, false},
{"backup+archive (no worlds)", func(p *Params) { p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, false, true},
{"all three", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, true, true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
p := testParams()
c.mutate(&p)
if got := reaperEnabled(p); got != c.want {
t.Errorf("reaperEnabled = %v, want %v", got, c.want)
if got := reaperEnabled(p); got != c.reap {
t.Errorf("reaperEnabled = %v, want %v", got, c.reap)
}
cj := findCronJob(Workloads(p))
if c.want && cj == nil {
t.Error("CronJob must be in Workloads when enabled")
objs := Workloads(p)
cj := findCronJob(objs)
if c.cron != (cj != nil) {
t.Fatalf("CronJob rendered = %v, want %v", cj != nil, c.cron)
}
if !c.want && cj != nil {
t.Error("CronJob must NOT be in Workloads when disabled")
var pv bool
for _, o := range objs {
if _, ok := o.(*corev1.PersistentVolume); ok {
pv = true
}
}
if pv != c.reap {
t.Errorf("worlds-root PV rendered = %v, want %v", pv, c.reap)
}
if cj != nil {
_, ctr := cronPodSpec(t, cj)
if got := contains(ctr.Args, "--retention-only"); got == c.reap {
t.Errorf("args = %v: --retention-only must be passed exactly when no world is reaped", ctr.Args)
}
}
})
}
}
// TestReaperCronJob_RetentionOnlyShape pins what the store-only run is left
// with: the config and the archive store, and nothing that reaches a world or
// the API — no worlds mount, no SMTP password, no service account token.
func TestReaperCronJob_RetentionOnlyShape(t *testing.T) {
p := reaperParams()
p.WorldsHostPath = ""
ps, c := cronPodSpec(t, reaperCronJob(p))
if want := []string{"--config", configFilePath, "--retention-only"}; !reflect.DeepEqual(c.Args, want) {
t.Errorf("args = %v, want %v", c.Args, want)
}
if volumeByName(ps.Volumes, worldsVolume) != nil || mountByName(c.VolumeMounts, worldsVolume) != nil {
t.Error("a retention-only run must not mount a worlds root")
}
if m := mountByName(c.VolumeMounts, backupVolume); m == nil || m.MountPath != p.ArchiveLocalPath || m.ReadOnly {
t.Errorf("backup must be mounted read-write at ArchiveLocalPath %q, got %+v", p.ArchiveLocalPath, m)
}
if len(c.Env) != 0 {
t.Errorf("env = %v, want none (it sends no warnings)", c.Env)
}
if ps.AutomountServiceAccountToken == nil || *ps.AutomountServiceAccountToken {
t.Error("a retention-only run never calls the API and must not mount a service account token")
}
if c.SecurityContext == nil || c.SecurityContext.Capabilities == nil || len(c.SecurityContext.Capabilities.Add) != 1 || c.SecurityContext.Capabilities.Add[0] != "DAC_OVERRIDE" {
t.Error("it reads back and deletes archives other identities wrote, so it keeps DAC_OVERRIDE")
}
}
// TestReaperCronJob_NodePin proves the optional multi-node pin: no selector by
// default (the single-node starter), and exactly the kubernetes.io/hostname
// selector when ReaperNode names the node holding the worlds hostPath.
+18 -3
View File
@@ -413,12 +413,27 @@ func (r *Reaper) RunOnce(ctx context.Context) (Summary, error) {
}
}
r.expireBackups(ctx, now, &sum)
r.verifyBackups(ctx, now, &sum)
r.sweepArchives(ctx, now, &sum)
r.retain(ctx, now, &sum)
return sum, nil
}
// RunRetention is the archive-store half of RunOnce on its own: backups past
// their expiry are deleted, archives are read back, and leftovers are swept,
// while no server is looked at and no world is touched. It needs no Cluster,
// which is what lets it run where the worlds are out of reach (an install with
// no worlds root): backups still leave the store when they expire there.
func (r *Reaper) RunRetention(ctx context.Context) Summary {
var sum Summary
r.retain(ctx, r.now(), &sum)
return sum
}
func (r *Reaper) retain(ctx context.Context, now time.Time, sum *Summary) {
r.expireBackups(ctx, now, sum)
r.verifyBackups(ctx, now, sum)
r.sweepArchives(ctx, now, sum)
}
// evaluate handles one server: exemption, reap, or warning. A returned error
// means the server was skipped (counted by the caller); nil covers the normal
// outcomes including "exempt" and "warned".
+27
View File
@@ -1222,3 +1222,30 @@ func TestReapFinishesInterruptedReap(t *testing.T) {
t.Fatalf("owner %q audits %+v", st.byName["lambda"].OwnerID, st.audits)
}
}
// data-durability-17: with no worlds root the store is still looked after.
// RunRetention expires, reads back and sweeps without listing a server or
// reaching the cluster (nil here, so any call would panic).
func TestRunRetentionTouchesNoWorld(t *testing.T) {
r, st, _, ar := newReaper(DefaultConfig(),
Candidate{Name: "idle", OwnerID: "user-1", LastActiveAt: idleBy(40 * Day)})
r.Cluster = nil
st.listErr = errors.New("servers must not be listed")
ca := checking(r, ar)
ca.sweepRemoved = []string{"/archives/.x-1.tar.gz.partial"}
st.backups = []*fakeBackup{
{id: "gone", server: "s1", ref: "ref-gone", size: 5, status: "present", createdAt: idleBy(120 * Day), expires: idleBy(1 * Day)},
{id: "keep", server: "s2", ref: "ref-keep", size: 5, status: "present", createdAt: idleBy(10 * Day), expires: testNow.Add(80 * Day)},
}
sum := r.RunRetention(context.Background())
if sum.Evaluated != 0 || sum.WorldsReaped != 0 || ar.archives != 0 {
t.Fatalf("retention looked at servers: %+v", sum)
}
if sum.BackupsExpired != 1 || sum.Verified != 1 || sum.Swept != 1 || sum.Failed() {
t.Fatalf("summary = %+v, want 1 expired, 1 read back, 1 swept", sum)
}
if len(ar.deletes) != 1 || ar.deletes[0] != "ref-gone" {
t.Fatalf("deleted archives = %v, want [ref-gone]", ar.deletes)
}
}