feat(reaper): 未配置世界目录时渲染只清理备份库存的 CronJob,默认安装也每日删除过期备份,安装器与清单生成器说明回收开关状态
This commit is contained in:
12 files changed
+300
-93
No files matched your search
@@ -39,9 +39,10 @@ type Object interface {
|
||||
// node-pressure eviction shield is the BUILT-IN system-cluster-critical
|
||||
// PriorityClass the pod templates reference (workloads.go controlPlanePriorityName),
|
||||
// not an object this bundle renders.
|
||||
// The reaper CronJob is also part of Workloads, rendered only when the retention
|
||||
// storage topology is supplied (WorldsHostPath + BackupPVC + ArchiveLocalPath —
|
||||
// workloads.go documents the gate and the shape-asserted hostPath caveat). The
|
||||
// The reaper CronJob is also part of Workloads: it reaps worlds when the full
|
||||
// storage topology is supplied (WorldsHostPath + BackupPVC + ArchiveLocalPath)
|
||||
// and only looks after the archive store when the worlds root is missing
|
||||
// (workloads.go documents both gates and the shape-asserted hostPath caveat). The
|
||||
// per-server StatefulSet is never a static manifest — the operator renders it at
|
||||
// reconcile time (internal/operator).
|
||||
func Objects(p Params) []Object {
|
||||
|
||||
@@ -25,17 +25,20 @@ import (
|
||||
// workloads_test.go evaluates those correspondences with the SAME selector
|
||||
// machinery K8s uses, because no cluster runs here.
|
||||
//
|
||||
// The reaper CronJob (spec §18 three-clock retention) renders ONLY when the
|
||||
// The reaper CronJob (spec §18 three-clock retention) reaps worlds ONLY when the
|
||||
// storage topology is supplied — WorldsHostPath + BackupPVC + ArchiveLocalPath,
|
||||
// gated by reaperEnabled. It is opt-in-when-configured rather than always-on
|
||||
// gated by reaperEnabled. World reaping is opt-in-when-configured rather than always-on
|
||||
// because archiving idle worlds means mounting where the worlds physically live,
|
||||
// and the spec keeps that open (§18/§19: tarLocal-on-local-path is the starter,
|
||||
// Longhorn/snapshot the documented evolution, and they do not share a mount
|
||||
// model). The starter model — a node-local worlds-root, exposed through a static
|
||||
// hostPath PV and mounted read-only — is the only one coherent with the operator's per-server
|
||||
// ReadWriteOnce world PVCs (a shared RWX worlds mount would contradict them), so
|
||||
// that is what renders; when the trio is absent no CronJob is emitted, which is
|
||||
// the fail-safe choice for a workload that deletes PVCs. The reaper resolver
|
||||
// that is what renders; when the trio is absent no world is ever reaped, which is
|
||||
// the fail-safe choice for a workload that deletes PVCs. With only the archive
|
||||
// store configured (BackupPVC + ArchiveLocalPath) the CronJob still renders, in
|
||||
// a retention-only shape that expires, reads back and sweeps backups and never
|
||||
// touches a world (retentionEnabled). The reaper resolver
|
||||
// (cmd/felis/reaper.resolveWorldDir) finds worlds either as <WorldsHostPath>/<pvc>
|
||||
// or in the stock local-path layout k3s writes under its storage root
|
||||
// (<pv-name>_<ns>_<pvc-name>, read from the live PVC), so pointing
|
||||
@@ -249,8 +252,11 @@ func Workloads(p Params) []Object {
|
||||
if p.BackupPVC != "" {
|
||||
objs = append(objs, backupPVC(p))
|
||||
}
|
||||
if reaperEnabled(p) {
|
||||
switch {
|
||||
case reaperEnabled(p):
|
||||
objs = append(objs, worldsRootPV(p), worldsRootPVC(p), reaperCronJob(p))
|
||||
case retentionEnabled(p):
|
||||
objs = append(objs, reaperCronJob(p))
|
||||
}
|
||||
return objs
|
||||
}
|
||||
@@ -289,7 +295,16 @@ const controlPlanePriorityName = "system-cluster-critical"
|
||||
// together so a partial configuration fails loudly rather than silently dropping
|
||||
// retention here.
|
||||
func reaperEnabled(p Params) bool {
|
||||
return p.WorldsHostPath != "" && p.BackupPVC != "" && p.ArchiveLocalPath != ""
|
||||
return p.WorldsHostPath != "" && retentionEnabled(p)
|
||||
}
|
||||
|
||||
// retentionEnabled reports whether the archive store can be looked after: the
|
||||
// backup PVC and the path it must be mounted at. Without a worlds root the
|
||||
// same CronJob renders in its retention-only shape (see reaperCronJob), so
|
||||
// backups past their expiry still leave the store on an install that never
|
||||
// reaps worlds; without it they would pile up until the node's disk filled.
|
||||
func retentionEnabled(p Params) bool {
|
||||
return p.BackupPVC != "" && p.ArchiveLocalPath != ""
|
||||
}
|
||||
|
||||
// APIDeployment renders the felis-api Deployment (spec §7). It runs as the
|
||||
@@ -600,37 +615,26 @@ func OperatorDeployment(p Params) *appsv1.Deployment {
|
||||
// pod loud if the worlds-root is absent, rather than silently creating an empty dir
|
||||
// and archiving nothing.
|
||||
//
|
||||
// Pre-conditions are the caller's: reaperCronJob assumes reaperEnabled(p) — it
|
||||
// dereferences WorldsHostPath / BackupPVC / ArchiveLocalPath without re-checking.
|
||||
// Retention only. With no WorldsHostPath the same CronJob runs `felis reaper
|
||||
// --retention-only`: it expires, reads back and sweeps the archive store and
|
||||
// never looks at a server. It then mounts no worlds root, reads no SMTP
|
||||
// password (it sends no warnings) and gets no service account token, since it
|
||||
// never calls the K8s API. It keeps the reaper's root + DAC_OVERRIDE identity:
|
||||
// the archives it reads back and deletes were written by that identity, and by
|
||||
// the backup Jobs.
|
||||
//
|
||||
// Pre-conditions are the caller's: reaperCronJob assumes retentionEnabled(p) —
|
||||
// it dereferences BackupPVC / ArchiveLocalPath without re-checking.
|
||||
func reaperCronJob(p Params) *batchv1.CronJob {
|
||||
p = p.withDefaults()
|
||||
labels := controlPlanePodLabels(ComponentReaper)
|
||||
|
||||
container := corev1.Container{
|
||||
Name: ComponentReaper,
|
||||
Image: p.FelisImage,
|
||||
Command: []string{felisBinaryPath, "reaper"},
|
||||
Args: []string{
|
||||
"--config", configFilePath,
|
||||
"--worlds-root", worldsMountPath,
|
||||
},
|
||||
// The [smtp] relay password for pre-reap warning emails — same optional
|
||||
// Secret felis-api reads. Namespace caveat: a secretKeyRef is
|
||||
// namespace-local, so this resolves against the minecraft-ns felis-smtp
|
||||
// mirror that the "configure email" screen refreshes (the felis-config
|
||||
// mirror it also refreshes is what puts [smtp] in this pod's config).
|
||||
// Absent Secret ⇒ empty env ⇒ the reaper logs suppressed warnings
|
||||
// instead of stamping them (never a failed pod).
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: SMTPPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
|
||||
LocalObjectReference: corev1.LocalObjectReference{Name: SMTPSecretName},
|
||||
Key: SMTPSecretPasswordKey,
|
||||
Optional: boolPtr(true),
|
||||
}}},
|
||||
},
|
||||
Args: []string{"--config", configFilePath, "--retention-only"},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: configVolume, MountPath: configMountPath, ReadOnly: true},
|
||||
{Name: worldsVolume, MountPath: worldsMountPath, ReadOnly: true},
|
||||
{Name: backupVolume, MountPath: p.ArchiveLocalPath},
|
||||
{Name: tmpVolume, MountPath: "/tmp"},
|
||||
},
|
||||
@@ -645,14 +649,6 @@ func reaperCronJob(p Params) *batchv1.CronJob {
|
||||
Secret: &corev1.SecretVolumeSource{SecretName: configSecretName},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: worldsVolume,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: worldsRootName(p), ReadOnly: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: backupVolume,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
@@ -662,6 +658,34 @@ func reaperCronJob(p Params) *batchv1.CronJob {
|
||||
{Name: tmpVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
|
||||
}
|
||||
|
||||
if p.WorldsHostPath != "" {
|
||||
container.Args = []string{"--config", configFilePath, "--worlds-root", worldsMountPath}
|
||||
// The [smtp] relay password for pre-reap warning emails — same optional
|
||||
// Secret felis-api reads. Namespace caveat: a secretKeyRef is
|
||||
// namespace-local, so this resolves against the minecraft-ns felis-smtp
|
||||
// mirror that the "configure email" screen refreshes (the felis-config
|
||||
// mirror it also refreshes is what puts [smtp] in this pod's config).
|
||||
// Absent Secret ⇒ empty env ⇒ the reaper logs suppressed warnings
|
||||
// instead of stamping them (never a failed pod).
|
||||
container.Env = []corev1.EnvVar{
|
||||
{Name: SMTPPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
|
||||
LocalObjectReference: corev1.LocalObjectReference{Name: SMTPSecretName},
|
||||
Key: SMTPSecretPasswordKey,
|
||||
Optional: boolPtr(true),
|
||||
}}},
|
||||
}
|
||||
container.VolumeMounts = append(container.VolumeMounts,
|
||||
corev1.VolumeMount{Name: worldsVolume, MountPath: worldsMountPath, ReadOnly: true})
|
||||
volumes = append(volumes, corev1.Volume{
|
||||
Name: worldsVolume,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: worldsRootName(p), ReadOnly: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return &batchv1.CronJob{
|
||||
TypeMeta: metav1.TypeMeta{APIVersion: "batch/v1", Kind: "CronJob"},
|
||||
// The CronJob lives in the MINECRAFT namespace: a Pod can only mount PVCs
|
||||
@@ -771,7 +795,8 @@ func worldsRootPVC(p Params) *corev1.PersistentVolumeClaim {
|
||||
// reaperPodSpec is the reaper Job's pod template. It lives apart from the CronJob
|
||||
// literal only so the optional node pin is one visible branch: with ReaperNode
|
||||
// set the pod carries a kubernetes.io/hostname selector, keeping the reaper on
|
||||
// the node that actually holds the worlds hostPath on a multi-node cluster.
|
||||
// the node that actually holds the worlds hostPath on a multi-node cluster. The
|
||||
// retention-only shape gets no service account token: it never calls the API.
|
||||
func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume) corev1.PodSpec {
|
||||
spec := corev1.PodSpec{
|
||||
ServiceAccountName: SAReaper,
|
||||
@@ -784,6 +809,9 @@ func reaperPodSpec(p Params, container corev1.Container, volumes []corev1.Volume
|
||||
if p.ReaperNode != "" {
|
||||
spec.NodeSelector = map[string]string{"kubernetes.io/hostname": p.ReaperNode}
|
||||
}
|
||||
if p.WorldsHostPath == "" {
|
||||
spec.AutomountServiceAccountToken = boolPtr(false)
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package platform
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -785,40 +786,83 @@ func reaperParams() Params {
|
||||
return p
|
||||
}
|
||||
|
||||
// TestReaperCronJob_Gating proves the reaper renders iff all three storage
|
||||
// coordinates are present: an incomplete configuration must produce NO CronJob
|
||||
// (the partial-flag mistake is rejected at the CLI; here the renderer fails safe).
|
||||
// TestReaperCronJob_Gating proves the reaper reaps iff all three storage
|
||||
// coordinates are present, and that the archive store alone (backup PVC + its
|
||||
// mount path) still renders the CronJob in its retention-only shape, with no
|
||||
// worlds-root pair: backups must expire on an install that never reaps worlds.
|
||||
// Anything less renders none (the partial-flag mistake is rejected at the CLI;
|
||||
// here the renderer fails safe).
|
||||
func TestReaperCronJob_Gating(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mutate func(p *Params)
|
||||
want bool
|
||||
name string
|
||||
mutate func(p *Params)
|
||||
reap, cron bool
|
||||
}{
|
||||
{"none", func(p *Params) {}, false},
|
||||
{"worlds only", func(p *Params) { p.WorldsHostPath = "/w" }, false},
|
||||
{"worlds+backup", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b" }, false},
|
||||
{"worlds+archive", func(p *Params) { p.WorldsHostPath = "/w"; p.ArchiveLocalPath = "/a" }, false},
|
||||
{"backup+archive (no worlds)", func(p *Params) { p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, false},
|
||||
{"all three", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, true},
|
||||
{"none", func(p *Params) {}, false, false},
|
||||
{"worlds only", func(p *Params) { p.WorldsHostPath = "/w" }, false, false},
|
||||
{"worlds+backup", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b" }, false, false},
|
||||
{"worlds+archive", func(p *Params) { p.WorldsHostPath = "/w"; p.ArchiveLocalPath = "/a" }, false, false},
|
||||
{"backup+archive (no worlds)", func(p *Params) { p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, false, true},
|
||||
{"all three", func(p *Params) { p.WorldsHostPath = "/w"; p.BackupPVC = "b"; p.ArchiveLocalPath = "/a" }, true, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
p := testParams()
|
||||
c.mutate(&p)
|
||||
if got := reaperEnabled(p); got != c.want {
|
||||
t.Errorf("reaperEnabled = %v, want %v", got, c.want)
|
||||
if got := reaperEnabled(p); got != c.reap {
|
||||
t.Errorf("reaperEnabled = %v, want %v", got, c.reap)
|
||||
}
|
||||
cj := findCronJob(Workloads(p))
|
||||
if c.want && cj == nil {
|
||||
t.Error("CronJob must be in Workloads when enabled")
|
||||
objs := Workloads(p)
|
||||
cj := findCronJob(objs)
|
||||
if c.cron != (cj != nil) {
|
||||
t.Fatalf("CronJob rendered = %v, want %v", cj != nil, c.cron)
|
||||
}
|
||||
if !c.want && cj != nil {
|
||||
t.Error("CronJob must NOT be in Workloads when disabled")
|
||||
var pv bool
|
||||
for _, o := range objs {
|
||||
if _, ok := o.(*corev1.PersistentVolume); ok {
|
||||
pv = true
|
||||
}
|
||||
}
|
||||
if pv != c.reap {
|
||||
t.Errorf("worlds-root PV rendered = %v, want %v", pv, c.reap)
|
||||
}
|
||||
if cj != nil {
|
||||
_, ctr := cronPodSpec(t, cj)
|
||||
if got := contains(ctr.Args, "--retention-only"); got == c.reap {
|
||||
t.Errorf("args = %v: --retention-only must be passed exactly when no world is reaped", ctr.Args)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaperCronJob_RetentionOnlyShape pins what the store-only run is left
|
||||
// with: the config and the archive store, and nothing that reaches a world or
|
||||
// the API — no worlds mount, no SMTP password, no service account token.
|
||||
func TestReaperCronJob_RetentionOnlyShape(t *testing.T) {
|
||||
p := reaperParams()
|
||||
p.WorldsHostPath = ""
|
||||
ps, c := cronPodSpec(t, reaperCronJob(p))
|
||||
if want := []string{"--config", configFilePath, "--retention-only"}; !reflect.DeepEqual(c.Args, want) {
|
||||
t.Errorf("args = %v, want %v", c.Args, want)
|
||||
}
|
||||
if volumeByName(ps.Volumes, worldsVolume) != nil || mountByName(c.VolumeMounts, worldsVolume) != nil {
|
||||
t.Error("a retention-only run must not mount a worlds root")
|
||||
}
|
||||
if m := mountByName(c.VolumeMounts, backupVolume); m == nil || m.MountPath != p.ArchiveLocalPath || m.ReadOnly {
|
||||
t.Errorf("backup must be mounted read-write at ArchiveLocalPath %q, got %+v", p.ArchiveLocalPath, m)
|
||||
}
|
||||
if len(c.Env) != 0 {
|
||||
t.Errorf("env = %v, want none (it sends no warnings)", c.Env)
|
||||
}
|
||||
if ps.AutomountServiceAccountToken == nil || *ps.AutomountServiceAccountToken {
|
||||
t.Error("a retention-only run never calls the API and must not mount a service account token")
|
||||
}
|
||||
if c.SecurityContext == nil || c.SecurityContext.Capabilities == nil || len(c.SecurityContext.Capabilities.Add) != 1 || c.SecurityContext.Capabilities.Add[0] != "DAC_OVERRIDE" {
|
||||
t.Error("it reads back and deletes archives other identities wrote, so it keeps DAC_OVERRIDE")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaperCronJob_NodePin proves the optional multi-node pin: no selector by
|
||||
// default (the single-node starter), and exactly the kubernetes.io/hostname
|
||||
// selector when ReaperNode names the node holding the worlds hostPath.
|
||||
|
||||
@@ -413,12 +413,27 @@ func (r *Reaper) RunOnce(ctx context.Context) (Summary, error) {
|
||||
}
|
||||
}
|
||||
|
||||
r.expireBackups(ctx, now, &sum)
|
||||
r.verifyBackups(ctx, now, &sum)
|
||||
r.sweepArchives(ctx, now, &sum)
|
||||
r.retain(ctx, now, &sum)
|
||||
return sum, nil
|
||||
}
|
||||
|
||||
// RunRetention is the archive-store half of RunOnce on its own: backups past
|
||||
// their expiry are deleted, archives are read back, and leftovers are swept,
|
||||
// while no server is looked at and no world is touched. It needs no Cluster,
|
||||
// which is what lets it run where the worlds are out of reach (an install with
|
||||
// no worlds root): backups still leave the store when they expire there.
|
||||
func (r *Reaper) RunRetention(ctx context.Context) Summary {
|
||||
var sum Summary
|
||||
r.retain(ctx, r.now(), &sum)
|
||||
return sum
|
||||
}
|
||||
|
||||
func (r *Reaper) retain(ctx context.Context, now time.Time, sum *Summary) {
|
||||
r.expireBackups(ctx, now, sum)
|
||||
r.verifyBackups(ctx, now, sum)
|
||||
r.sweepArchives(ctx, now, sum)
|
||||
}
|
||||
|
||||
// evaluate handles one server: exemption, reap, or warning. A returned error
|
||||
// means the server was skipped (counted by the caller); nil covers the normal
|
||||
// outcomes including "exempt" and "warned".
|
||||
|
||||
@@ -1222,3 +1222,30 @@ func TestReapFinishesInterruptedReap(t *testing.T) {
|
||||
t.Fatalf("owner %q audits %+v", st.byName["lambda"].OwnerID, st.audits)
|
||||
}
|
||||
}
|
||||
|
||||
// data-durability-17: with no worlds root the store is still looked after.
|
||||
// RunRetention expires, reads back and sweeps without listing a server or
|
||||
// reaching the cluster (nil here, so any call would panic).
|
||||
func TestRunRetentionTouchesNoWorld(t *testing.T) {
|
||||
r, st, _, ar := newReaper(DefaultConfig(),
|
||||
Candidate{Name: "idle", OwnerID: "user-1", LastActiveAt: idleBy(40 * Day)})
|
||||
r.Cluster = nil
|
||||
st.listErr = errors.New("servers must not be listed")
|
||||
ca := checking(r, ar)
|
||||
ca.sweepRemoved = []string{"/archives/.x-1.tar.gz.partial"}
|
||||
st.backups = []*fakeBackup{
|
||||
{id: "gone", server: "s1", ref: "ref-gone", size: 5, status: "present", createdAt: idleBy(120 * Day), expires: idleBy(1 * Day)},
|
||||
{id: "keep", server: "s2", ref: "ref-keep", size: 5, status: "present", createdAt: idleBy(10 * Day), expires: testNow.Add(80 * Day)},
|
||||
}
|
||||
|
||||
sum := r.RunRetention(context.Background())
|
||||
if sum.Evaluated != 0 || sum.WorldsReaped != 0 || ar.archives != 0 {
|
||||
t.Fatalf("retention looked at servers: %+v", sum)
|
||||
}
|
||||
if sum.BackupsExpired != 1 || sum.Verified != 1 || sum.Swept != 1 || sum.Failed() {
|
||||
t.Fatalf("summary = %+v, want 1 expired, 1 read back, 1 swept", sum)
|
||||
}
|
||||
if len(ar.deletes) != 1 || ar.deletes[0] != "ref-gone" {
|
||||
t.Fatalf("deleted archives = %v, want [ref-gone]", ar.deletes)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user