fix(install): backups exist on a default install; retention resolves real world dirs (#6)
Three faces of one gap, all on the supported install path: - Backup/restore answered 503 out of the box: nothing ever rendered the archive PVC, so FELIS_BACKUP_PVC was unset. The bundle now renders the PVC (Minecraft namespace, RWO 10Gi, cluster default class) and 'felis manifests' names it by default (--backup-pvc= is the explicit no-store shape); bootstrap passes it through so the generated felis.toml [archive] local_path and the jobs' mount path come from one variable. - Retention was unreachable: bootstrap never passed the reaper flags. It now forwards FELIS_WORLDS_HOST_PATH/FELIS_ARCHIVE_LOCAL_PATH, so one env enables the daily CronJob; unset keeps today's fail-safe (no reaper, nothing deleted). - Even when enabled it could not find a world on a stock install: resolveWorldDir now also resolves the exact local-path directory <pv-name>_<ns>_<pvc-name> read from the live PVC's volumeName (never a glob, so a stale deleted PV's bytes can't be archived in place of the current world). Reaper Role gains persistentvolumeclaims:get (weaker than the delete it already held). README (zh/en) stops promising automatic/scheduled backups and states retention is opt-in. bootstrap_test covers the env->flag contract.
This commit is contained in:
14 files changed
+394
-82
No files matched your search
+23
-17
@@ -26,8 +26,10 @@ func (m *multiFlag) Set(v string) error {
|
||||
// felis-reaper identity only when the retention reaper is enabled, gated with
|
||||
// its CronJob), the weak build/restore Job SAs, the build/minecraft
|
||||
// NetworkPolicies, and the running control-plane workloads (felis-api/operator
|
||||
// Deployments + the in-cluster registry Deployment/Service/PVC) — as a single
|
||||
// multi-document YAML stream on stdout, ready for `kubectl apply -f -`.
|
||||
// Deployments + the in-cluster registry Deployment/Service/PVC + the
|
||||
// world-archive PVC that backs backup/restore, unless --backup-pvc is emptied)
|
||||
// — as a single multi-document YAML stream on stdout, ready for
|
||||
// `kubectl apply -f -`.
|
||||
//
|
||||
// It is a pure renderer: it never contacts a cluster and holds no credentials.
|
||||
// --velocity-cidr records the proxy host addresses allowed by the game NetworkPolicy.
|
||||
@@ -44,8 +46,8 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
|
||||
panelNodePort := fs.Int("panel-node-port", int(platform.DefaultPanelNodePort), "NodePort that exposes the built-in HTTPS panel/API origin")
|
||||
felisImage := fs.String("felis-image", "", "container image the felis-api/operator Deployments run, also passed through as FELIS_IMAGE (REQUIRED)")
|
||||
registryImage := fs.String("registry-image", "", "in-cluster registry image (default: registry:2)")
|
||||
backupPVC := fs.String("backup-pvc", "", "name of the backup PVC advertised to the restore executor via FELIS_BACKUP_PVC (default none = restore endpoint returns 503)")
|
||||
worldsHostPath := fs.String("worlds-host-path", "", "node directory under which each world PVC is visible as <path>/<pvc>; enables the reaper CronJob (requires --backup-pvc and --archive-local-path)")
|
||||
backupPVC := fs.String("backup-pvc", "felis-backups", "name of the world-archive PVC this bundle renders in the Minecraft namespace and advertises to the backup/restore executors via FELIS_BACKUP_PVC (default: felis-backups; pass an empty value to render none, leaving backup/restore answering 503)")
|
||||
worldsHostPath := fs.String("worlds-host-path", "", "node directory the reaper reads worlds from: each world PVC resolves as <path>/<pvc>, or as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (k3s storage root: /var/lib/rancher/k3s/storage); enables the reaper CronJob (requires --archive-local-path and a non-empty --backup-pvc)")
|
||||
archiveLocalPath := fs.String("archive-local-path", "", "path the backup PVC is mounted at in the reaper CronJob; MUST equal felis.toml [archive] local_path")
|
||||
var velocityCIDRs multiFlag
|
||||
fs.Var(&velocityCIDRs, "velocity-cidr", "CIDR of a Velocity proxy host allowed to reach game port 25565 (repeatable, REQUIRED)")
|
||||
@@ -82,17 +84,20 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
|
||||
return 2
|
||||
}
|
||||
|
||||
// Retention/reaper rendering is opt-in and needs all three storage coordinates
|
||||
// together: where worlds live (to read+archive them), the backup PVC (to write
|
||||
// archives into), and the path it is mounted at (which MUST equal felis.toml
|
||||
// [archive] local_path so tarLocal's absolute archive refs resolve). A partial
|
||||
// configuration is almost certainly an operator mistake, so fail loud rather than
|
||||
// silently drop retention. Asking for it without the other two is rejected; an
|
||||
// empty trio renders the bundle WITHOUT the reaper and says so.
|
||||
// Retention/reaper rendering is opt-in and needs a storage topology together:
|
||||
// where worlds live (to read+archive them), a backup PVC (to write archives
|
||||
// into — rendered from --backup-pvc), and the path it is mounted at (which MUST
|
||||
// equal felis.toml [archive] local_path so tarLocal's absolute archive refs
|
||||
// resolve). A partial configuration is almost certainly an operator mistake, so
|
||||
// fail loud rather than silently drop retention or render a reaper with nowhere
|
||||
// to write. The backup PVC itself defaults to felis-backups (it is what makes a
|
||||
// default install's backup endpoint work at all); retention additionally needs
|
||||
// --worlds-host-path.
|
||||
if *worldsHostPath != "" {
|
||||
if *backupPVC == "" || *archiveLocalPath == "" {
|
||||
fmt.Fprintln(stderr, "felis manifests: --worlds-host-path enables the reaper CronJob and requires "+
|
||||
"--backup-pvc and --archive-local-path too (--archive-local-path must equal felis.toml [archive] local_path)")
|
||||
"--archive-local-path (must equal felis.toml [archive] local_path) and a non-empty --backup-pvc "+
|
||||
"(the archive store; default felis-backups)")
|
||||
return 2
|
||||
}
|
||||
// The reaper WILL render. Two deployment preconditions this generator cannot
|
||||
@@ -102,15 +107,16 @@ func cmdManifests(args []string, stdout, stderr io.Writer) int {
|
||||
// flag/field docs, but nobody deploying from stdout reads those.)
|
||||
fmt.Fprintf(stderr, "felis manifests: note: rendering the retention reaper CronJob (worlds hostPath %q). "+
|
||||
"Two preconditions are NOT verified here:\n"+
|
||||
" - each world PVC must be visible at %s/<pvc> on the node: a stock local-path-provisioner lays "+
|
||||
"volumes under PV-name paths (.../pvc-<uuid>_<ns>_<pvc>/), so unless the worlds StorageClass is "+
|
||||
"arranged to expose <path>/<pvc>, the reaper tars an empty directory;\n"+
|
||||
" - the node's world volumes must actually live below %s: the reaper resolves a world as "+
|
||||
"%s/<pvc>, then as the stock local-path directory <path>/<pv-name>_<ns>_<pvc-name> (what k3s "+
|
||||
"writes under /var/lib/rancher/k3s/storage). Any other provisioner needs its volumes exposed as "+
|
||||
"<path>/<pvc>, or each candidate's archive fails and the world is preserved;\n"+
|
||||
" - the CronJob sets NO nodeSelector: a single-node starter pins it to the worlds implicitly, but "+
|
||||
"on a multi-node cluster you MUST add a nodeSelector for the node holding the worlds, or the reaper "+
|
||||
"may schedule where the hostPath is empty.\n", *worldsHostPath, *worldsHostPath)
|
||||
"may schedule where the hostPath is empty.\n", *worldsHostPath, *worldsHostPath, *worldsHostPath)
|
||||
} else {
|
||||
fmt.Fprintln(stderr, "felis manifests: note: retention reaper CronJob not rendered "+
|
||||
"(pass --worlds-host-path, --backup-pvc and --archive-local-path to enable it)")
|
||||
"(pass --worlds-host-path and --archive-local-path — the archive PVC defaults to felis-backups — to enable it)")
|
||||
}
|
||||
|
||||
out, err := platform.RenderYAML(platform.Params{
|
||||
|
||||
@@ -77,6 +77,10 @@ func TestManifestsRendersBundle(t *testing.T) {
|
||||
"10.0.0.5/32",
|
||||
// The felis image flows through to the Deployments.
|
||||
"registry.felis.svc:5000/felis:v1",
|
||||
// Backup works out of the box: the archive PVC renders and the api gets
|
||||
// the env that wires the backup/restore executors to it.
|
||||
"name: felis-backups",
|
||||
"name: FELIS_BACKUP_PVC",
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("rendered bundle missing %q", want)
|
||||
@@ -95,15 +99,19 @@ func TestManifestsRendersBundle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestManifestsReaperRequiresTrio proves --worlds-host-path is a fail-loud opt-in:
|
||||
// asking for the reaper without the backup PVC and its mount path (which must equal
|
||||
// [archive] local_path) is rejected rather than silently dropping retention.
|
||||
func TestManifestsReaperRequiresTrio(t *testing.T) {
|
||||
// TestManifestsReaperRequiresStorage proves --worlds-host-path is a fail-loud
|
||||
// opt-in: asking for the reaper without a writable archive store (the backup PVC,
|
||||
// which defaults to felis-backups but can be emptied) and its mount path (which
|
||||
// must equal [archive] local_path) is rejected rather than silently dropping
|
||||
// retention or deleting worlds it could not archive first.
|
||||
func TestManifestsReaperRequiresStorage(t *testing.T) {
|
||||
base := []string{"manifests", "--felis-image", "reg/felis:test", "--velocity-cidr", "10.0.0.5/32", "--worlds-host-path", "/var/lib/felis/worlds"}
|
||||
for _, extra := range [][]string{
|
||||
{}, // neither backup-pvc nor archive-local-path
|
||||
{"--backup-pvc", "felis-backups"}, // missing archive-local-path
|
||||
{"--archive-local-path", "/backups"}, // missing backup-pvc
|
||||
{}, // missing archive-local-path (backup-pvc defaults)
|
||||
{"--backup-pvc", "other"}, // still missing archive-local-path
|
||||
// A reaper with no archive store would have nowhere to write the archive
|
||||
// it must verify before deleting a world; emptying the PVC is rejected.
|
||||
{"--archive-local-path", "/backups", "--backup-pvc="},
|
||||
} {
|
||||
var out, errBuf bytes.Buffer
|
||||
code := run(append(append([]string{}, base...), extra...), &out, &errBuf)
|
||||
@@ -119,6 +127,23 @@ func TestManifestsReaperRequiresTrio(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestManifestsBackupPVCOptOut proves --backup-pvc= renders a bundle with no
|
||||
// archive store at all: no PVC and no FELIS_BACKUP_PVC env, so backup/restore
|
||||
// answer 503 instead of pointing Jobs at a claim nobody provisions.
|
||||
func TestManifestsBackupPVCOptOut(t *testing.T) {
|
||||
var out, errBuf bytes.Buffer
|
||||
code := run([]string{"manifests", "--felis-image", "reg/felis:test",
|
||||
"--velocity-cidr", "10.0.0.5/32", "--backup-pvc="}, &out, &errBuf)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, errBuf.String())
|
||||
}
|
||||
for _, absent := range []string{"felis-backups", "FELIS_BACKUP_PVC"} {
|
||||
if strings.Contains(out.String(), absent) {
|
||||
t.Errorf("--backup-pvc= bundle must not contain %q", absent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestManifestsRendersReaper proves the happy path with the full retention trio:
|
||||
// a batch/v1 CronJob is emitted, named felis-reaper, mounting the backup PVC at the
|
||||
// supplied archive path.
|
||||
@@ -150,7 +175,7 @@ func TestManifestsRendersReaper(t *testing.T) {
|
||||
// this generator cannot verify (else a misarranged hostPath silently no-ops
|
||||
// retention): the <path>/<pvc> arrangement-dependency and the multi-node
|
||||
// nodeSelector hazard.
|
||||
for _, want := range []string{"local-path-provisioner", "nodeSelector"} {
|
||||
for _, want := range []string{"local-path", "nodeSelector"} {
|
||||
if !strings.Contains(errBuf.String(), want) {
|
||||
t.Errorf("reaper render must warn operators about %q on stderr, got %q", want, errBuf.String())
|
||||
}
|
||||
|
||||
+58
-20
@@ -1,9 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -14,6 +16,7 @@ import (
|
||||
"felis.lolicon.best/internal/config"
|
||||
"felis.lolicon.best/internal/reaper"
|
||||
"felis.lolicon.best/internal/store"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
@@ -30,7 +33,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("reaper", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
|
||||
worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>)")
|
||||
worldsRoot := fs.String("worlds-root", "/worlds", "mount root under which world PVCs are visible (tarLocal: <root>/<pvc>, else the stock local-path <root>/<pv-name>_<ns>_<pvc-name>)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
@@ -47,21 +50,8 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
archiver, err := buildArchiver(cfg, *worldsRoot)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis reaper: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
ctx := ctrl.SetupSignalHandler()
|
||||
|
||||
drv, err := store.Open(ctx, cfg.Database.URL)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
defer drv.Close()
|
||||
|
||||
scheme := runtime.NewScheme()
|
||||
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
|
||||
utilruntime.Must(v1alpha1.AddToScheme(scheme))
|
||||
@@ -71,6 +61,19 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
archiver, err := buildArchiver(ctx, cfg, *worldsRoot, cl)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis reaper: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
drv, err := store.Open(ctx, cfg.Database.URL)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
defer drv.Close()
|
||||
|
||||
r := &reaper.Reaper{
|
||||
Cfg: rcfg,
|
||||
Store: reaper.NewPGStore(drv.DB()),
|
||||
@@ -122,22 +125,57 @@ func reaperConfig(cfg *config.Config) (reaper.Config, error) {
|
||||
}
|
||||
|
||||
// buildArchiver constructs the WorldArchiver. Only tarLocal is implemented in
|
||||
// this build; the resolver maps each world PVC to <worldsRoot>/<pvc>, the mount
|
||||
// convention the reaper Job is deployed with.
|
||||
func buildArchiver(cfg *config.Config, worldsRoot string) (backup.WorldArchiver, error) {
|
||||
// this build; the resolver maps each world PVC to its directory under worldsRoot
|
||||
// (resolveWorldDir).
|
||||
func buildArchiver(ctx context.Context, cfg *config.Config, worldsRoot string, cl client.Client) (backup.WorldArchiver, error) {
|
||||
switch cfg.Archive.Store {
|
||||
case "tarLocal":
|
||||
return &backup.TarLocal{
|
||||
BackupRoot: cfg.Archive.LocalPath,
|
||||
Resolve: func(pvc string) (string, error) {
|
||||
return filepath.Join(worldsRoot, pvc), nil
|
||||
},
|
||||
Resolve: resolveWorldDir(ctx, cl, cfg.K8s.Namespace, worldsRoot),
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("[archive] store %q is not implemented in this build (only tarLocal)", cfg.Archive.Store)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveWorldDir maps a world PVC to its directory under worldsRoot, supporting
|
||||
// the two layouts a Felis host actually has:
|
||||
//
|
||||
// 1. <root>/<pvc> — the reaper's documented arrangement (worlds exposed by PVC
|
||||
// name, e.g. via mounting each volume or a crafted storage class).
|
||||
// 2. <root>/<pv-name>_<namespace>_<pvc-name> — what a stock k3s install gets:
|
||||
// local-path-provisioner stores every volume under its storage root as that
|
||||
// exact directory name. Without this arm, retention on a default install could
|
||||
// only ever fail to find a world (a no-op reaper, or worse an operator
|
||||
// arranging paths by hand).
|
||||
//
|
||||
// The second path is derived EXACTLY from the live PVC's spec.volumeName, never
|
||||
// from a glob: a leftover directory of an old, deleted PV must never be mistaken
|
||||
// for the world the PVC currently binds, because the reaper archives the resolved
|
||||
// directory and then deletes that PVC — archiving stale bytes and deleting the
|
||||
// real world would be data loss. When neither path exists the first is returned,
|
||||
// so the archive walk fails loudly against the documented path.
|
||||
func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoot string) backup.PVCResolver {
|
||||
return func(pvc string) (string, error) {
|
||||
direct := filepath.Join(worldsRoot, pvc)
|
||||
if _, err := os.Stat(direct); err == nil {
|
||||
return direct, nil
|
||||
}
|
||||
var claim corev1.PersistentVolumeClaim
|
||||
if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: pvc}, &claim); err != nil {
|
||||
return "", fmt.Errorf("resolve world PVC %s: %w", pvc, err)
|
||||
}
|
||||
if pv := claim.Spec.VolumeName; pv != "" {
|
||||
volDir := filepath.Join(worldsRoot, fmt.Sprintf("%s_%s_%s", pv, claim.Namespace, claim.Name))
|
||||
if _, err := os.Stat(volDir); err == nil {
|
||||
return volDir, nil
|
||||
}
|
||||
}
|
||||
return direct, nil
|
||||
}
|
||||
}
|
||||
|
||||
// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
|
||||
// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
|
||||
func parseSpanDuration(s string) (time.Duration, error) {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
)
|
||||
|
||||
// TestResolveWorldDir pins the two world layouts the reaper must find, and the
|
||||
// fail-closed miss. The stock local-path arm is derived from the live PVC's
|
||||
// volumeName — a name-based guess (glob) could tar a stale deleted PV's bytes and
|
||||
// then delete the current world, which is why it is read from the API instead.
|
||||
func TestResolveWorldDir(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
|
||||
// Arrange a world under the documented <root>/<pvc> layout.
|
||||
named := filepath.Join(root, "world-named-0")
|
||||
if err := os.MkdirAll(named, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Arrange a second world the way k3s local-path stores it.
|
||||
pvDir := filepath.Join(root, "pvc-11111111-2222-3333-4444-555555555555_minecraft_world-live-0")
|
||||
if err := os.MkdirAll(pvDir, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
claim := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "world-live-0", Namespace: "minecraft"},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
VolumeName: "pvc-11111111-2222-3333-4444-555555555555",
|
||||
},
|
||||
}
|
||||
cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(claim).Build()
|
||||
resolve := resolveWorldDir(ctx, cl, "minecraft", root)
|
||||
|
||||
t.Run("documented name layout wins", func(t *testing.T) {
|
||||
got, err := resolve("world-named-0")
|
||||
if err != nil || got != named {
|
||||
t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, named)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stock local-path layout resolves exactly", func(t *testing.T) {
|
||||
got, err := resolve("world-live-0")
|
||||
if err != nil || got != pvDir {
|
||||
t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, pvDir)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("neither layout present falls back to the documented path", func(t *testing.T) {
|
||||
// The claim exists but its directory does not: return the documented path so
|
||||
// the archive walk fails there, and the reaper preserves the world.
|
||||
missing := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "world-gone-0", Namespace: "minecraft"},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "pvc-99999999-0000-0000-0000-000000000000"},
|
||||
}
|
||||
cl := fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(missing).Build()
|
||||
got, err := resolveWorldDir(ctx, cl, "minecraft", root)("world-gone-0")
|
||||
if err != nil || got != filepath.Join(root, "world-gone-0") {
|
||||
t.Fatalf("resolve = (%q, %v), want (%q, nil)", got, err, filepath.Join(root, "world-gone-0"))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown pvc is an error, not a guess", func(t *testing.T) {
|
||||
_, err := resolve("world-unknown-0")
|
||||
if err == nil || !strings.Contains(err.Error(), "resolve world PVC world-unknown-0") {
|
||||
t.Fatalf("err = %v, want a resolve-world-PVC error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user