package api import ( "context" "errors" "testing" "time" "felis.lolicon.best/internal/backupjob" "felis.lolicon.best/internal/maintenance" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) var _ ScheduledBackuper = (*backupjob.Backuper)(nil) // fakeScheduledBackuper is a Backuper that can also take a scheduled backup. type fakeScheduledBackuper struct { fakeBackuper err error scheduled []ScheduledCandidate } func (f *fakeScheduledBackuper) BackupScheduled(_ context.Context, name, formerOwner string) error { f.scheduled = append(f.scheduled, ScheduledCandidate{Name: name, OwnerID: formerOwner}) return f.err } type fakeScheduleStore struct { due []ScheduledCandidate gotBefore time.Time calls int } func (f *fakeScheduleStore) ScheduledBackupCandidates(_ context.Context, before time.Time) ([]ScheduledCandidate, error) { f.calls++ f.gotBefore = before return f.due, nil } type fakeWorldJobs struct { running int err error } func (f *fakeWorldJobs) RunningWorldJobs(context.Context) (int, error) { return f.running, f.err } func TestBackupScheduler(t *testing.T) { const every = 24 * time.Hour type rig struct { a *API repo *fakeRepo cl *fakeCluster b *fakeScheduledBackuper store *fakeScheduleStore jobs *fakeWorldJobs s *BackupScheduler } mk := func() rig { repo := newFakeRepo() cl := newFakeCluster() for _, n := range []string{"alpha", "bravo"} { cl.byName[n] = &ServerInfo{Name: n, Phase: "Stopped"} } a := newTestAPI(repo, cl) b := &fakeScheduledBackuper{} a.Backuper = b store := &fakeScheduleStore{due: []ScheduledCandidate{{"alpha", "usr-a"}, {"bravo", "usr-b"}}} jobs := &fakeWorldJobs{} return rig{a, repo, cl, b, store, jobs, &BackupScheduler{API: a, Store: store, Jobs: jobs, Every: every}} } tick := func(t *testing.T, r rig) { t.Helper() if err := r.s.Tick(context.Background()); err != nil { t.Fatalf("Tick: %v", err) } } launched := func(r rig) []ScheduledCandidate { return r.b.scheduled } t.Run("backs up the first due world as its owner, one per tick", func(t *testing.T) { r := mk() tick(t, r) if got := launched(r); len(got) != 1 || got[0] != (ScheduledCandidate{"alpha", "usr-a"}) { t.Fatalf("launched = %+v; want alpha as usr-a only", got) } if want := r.a.now().Add(-every); !r.store.gotBefore.Equal(want) { t.Fatalf("asked for points before %v; want %v", r.store.gotBefore, want) } if len(r.cl.acquired) != 1 || r.cl.acquired[0] != "alpha:"+maintenance.KindBackup || len(r.cl.released) != 1 || r.cl.released[0] != "alpha" { t.Fatalf("lock: acquired %v released %v; want alpha held as a backup and let go", r.cl.acquired, r.cl.released) } if len(r.repo.audits) != 1 || r.repo.audits[0].Action != "backup.scheduled" || r.repo.audits[0].ServerName != "alpha" { t.Fatalf("audits = %+v; want one backup.scheduled of alpha", r.repo.audits) } if at, _ := r.repo.LastBackupRequest(context.Background(), "alpha", time.Time{}); !at.IsZero() { t.Fatal("the scheduled backup started the owner's manual cooldown") } if r.b.calls != 0 { t.Fatal("the scheduled backup went through the manual Backup") } }) t.Run("a running or busy world waits for the next tick", func(t *testing.T) { for _, err := range []error{ErrNotStopped, &MaintenanceBusyError{Kind: maintenance.KindRestore}, ErrMaintenanceInProgress} { r := mk() r.cl.maintErr["alpha"] = err tick(t, r) if got := launched(r); len(got) != 1 || got[0].Name != "bravo" { t.Fatalf("%v: launched = %+v; want bravo", err, got) } delete(r.cl.maintErr, "alpha") r.b.scheduled = nil tick(t, r) if got := launched(r); len(got) != 1 || got[0].Name != "alpha" { t.Fatalf("%v: once alpha is free, launched = %+v; want alpha", err, got) } } }) t.Run("waits while a backup or restore job runs", func(t *testing.T) { r := mk() r.jobs.running = 1 tick(t, r) if len(launched(r)) != 0 || len(r.cl.acquired) != 0 { t.Fatalf("launched %+v, acquired %v beside a running job", launched(r), r.cl.acquired) } r.jobs.running = 0 tick(t, r) if len(launched(r)) != 1 { t.Fatalf("launched = %+v once the job finished", launched(r)) } }) t.Run("pauses while the store is full", func(t *testing.T) { r := mk() r.a.BackupStoreCap = 100 r.repo.backups = []fakeBackup{{view: BackupView{ID: "bk1", ServerName: "bravo", Status: "present", SizeBytes: 100}}} tick(t, r) if len(launched(r)) != 0 { t.Fatalf("launched = %+v into a full store", launched(r)) } r.repo.backups[0].view.SizeBytes = 99 tick(t, r) if len(launched(r)) != 1 { t.Fatalf("launched = %+v below the cap", launched(r)) } }) t.Run("a world whose backup failed is retried after a quarter period", func(t *testing.T) { r := mk() r.store.due = r.store.due[:1] r.b.err = errors.New("apiserver down") if err := r.s.Tick(context.Background()); err == nil { t.Fatal("Tick hid the failed launch") } if len(r.cl.released) != 1 { t.Fatalf("released = %v; the lock must go when the launch fails", r.cl.released) } r.b.err = nil start := r.a.now() r.a.Now = func() time.Time { return start.Add(every/4 - time.Minute) } tick(t, r) if len(launched(r)) != 1 { t.Fatalf("launched = %+v; retried before a quarter period", launched(r)) } r.a.Now = func() time.Time { return start.Add(every / 4) } tick(t, r) if len(launched(r)) != 2 { t.Fatalf("launched = %+v; not retried after a quarter period", launched(r)) } }) t.Run("a world without a volume is passed over", func(t *testing.T) { r := mk() r.cl.noWorld["alpha"] = true tick(t, r) if got := launched(r); len(got) != 1 || got[0].Name != "bravo" || len(r.cl.acquired) != 1 { t.Fatalf("launched = %+v, acquired %v; want bravo alone", got, r.cl.acquired) } }) t.Run("a world deleted since the listing is passed over", func(t *testing.T) { r := mk() r.cl.maintErr["alpha"] = ErrNotFound tick(t, r) if got := launched(r); len(got) != 1 || got[0].Name != "bravo" { t.Fatalf("launched = %+v; want bravo", got) } }) t.Run("a lock failure stops the tick", func(t *testing.T) { r := mk() r.cl.maintErr["alpha"] = errors.New("conflict storm") if err := r.s.Tick(context.Background()); err == nil || len(launched(r)) != 0 { t.Fatalf("Tick = %v, launched %+v; want the error and nothing started", err, launched(r)) } }) t.Run("off without a scheduling backuper or a period", func(t *testing.T) { r := mk() r.a.Backuper = &fakeBackuper{} tick(t, r) r2 := mk() r2.s.Every = 0 tick(t, r2) if r.store.calls != 0 || r2.store.calls != 0 || len(r2.b.scheduled) != 0 { t.Fatal("the scheduler ran while off") } }) } // The jobs route marks the executor's scheduled backups, and the scheduler // counts every unfinished backup and restore Job and nothing else. func TestK8sScheduledBackupJobs(t *testing.T) { scheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(scheme); err != nil { t.Fatal(err) } params := func(name string, scheduled bool) backupjob.JobParams { return backupjob.JobParams{ Server: "survival", JobName: name, WorldPVC: "world-survival-0", BackupPVC: "felis-backups", Namespace: "minecraft", Image: "felis:1", ConfigSecret: "felis-config", ConfigMount: "/etc/felis", Scheduled: scheduled, } } scheduled, err := backupjob.BackupJob(params("backup-survival-aa", true)) if err != nil { t.Fatal(err) } plain, err := backupjob.BackupJob(params("backup-survival-bb", false)) if err != nil { t.Fatal(err) } plain.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} restoring := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "restore-other-cc", Labels: map[string]string{jobServerLabel: "other", jobManagedByLabel: jobManagedByRestore}}} foreign := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "files-survival-dd", Labels: map[string]string{jobServerLabel: "survival", jobManagedByLabel: "felis-files"}}} c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(scheduled, plain, restoring, foreign). WithStatusSubresource(&batchv1.Job{}).Build() k := NewK8sJobStatus(c, "minecraft") ctx := context.Background() if n, err := k.RunningWorldJobs(ctx); err != nil || n != 2 { t.Fatalf("RunningWorldJobs = %d, %v; want the scheduled backup and the restore", n, err) } jobs, err := k.LatestJobs(ctx, "survival") if err != nil { t.Fatal(err) } marks := map[string]bool{} for _, j := range jobs { marks[j.Name] = j.Scheduled } if len(marks) != 2 || !marks["backup-survival-aa"] || marks["backup-survival-bb"] { t.Fatalf("scheduled marks = %v; want only backup-survival-aa", marks) } }