564 lines
22 KiB
Go
564 lines
22 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"felis.lolicon.best/internal/api"
|
|
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
|
"felis.lolicon.best/internal/naming"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
)
|
|
|
|
// bnRig drives backupNowRun against scripted servers and Jobs on a fake clock that
|
|
// moves only when the run sleeps.
|
|
type bnRig struct {
|
|
out, errw bytes.Buffer
|
|
clock time.Time
|
|
events []string
|
|
worlds []backupNowWorld
|
|
// stopAfter is how many polls a halted server takes to stop; -1 never does.
|
|
stopAfter map[string]int
|
|
polls map[string]int
|
|
reqErr map[string]error
|
|
// states is what the server's new backup Job reports on each poll after the
|
|
// request, the last one repeating; "" is no Job.
|
|
states map[string][]string
|
|
messages map[string]string
|
|
jobPolls map[string]int
|
|
// stopErr fails a server's stop polls; jobsFail fails the nth (1-based) Job
|
|
// list of a server.
|
|
stopErr map[string]error
|
|
jobsFail map[string]int
|
|
jobCalls map[string]int
|
|
// ctx is the run's context, and onAct runs after each halt and request.
|
|
ctx context.Context
|
|
onAct func(event string)
|
|
}
|
|
|
|
func newBNRig(worlds ...backupNowWorld) *bnRig {
|
|
return &bnRig{
|
|
clock: time.Unix(1_800_000_000, 0),
|
|
worlds: worlds,
|
|
stopAfter: map[string]int{},
|
|
polls: map[string]int{},
|
|
reqErr: map[string]error{},
|
|
states: map[string][]string{},
|
|
messages: map[string]string{},
|
|
jobPolls: map[string]int{},
|
|
stopErr: map[string]error{},
|
|
jobsFail: map[string]int{},
|
|
jobCalls: map[string]int{},
|
|
ctx: context.Background(),
|
|
onAct: func(string) {},
|
|
}
|
|
}
|
|
|
|
func (g *bnRig) run(names []string, yes, stop bool) int {
|
|
r := backupNowRun{
|
|
out: &g.out, errw: &g.errw, ns: "minecraft",
|
|
list: func(context.Context) ([]backupNowWorld, error) {
|
|
return append([]backupNowWorld(nil), g.worlds...), nil
|
|
},
|
|
stopped: func(_ context.Context, name string) (bool, error) {
|
|
g.polls[name]++
|
|
if err := g.stopErr[name]; err != nil {
|
|
return false, err
|
|
}
|
|
n := g.stopAfter[name]
|
|
return n >= 0 && g.polls[name] > n, nil
|
|
},
|
|
halt: func(_ context.Context, name string) error {
|
|
g.events = append(g.events, "halt "+name)
|
|
g.onAct("halt " + name)
|
|
return nil
|
|
},
|
|
request: func(_ context.Context, name string) error {
|
|
g.events = append(g.events, "request "+name)
|
|
g.onAct("request " + name)
|
|
if err := g.reqErr[name]; err != nil {
|
|
return err
|
|
}
|
|
g.jobPolls[name] = 0
|
|
return nil
|
|
},
|
|
jobs: func(_ context.Context, name string) ([]api.AsyncJob, error) {
|
|
// Every server has an older finished backup, and a restore Job that
|
|
// shows up with the new backup: neither is the Job to wait for.
|
|
g.jobCalls[name]++
|
|
if g.jobCalls[name] == g.jobsFail[name] {
|
|
return nil, errors.New("the apiserver is gone")
|
|
}
|
|
out := []api.AsyncJob{{Name: "backup-" + name + "-old", Kind: "backup", State: "succeeded"}}
|
|
n, requested := g.jobPolls[name]
|
|
if !requested {
|
|
return out, nil
|
|
}
|
|
g.jobPolls[name] = n + 1
|
|
states := g.states[name]
|
|
if len(states) == 0 {
|
|
states = []string{"succeeded"}
|
|
}
|
|
state := states[min(n, len(states)-1)]
|
|
if state == "" {
|
|
return out, nil
|
|
}
|
|
return append([]api.AsyncJob{
|
|
{Name: "restore-" + name + "-x", Kind: "restore", State: "succeeded"},
|
|
{Name: "backup-" + name + "-new", Kind: "backup", State: state, Message: g.messages[name]},
|
|
}, out...), nil
|
|
},
|
|
now: func() time.Time { return g.clock },
|
|
sleep: func(_ context.Context, d time.Duration) { g.clock = g.clock.Add(d) },
|
|
}
|
|
return r.run(g.ctx, names, yes, stop)
|
|
}
|
|
|
|
func stoppedWorld(name string) backupNowWorld {
|
|
return backupNowWorld{name: name, phase: "Stopped", stopped: true, hasWorld: true}
|
|
}
|
|
|
|
func runningWorld(name string) backupNowWorld {
|
|
return backupNowWorld{name: name, phase: "Running", hasWorld: true}
|
|
}
|
|
|
|
func emptyWorld(name string) backupNowWorld {
|
|
return backupNowWorld{name: name, phase: "Stopped", stopped: true}
|
|
}
|
|
|
|
const (
|
|
bnManualKeep = "Each archive is a manual backup: a server that already holds [archive] manual_keep of them loses its oldest.\n"
|
|
bnStopping = "Stopping disconnects the players on those servers, and they stay stopped afterwards.\n"
|
|
bnOffsite = "The archives reach the off-site bucket with the hourly copy; sudo systemctl start felis-offsite.service sends them now.\n"
|
|
)
|
|
|
|
func TestBackupNowPlanChangesNothing(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
stop bool
|
|
want string
|
|
}{
|
|
{false, "felis backup-now: 4 server(s), backed up one at a time:\n" +
|
|
" alpha Stopped back up\n" +
|
|
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
|
|
" bravo Running skip: running (stop it first, or pass -stop)\n" +
|
|
" delta Starting skip: running (stop it first, or pass -stop)\n" +
|
|
bnManualKeep +
|
|
"Nothing changed. Run again with -yes to back them up.\n"},
|
|
{true, "felis backup-now: 4 server(s), backed up one at a time:\n" +
|
|
" alpha Stopped back up\n" +
|
|
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
|
|
" bravo Running stop, then back up\n" +
|
|
" delta Starting stop, then back up\n" +
|
|
bnManualKeep + bnStopping +
|
|
"Nothing changed. Run again with -yes to back them up.\n"},
|
|
} {
|
|
t.Run(fmt.Sprintf("stop=%v", tc.stop), func(t *testing.T) {
|
|
delta := runningWorld("delta")
|
|
delta.phase = "Starting"
|
|
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), delta)
|
|
if code := g.run(nil, false, tc.stop); code != 0 {
|
|
t.Fatalf("exit = %d, want 0; stderr %q", code, g.errw.String())
|
|
}
|
|
if g.out.String() != tc.want {
|
|
t.Fatalf("plan =\n%s\nwant\n%s", g.out.String(), tc.want)
|
|
}
|
|
if len(g.events) != 0 || len(g.polls) != 0 {
|
|
t.Fatalf("the plan acted: events %v, polls %v", g.events, g.polls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A plan that saves nothing says so, without the manual_keep warning; a running
|
|
// server counts as something to save once -stop is given.
|
|
func TestBackupNowPlanWithNothingToSave(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
stop bool
|
|
want string
|
|
}{
|
|
{false, "felis backup-now: 2 server(s), backed up one at a time:\n" +
|
|
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
|
|
" bravo Running skip: running (stop it first, or pass -stop)\n" +
|
|
"Nothing to back up.\n"},
|
|
{true, "felis backup-now: 2 server(s), backed up one at a time:\n" +
|
|
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
|
|
" bravo Running stop, then back up\n" +
|
|
bnManualKeep + bnStopping +
|
|
"Nothing changed. Run again with -yes to back them up.\n"},
|
|
} {
|
|
g := newBNRig(runningWorld("bravo"), emptyWorld("charlie"))
|
|
if code := g.run(nil, false, tc.stop); code != 0 {
|
|
t.Fatalf("stop=%v: exit = %d, want 0", tc.stop, code)
|
|
}
|
|
if g.out.String() != tc.want {
|
|
t.Fatalf("stop=%v: plan =\n%s\nwant\n%s", tc.stop, g.out.String(), tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBackupNowBacksUpEachWorldInTurn(t *testing.T) {
|
|
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"), emptyWorld("charlie"), stoppedWorld("delta"))
|
|
g.states["alpha"] = []string{"", "running", "succeeded"}
|
|
g.states["delta"] = []string{"running", "failed"}
|
|
g.messages["delta"] = "felis backup: not enough free disk for the archive"
|
|
g.stopAfter["bravo"] = 2
|
|
g.states["bravo"] = []string{"running", "running", "running", "succeeded"}
|
|
|
|
code := g.run(nil, true, true)
|
|
if code != 1 {
|
|
t.Fatalf("exit = %d, want 1 (delta failed)", code)
|
|
}
|
|
if want := []string{"request alpha", "request delta", "halt bravo", "request bravo"}; !reflect.DeepEqual(g.events, want) {
|
|
t.Fatalf("events = %v, want %v", g.events, want)
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), bnStopping+"")
|
|
want := " alpha: backing up\n" +
|
|
" alpha: archived in 4s\n" +
|
|
" delta: backing up\n" +
|
|
" delta: failed: felis backup: not enough free disk for the archive (kubectl -n minecraft logs job/backup-delta-new)\n" +
|
|
" bravo: stopping\n" +
|
|
" bravo: stopped after 4s\n" +
|
|
" bravo: backing up\n" +
|
|
" bravo: archived in 6s\n" +
|
|
"2 backed up, 1 failed, 1 without a world.\n" +
|
|
"Left stopped: bravo. Start them from the panel when you are done.\n" +
|
|
bnOffsite
|
|
if run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
}
|
|
|
|
func TestBackupNowSkipsRunningServersWithoutStop(t *testing.T) {
|
|
g := newBNRig(stoppedWorld("alpha"), runningWorld("bravo"))
|
|
if code := g.run(nil, true, false); code != 1 {
|
|
t.Fatalf("exit = %d, want 1 (bravo was not backed up)", code)
|
|
}
|
|
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
|
|
t.Fatalf("events = %v, want %v", g.events, want)
|
|
}
|
|
if len(g.polls) != 0 {
|
|
t.Fatalf("polled a server it did not stop: %v", g.polls)
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), "Nothing changed")
|
|
if run != "" {
|
|
t.Fatalf("-yes printed the plan's closing line")
|
|
}
|
|
_, run, _ = strings.Cut(g.out.String(), bnManualKeep)
|
|
want := " alpha: backing up\n alpha: archived in 0s\n1 backed up, 1 skipped (running).\n" + bnOffsite
|
|
if run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
}
|
|
|
|
func TestBackupNowExitsCleanWhenEverythingIsSaved(t *testing.T) {
|
|
// -stop with nothing running stops nothing and says nothing about stopping.
|
|
g := newBNRig(stoppedWorld("alpha"), emptyWorld("charlie"))
|
|
if code := g.run(nil, true, true); code != 0 {
|
|
t.Fatalf("exit = %d, want 0; output\n%s", code, g.out.String())
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
|
|
if want := " alpha: backing up\n alpha: archived in 0s\n1 backed up, 1 without a world.\n" + bnOffsite; run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
|
|
g = newBNRig(emptyWorld("charlie"))
|
|
if code := g.run(nil, true, false); code != 0 {
|
|
t.Fatalf("exit = %d, want 0 for a server with nothing to save", code)
|
|
}
|
|
// Nothing to archive: no manual_keep warning, and no off-site hint.
|
|
if want := "felis backup-now: 1 server(s), backed up one at a time:\n" +
|
|
" charlie Stopped skip: no world volume (never started, nothing to save)\n" +
|
|
"0 backed up, 1 without a world.\n"; g.out.String() != want {
|
|
t.Fatalf("output =\n%s\nwant\n%s", g.out.String(), want)
|
|
}
|
|
if len(g.events) != 0 {
|
|
t.Fatalf("events = %v, want none", g.events)
|
|
}
|
|
}
|
|
|
|
func TestBackupNowStopsAtAnUnreachableAPI(t *testing.T) {
|
|
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), runningWorld("charlie"))
|
|
g.reqErr["alpha"] = fmt.Errorf("%w: dial tcp 10.43.0.9:8081: connect: connection refused", errBackupAPIUnreachable)
|
|
if code := g.run(nil, true, true); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
|
|
t.Fatalf("events = %v, want %v: nothing after the API proved unreachable, and no server stopped", g.events, want)
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), bnStopping)
|
|
want := " alpha: failed: felis-api unreachable (a backup needs it alive): dial tcp 10.43.0.9:8081: connect: connection refused\n" +
|
|
"Stopped: nothing more can be backed up until felis-api answers (kubectl -n felis get pods).\n" +
|
|
"0 backed up, 1 failed.\n"
|
|
if run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
|
|
// Any other refusal is that world's alone: the run goes on.
|
|
g = newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
|
|
g.reqErr["alpha"] = errors.New("felis-api: the world is being restored")
|
|
if code := g.run(nil, true, false); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
if want := []string{"request alpha", "request bravo"}; !reflect.DeepEqual(g.events, want) {
|
|
t.Fatalf("events = %v, want %v", g.events, want)
|
|
}
|
|
}
|
|
|
|
func TestBackupNowGivesUpOnAServerThatDoesNotStop(t *testing.T) {
|
|
g := newBNRig(runningWorld("bravo"), runningWorld("echo"))
|
|
g.stopAfter["bravo"] = -1
|
|
if code := g.run(nil, true, true); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
if want := []string{"halt bravo", "halt echo", "request echo"}; !reflect.DeepEqual(g.events, want) {
|
|
t.Fatalf("events = %v, want %v", g.events, want)
|
|
}
|
|
// One poll at the start and one per 2s sleep up to the 10-minute mark.
|
|
if g.polls["bravo"] != 301 {
|
|
t.Fatalf("bravo polled %d times, want 301", g.polls["bravo"])
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), bnStopping)
|
|
want := " bravo: stopping\n" +
|
|
" bravo: failed: did not stop within 10m0s (kubectl -n minecraft describe minecraftserver bravo)\n" +
|
|
" echo: stopping\n" +
|
|
" echo: stopped after 0s\n" +
|
|
" echo: backing up\n" +
|
|
" echo: archived in 0s\n" +
|
|
"1 backed up, 1 failed.\n" +
|
|
"Left stopped: bravo, echo. Start them from the panel when you are done.\n" +
|
|
bnOffsite
|
|
if run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
}
|
|
|
|
func TestBackupNowReportsAJobThatNeverRuns(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
states []string
|
|
polls int
|
|
want string
|
|
}{
|
|
{"never appears", []string{""}, 31,
|
|
"felis-api accepted the backup, but no backup Job appeared within 1m0s (kubectl -n minecraft get jobs)"},
|
|
{"deleted while running", []string{"running", "running", ""}, 3,
|
|
"its backup Job backup-alpha-new was deleted before it finished"},
|
|
{"fails without a message", []string{"failed"}, 1,
|
|
"the backup Job failed (kubectl -n minecraft logs job/backup-alpha-new)"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
g := newBNRig(stoppedWorld("alpha"))
|
|
g.states["alpha"] = tc.states
|
|
if code := g.run(nil, true, false); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
if !strings.Contains(g.out.String(), " alpha: failed: "+tc.want+"\n") {
|
|
t.Fatalf("output =\n%s\nwant the line %q", g.out.String(), tc.want)
|
|
}
|
|
if g.jobPolls["alpha"] != tc.polls {
|
|
t.Fatalf("polled the Jobs %d times after the request, want %d", g.jobPolls["alpha"], tc.polls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBackupNowNamedServers(t *testing.T) {
|
|
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"), stoppedWorld("delta"))
|
|
if code := g.run([]string{"delta", "alpha", "delta"}, true, false); code != 0 {
|
|
t.Fatalf("exit = %d, want 0", code)
|
|
}
|
|
if want := []string{"request delta", "request alpha"}; !reflect.DeepEqual(g.events, want) {
|
|
t.Fatalf("events = %v, want %v", g.events, want)
|
|
}
|
|
if !strings.HasPrefix(g.out.String(), "felis backup-now: 2 server(s), backed up one at a time:\n delta Stopped back up\n alpha Stopped back up\n") {
|
|
t.Fatalf("plan =\n%s", g.out.String())
|
|
}
|
|
|
|
for _, tc := range []struct{ name, want string }{
|
|
{"login", "felis backup-now: login is a system server: its world is rebuilt by felis setup and has no backups\n"},
|
|
{"lobby", "felis backup-now: lobby is a system server: its world is rebuilt by felis setup and has no backups\n"},
|
|
{"nope", "felis backup-now: no server named \"nope\"\n"},
|
|
} {
|
|
g := newBNRig(stoppedWorld("alpha"))
|
|
if code := g.run([]string{"alpha", tc.name}, true, false); code != 2 {
|
|
t.Fatalf("%s: exit = %d, want 2", tc.name, code)
|
|
}
|
|
if g.errw.String() != tc.want {
|
|
t.Fatalf("%s: stderr = %q, want %q", tc.name, g.errw.String(), tc.want)
|
|
}
|
|
if len(g.events) != 0 || g.out.Len() != 0 {
|
|
t.Fatalf("%s: acted on a bad name: events %v, output %q", tc.name, g.events, g.out.String())
|
|
}
|
|
}
|
|
|
|
g = newBNRig()
|
|
if code := g.run(nil, true, false); code != 0 || g.out.String() != "felis backup-now: there are no user servers\n" {
|
|
t.Fatalf("empty fleet: exit %d, output %q", code, g.out.String())
|
|
}
|
|
}
|
|
|
|
// The world list mirrors the backup route's own gate, so the plan says exactly what
|
|
// the route would refuse.
|
|
func TestListBackupNowWorlds(t *testing.T) {
|
|
withStatus := func(ms *v1alpha1.MinecraftServer, ready bool) *v1alpha1.MinecraftServer {
|
|
ms.Status.Ready = ready
|
|
return ms
|
|
}
|
|
pvc := func(server, ns string) *corev1.PersistentVolumeClaim {
|
|
return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: naming.WorldPVCName(server), Namespace: ns}}
|
|
}
|
|
pod := func(name, ns string, labels map[string]string) *corev1.Pod {
|
|
return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}}
|
|
}
|
|
gameLabels := func(server string) map[string]string {
|
|
return map[string]string{v1alpha1.LabelServer: server, v1alpha1.LabelComponent: "server"}
|
|
}
|
|
other := mcServer("zulu", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
|
|
other.Namespace = "elsewhere"
|
|
hotel := mcServer("hotel", v1alpha1.DesiredStopped, "")
|
|
objs := []client.Object{
|
|
mcServer("alpha", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("alpha", haltNS),
|
|
// A backup Job's pod carries the server label with its own component, and
|
|
// a game pod of the same name in another namespace is somebody else's.
|
|
pod("backup-alpha-1-x", haltNS, map[string]string{v1alpha1.LabelServer: "alpha", "app.kubernetes.io/component": "world-backup"}),
|
|
pod("alpha-0", "elsewhere", gameLabels("alpha")),
|
|
withStatus(mcServer("bravo", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning), true), pvc("bravo", haltNS), pod("bravo-0", haltNS, gameLabels("bravo")),
|
|
mcServer("charlie", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped),
|
|
mcServer("delta", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("delta", haltNS), pod("delta-0", haltNS, gameLabels("delta")),
|
|
mcServer("echo", v1alpha1.DesiredStopped, v1alpha1.PhaseStopping), pvc("echo", haltNS),
|
|
mcServer("foxtrot", v1alpha1.DesiredRunning, v1alpha1.PhaseStopped), pvc("foxtrot", haltNS),
|
|
withStatus(mcServer("golf", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), true), pvc("golf", haltNS),
|
|
hotel, pvc("hotel", haltNS),
|
|
mcServer("login", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("login", haltNS),
|
|
mcServer("lobby", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped), pvc("lobby", haltNS),
|
|
other, pvc("zulu", "elsewhere"),
|
|
}
|
|
got, err := listBackupNowWorlds(context.Background(), haltClient(t, objs...), haltNS)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []backupNowWorld{
|
|
{name: "alpha", phase: "Stopped", stopped: true, hasWorld: true},
|
|
{name: "bravo", phase: "Running", hasWorld: true},
|
|
{name: "charlie", phase: "Stopped", stopped: true},
|
|
{name: "delta", phase: "Stopped", hasWorld: true}, // its pod is still going
|
|
{name: "echo", phase: "Stopping", hasWorld: true}, // still stopping
|
|
{name: "foxtrot", phase: "Stopped", hasWorld: true}, // asked to start
|
|
{name: "golf", phase: "Stopped", hasWorld: true}, // still reports ready
|
|
{name: "hotel", phase: "Stopped", hasWorld: true}, // not reconciled yet: the desired state shows
|
|
}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("worlds =\n%+v\nwant\n%+v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestBackupNowStopsWhenInterrupted(t *testing.T) {
|
|
t.Run("between worlds", func(t *testing.T) {
|
|
g := newBNRig(stoppedWorld("alpha"), stoppedWorld("bravo"))
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
g.ctx = ctx
|
|
g.onAct = func(e string) {
|
|
if e == "request alpha" {
|
|
cancel()
|
|
}
|
|
}
|
|
if code := g.run(nil, true, false); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
if want := []string{"request alpha"}; !reflect.DeepEqual(g.events, want) {
|
|
t.Fatalf("events = %v, want %v", g.events, want)
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), bnManualKeep)
|
|
want := " alpha: backing up\n alpha: archived in 0s\n" +
|
|
"Interrupted: a backup Job already started runs to its end.\n" +
|
|
"1 backed up.\n" + bnOffsite
|
|
if run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
})
|
|
t.Run("while a Job runs", func(t *testing.T) {
|
|
g := newBNRig(stoppedWorld("alpha"))
|
|
g.states["alpha"] = []string{"running"}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
g.ctx = ctx
|
|
g.onAct = func(string) { cancel() }
|
|
if code := g.run(nil, true, false); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
if g.jobPolls["alpha"] != 1 {
|
|
t.Fatalf("polled the Job %d times after the interrupt, want 1", g.jobPolls["alpha"])
|
|
}
|
|
if !strings.Contains(g.out.String(), " alpha: failed: context canceled\nInterrupted: a backup Job already started runs to its end.\n0 backed up, 1 failed.\n") {
|
|
t.Fatalf("output =\n%s", g.out.String())
|
|
}
|
|
})
|
|
t.Run("while a server stops", func(t *testing.T) {
|
|
g := newBNRig(runningWorld("bravo"))
|
|
g.stopAfter["bravo"] = -1
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
g.ctx = ctx
|
|
g.onAct = func(string) { cancel() }
|
|
if code := g.run(nil, true, true); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
if g.polls["bravo"] != 1 {
|
|
t.Fatalf("polled bravo %d times after the interrupt, want 1", g.polls["bravo"])
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), bnStopping)
|
|
want := " bravo: stopping\n bravo: failed: context canceled\n" +
|
|
"Interrupted: a backup Job already started runs to its end.\n" +
|
|
"0 backed up, 1 failed.\n" +
|
|
"Left stopped: bravo. Start them from the panel when you are done.\n"
|
|
if run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestBackupNowReportsClusterErrors(t *testing.T) {
|
|
g := newBNRig(runningWorld("bravo"))
|
|
g.stopErr["bravo"] = errors.New("the apiserver is gone")
|
|
if code := g.run(nil, true, true); code != 1 {
|
|
t.Fatalf("exit = %d, want 1", code)
|
|
}
|
|
_, run, _ := strings.Cut(g.out.String(), bnStopping)
|
|
// The stop was asked for, so the server stays stopped whatever the poll said.
|
|
want := " bravo: stopping\n bravo: failed: the apiserver is gone\n0 backed up, 1 failed.\n" +
|
|
"Left stopped: bravo. Start them from the panel when you are done.\n"
|
|
if run != want {
|
|
t.Fatalf("run =\n%s\nwant\n%s", run, want)
|
|
}
|
|
|
|
for _, tc := range []struct {
|
|
call int
|
|
events []string
|
|
}{
|
|
{1, nil}, // before the request: nothing is asked for
|
|
{2, []string{"request alpha"}}, // the first poll after it
|
|
} {
|
|
g := newBNRig(stoppedWorld("alpha"))
|
|
g.jobsFail["alpha"] = tc.call
|
|
if code := g.run(nil, true, false); code != 1 {
|
|
t.Fatalf("call %d: exit = %d, want 1", tc.call, code)
|
|
}
|
|
if !reflect.DeepEqual(g.events, tc.events) {
|
|
t.Fatalf("call %d: events = %v, want %v", tc.call, g.events, tc.events)
|
|
}
|
|
if !strings.Contains(g.out.String(), " alpha: failed: list its backup Jobs: the apiserver is gone\n") {
|
|
t.Fatalf("call %d: output =\n%s", tc.call, g.out.String())
|
|
}
|
|
}
|
|
}
|