feat(restore): 恢复前默认为当前世界做安全快照并串接恢复 Job，快照失败则不恢复；并发恢复另一备份返回 409

internal/maintenance/maintenance.go

@@ -20,6 +20,12 @@
 // A lock older than Grace with no Job behind it is stale (felis-api died between
 // the two writes) and holds nothing.
 //
+// A restore that starts with a safety snapshot is two Jobs in a row: the backup
+// Job carries the restore to run after it (LabelThenRestore), and felis-api
+// creates the restore Job once the backup has succeeded. The backup Job keeps
+// holding the volume, as a restore, from its creation until felis-api has
+// settled what follows it, so nothing can wake the server between the two Jobs.
+//
 // File reads and listings are not holders. They mount the volume read-only for a
 // second or two, and a server starting beside one cannot hurt either side, so
 // nobody waits for them.
@@ -50,6 +56,26 @@ const (
 	// LabelFilesMode is the file-editor operation (list, read, write) a files Job
 	// performs. Only write holds the volume.
 	LabelFilesMode = "felis.lolicon.best/files-mode"
+
+	// LabelThenRestore marks a backup Job that is the safety snapshot in front of
+	// a restore. Its value is the chain's state: ThenRestorePending until felis-api
+	// settles it, then ThenRestoreStarted or ThenRestoreAbandoned. A label, so the
+	// settling loop finds pending chains with a selector.
+	LabelThenRestore = "felis.lolicon.best/then-restore"
+	// AnnotationRestoreRef / AnnotationRestoreBackupID name the backup the chained
+	// restore extracts: its archive ref and its world_backups id.
+	AnnotationRestoreRef      = "felis.lolicon.best/restore-ref"
+	AnnotationRestoreBackupID = "felis.lolicon.best/restore-backup-id"
+	// AnnotationThenRestoreReason is the code saying why a chain was abandoned
+	// (internal/api defines the codes).
+	AnnotationThenRestoreReason = "felis.lolicon.best/then-restore-reason"
+)
+
+// States of LabelThenRestore.
+const (
+	ThenRestorePending   = "pending"
+	ThenRestoreStarted   = "started"
+	ThenRestoreAbandoned = "abandoned"
 )
 
 // Kinds of holder.
@@ -82,6 +108,13 @@ func JobKind(j *batchv1.Job) (string, bool) {
 	return "", false
 }
 
+// RestorePending reports whether j is a safety-snapshot backup Job whose restore
+// felis-api has not yet started or abandoned. Such a Job holds the volume as a
+// restore whether or not it has finished.
+func RestorePending(j *batchv1.Job) bool {
+	return j.Labels[LabelManagedBy] == "felis-backup" && j.Labels[LabelThenRestore] == ThenRestorePending
+}
+
 // JobFinished reports whether a Job has reached a terminal condition. The
 // success/failure-target conditions count as terminal: the Job controller sets
 // them the moment the outcome is decided, before it finishes tearing the pods
@@ -119,13 +152,19 @@ func parseLock(v string) (string, time.Time, bool) {
 }
 
 // Holder reports what, if anything, holds the server's world volume at `now`:
-// the first unfinished maintenance Job among jobs, else a lock in annotations
-// younger than Grace. jobs may contain unrelated Jobs; only the server's own
-// holders count.
+// the first unfinished maintenance Job among jobs (or a safety snapshot whose
+// restore is still pending), else a lock in annotations younger than Grace. jobs
+// may contain unrelated Jobs; only the server's own holders count.
 func Holder(server string, annotations map[string]string, jobs []batchv1.Job, now time.Time) (string, bool) {
 	for i := range jobs {
 		j := &jobs[i]
-		if j.Labels[LabelServer] != server || JobFinished(j) {
+		if j.Labels[LabelServer] != server {
+			continue
+		}
+		if RestorePending(j) {
+			return KindRestore, true
+		}
+		if JobFinished(j) {
 			continue
 		}
 		if kind, ok := JobKind(j); ok {

internal/maintenance/maintenance_test.go

@@ -125,6 +125,52 @@ func TestHolderFromJobs(t *testing.T) {
 	}
 }
 
+// A safety snapshot holds the volume as a restore from its creation until
+// felis-api settles the chain, finished or not, so nothing wakes the server
+// between the backup Job and the restore Job it is followed by.
+func TestHolderFromRestoreChain(t *testing.T) {
+	j, err := backupjob.BackupJob(backupjob.JobParams{
+		Server: "survival", WorldPVC: "world-survival-0", BackupPVC: "felis-backups",
+		Namespace: "minecraft", Image: "felis:1", ConfigSecret: "felis-config", ConfigMount: "/etc/felis",
+		RestoreRef: "/backups/survival/a.tar.gz", RestoreBackupID: "bk-1",
+	})
+	if err != nil {
+		t.Fatalf("BackupJob: %v", err)
+	}
+	if j.Annotations[AnnotationRestoreRef] != "/backups/survival/a.tar.gz" || j.Annotations[AnnotationRestoreBackupID] != "bk-1" {
+		t.Fatalf("chain annotations = %v", j.Annotations)
+	}
+	if !RestorePending(j) {
+		t.Fatalf("a fresh safety snapshot is not a pending chain: labels %v", j.Labels)
+	}
+	for _, tc := range []struct {
+		name string
+		job  batchv1.Job
+		held bool
+		kind string
+	}{
+		{"running", *j, true, KindRestore},
+		{"succeeded, restore not started yet", finished(*j, batchv1.JobComplete), true, KindRestore},
+		{"failed, not settled yet", finished(*j, batchv1.JobFailed), true, KindRestore},
+		{"restore started", settled(finished(*j, batchv1.JobComplete), ThenRestoreStarted), false, ""},
+		{"abandoned", settled(finished(*j, batchv1.JobFailed), ThenRestoreAbandoned), false, ""},
+		{"abandoned while running", settled(*j, ThenRestoreAbandoned), true, KindBackup},
+	} {
+		kind, held := Holder("survival", nil, []batchv1.Job{tc.job}, now)
+		if held != tc.held || kind != tc.kind {
+			t.Errorf("%s: Holder = %q, %v; want %q, %v", tc.name, kind, held, tc.kind, tc.held)
+		}
+	}
+	if plain := backupJob(t, "survival"); RestorePending(&plain) {
+		t.Fatal("a plain backup is a pending chain")
+	}
+}
+
+func settled(j batchv1.Job, state string) batchv1.Job {
+	j.Labels = map[string]string{LabelServer: j.Labels[LabelServer], LabelManagedBy: j.Labels[LabelManagedBy], LabelThenRestore: state}
+	return j
+}
+
 func TestHolderFromLock(t *testing.T) {
 	for _, tc := range []struct {
 		name  string