feat(restore): 恢复前默认为当前世界做安全快照并串接恢复 Job,快照失败则不恢复;并发恢复另一备份返回 409
This commit is contained in:
33 files changed
+1260
-105
No files matched your search
@@ -187,6 +187,24 @@ func (b *Backuper) Backup(ctx context.Context, serverName, formerOwner string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// BackupThenRestore enqueues the safety snapshot in front of a restore: a backup
|
||||
// Job like Backup's, recorded as a pre_restore backup and labelled with the
|
||||
// restore to run once it succeeds (backupID, backupRef). felis-api creates that
|
||||
// restore Job when the snapshot finishes and gives it up if the snapshot fails,
|
||||
// so the world is never overwritten without a way back; the snapshot Job holds
|
||||
// the world volume as a restore until then (internal/maintenance).
|
||||
func (b *Backuper) BackupThenRestore(ctx context.Context, serverName, formerOwner, backupID, backupRef string) error {
|
||||
p := b.jobParams(serverName, formerOwner)
|
||||
p.RestoreRef, p.RestoreBackupID = backupRef, backupID
|
||||
if err := b.Jobs.CreateBackupJob(ctx, p); err != nil {
|
||||
if errors.Is(err, ErrAlreadyExists) {
|
||||
return nil // suffix collision — treat as enqueued
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jobNameSuffix is a short random hex tag that makes each backup Job name unique.
|
||||
// 32 bits is ample: collisions only matter within a single Job's TTL window across
|
||||
// a handful of manual backups.
|
||||
|
||||
@@ -41,3 +41,21 @@ func TestBackupMintsUniqueJobNamePerCall(t *testing.T) {
|
||||
t.Errorf("two backups reused the Job name %q — retry would silently no-op", jobs.got[0].JobName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupThenRestoreChainsTheRestore(t *testing.T) {
|
||||
jobs := &captureJobs{}
|
||||
b := &Backuper{Jobs: jobs, Config: Config{Image: "img", BackupPVC: "pvc"}}
|
||||
if err := b.BackupThenRestore(context.Background(), "survival", "usr-1", "bk-1", "/backups/a.tar.gz"); err != nil {
|
||||
t.Fatalf("BackupThenRestore: %v", err)
|
||||
}
|
||||
if len(jobs.got) != 1 {
|
||||
t.Fatalf("created %d jobs, want 1", len(jobs.got))
|
||||
}
|
||||
p := jobs.got[0]
|
||||
if p.RestoreRef != "/backups/a.tar.gz" || p.RestoreBackupID != "bk-1" || p.FormerOwner != "usr-1" {
|
||||
t.Errorf("params = %+v", p)
|
||||
}
|
||||
if !strings.HasPrefix(p.JobName, BackupJobName("survival")+"-") {
|
||||
t.Errorf("JobName %q", p.JobName)
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,16 @@ const (
|
||||
managedByValue = "felis-backup"
|
||||
componentValue = "world-backup"
|
||||
|
||||
// The restore chain a safety snapshot carries (internal/maintenance keeps the
|
||||
// canonical copies; maintenance_test pins these against them).
|
||||
labelThenRestore = "felis.lolicon.best/then-restore"
|
||||
thenRestorePending = "pending"
|
||||
annotationRestoreRef = "felis.lolicon.best/restore-ref"
|
||||
annotationRestoreBackupID = "felis.lolicon.best/restore-backup-id"
|
||||
|
||||
// ReasonPreRestore is the world_backups reason of a safety snapshot.
|
||||
ReasonPreRestore = "pre_restore"
|
||||
|
||||
worldVolume = "world"
|
||||
backupVolume = "backup"
|
||||
configVolume = "config"
|
||||
@@ -55,6 +65,14 @@ type JobParams struct {
|
||||
RunAsGroup int64
|
||||
FSGroup int64
|
||||
|
||||
// RestoreRef / RestoreBackupID make this backup the safety snapshot in front
|
||||
// of a restore: the Job is labelled as a pending chain and names the backup
|
||||
// felis-api restores once the snapshot succeeds (internal/maintenance). The
|
||||
// snapshot is recorded as ReasonPreRestore, and its prune spares the backup
|
||||
// the restore will extract.
|
||||
RestoreRef string
|
||||
RestoreBackupID string
|
||||
|
||||
TTLAfterFinished time.Duration
|
||||
}
|
||||
|
||||
@@ -106,6 +124,9 @@ func BackupJob(p JobParams) (*batchv1.Job, error) {
|
||||
if p.ConfigSecret == "" {
|
||||
return nil, fmt.Errorf("backup: config secret name is required")
|
||||
}
|
||||
if p.RestoreRef != "" && p.RestoreBackupID == "" {
|
||||
return nil, fmt.Errorf("backup: a chained restore needs the backup id")
|
||||
}
|
||||
limits, err := resourceLimits(p.CPULimit, p.MemLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -129,6 +150,9 @@ func BackupJob(p JobParams) (*batchv1.Job, error) {
|
||||
if p.FormerOwner != "" {
|
||||
args = append(args, "--former-owner", p.FormerOwner)
|
||||
}
|
||||
if p.RestoreRef != "" {
|
||||
args = append(args, "--reason", ReasonPreRestore, "--protect", p.RestoreBackupID)
|
||||
}
|
||||
|
||||
container := corev1.Container{
|
||||
Name: "backup",
|
||||
@@ -167,12 +191,22 @@ func BackupJob(p JobParams) (*batchv1.Job, error) {
|
||||
if name == "" {
|
||||
name = BackupJobName(p.Server)
|
||||
}
|
||||
meta := metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: p.Namespace,
|
||||
Labels: backupLabels(p),
|
||||
}
|
||||
if p.RestoreRef != "" {
|
||||
// On the Job only: felis-api settles the chain by patching this label, and
|
||||
// the pods never need it.
|
||||
meta.Labels[labelThenRestore] = thenRestorePending
|
||||
meta.Annotations = map[string]string{
|
||||
annotationRestoreRef: p.RestoreRef,
|
||||
annotationRestoreBackupID: p.RestoreBackupID,
|
||||
}
|
||||
}
|
||||
job := &batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: p.Namespace,
|
||||
Labels: backupLabels(p),
|
||||
},
|
||||
ObjectMeta: meta,
|
||||
Spec: batchv1.JobSpec{
|
||||
// One shot: a wedged archive must not loop. The TTL GCs the finished Job
|
||||
// so a later backup of the same server is not blocked forever by a stale
|
||||
|
||||
@@ -199,6 +199,44 @@ func TestBackupJobArgsCarryServerAndOwner(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A safety snapshot records itself as pre_restore, spares the backup the chained
|
||||
// restore extracts from its prune, and carries the chain on the Job (only there:
|
||||
// felis-api settles it by patching the Job's label).
|
||||
func TestBackupJobCarriesTheRestoreChain(t *testing.T) {
|
||||
p := sampleJobParams()
|
||||
p.RestoreRef, p.RestoreBackupID = "/backups/survival/a.tar.gz", "bk-1"
|
||||
job, err := BackupJob(p)
|
||||
if err != nil {
|
||||
t.Fatalf("BackupJob: %v", err)
|
||||
}
|
||||
args := job.Spec.Template.Spec.Containers[0].Args
|
||||
if !argsContain(args, "--reason", ReasonPreRestore) || !argsContain(args, "--protect", "bk-1") {
|
||||
t.Errorf("args = %v, want --reason %s --protect bk-1", args, ReasonPreRestore)
|
||||
}
|
||||
if job.Labels[labelThenRestore] != thenRestorePending {
|
||||
t.Errorf("job labels = %v, want %s=%s", job.Labels, labelThenRestore, thenRestorePending)
|
||||
}
|
||||
if _, ok := job.Spec.Template.Labels[labelThenRestore]; ok {
|
||||
t.Errorf("pod template carries the chain label: %v", job.Spec.Template.Labels)
|
||||
}
|
||||
if job.Annotations[annotationRestoreRef] != p.RestoreRef || job.Annotations[annotationRestoreBackupID] != "bk-1" {
|
||||
t.Errorf("job annotations = %v", job.Annotations)
|
||||
}
|
||||
|
||||
plain, err := BackupJob(sampleJobParams())
|
||||
if err != nil {
|
||||
t.Fatalf("BackupJob(plain): %v", err)
|
||||
}
|
||||
if _, ok := plain.Labels[labelThenRestore]; ok || len(plain.Annotations) != 0 {
|
||||
t.Errorf("a plain backup carries a chain: labels %v annotations %v", plain.Labels, plain.Annotations)
|
||||
}
|
||||
for _, a := range plain.Spec.Template.Spec.Containers[0].Args {
|
||||
if a == "--reason" || a == "--protect" {
|
||||
t.Errorf("a plain backup passes %s: %v", a, plain.Spec.Template.Spec.Containers[0].Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupJobRejectsMissingInputs(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -208,6 +246,7 @@ func TestBackupJobRejectsMissingInputs(t *testing.T) {
|
||||
{"no world pvc", func(p *JobParams) { p.WorldPVC = "" }},
|
||||
{"no backup pvc", func(p *JobParams) { p.BackupPVC = "" }},
|
||||
{"no config secret", func(p *JobParams) { p.ConfigSecret = "" }},
|
||||
{"chain without backup id", func(p *JobParams) { p.RestoreRef = "/backups/a.tar.gz" }},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
p := sampleJobParams()
|
||||
|
||||
Reference in new issue
Block a user