feat(restore): 恢复前默认为当前世界做安全快照并串接恢复 Job,快照失败则不恢复;并发恢复另一备份返回 409

This commit is contained in:
Lemon-miaow committed 2026-09-25 02:52:18 +08:00
1 parent d44243c0ac
commit 489eff4494
33 files changed
+1260 -105

No files matched your search

+18
View File
@@ -187,6 +187,24 @@ func (b *Backuper) Backup(ctx context.Context, serverName, formerOwner string) e
return nil
}
// BackupThenRestore enqueues the safety snapshot in front of a restore: a backup
// Job like Backup's, recorded as a pre_restore backup and labelled with the
// restore to run once it succeeds (backupID, backupRef). felis-api creates that
// restore Job when the snapshot finishes and gives it up if the snapshot fails,
// so the world is never overwritten without a way back; the snapshot Job holds
// the world volume as a restore until then (internal/maintenance).
func (b *Backuper) BackupThenRestore(ctx context.Context, serverName, formerOwner, backupID, backupRef string) error {
p := b.jobParams(serverName, formerOwner)
p.RestoreRef, p.RestoreBackupID = backupRef, backupID
if err := b.Jobs.CreateBackupJob(ctx, p); err != nil {
if errors.Is(err, ErrAlreadyExists) {
return nil // suffix collision — treat as enqueued
}
return err
}
return nil
}
// jobNameSuffix is a short random hex tag that makes each backup Job name unique.
// 32 bits is ample: collisions only matter within a single Job's TTL window across
// a handful of manual backups.
+18
View File
@@ -41,3 +41,21 @@ func TestBackupMintsUniqueJobNamePerCall(t *testing.T) {
t.Errorf("two backups reused the Job name %q — retry would silently no-op", jobs.got[0].JobName)
}
}
func TestBackupThenRestoreChainsTheRestore(t *testing.T) {
jobs := &captureJobs{}
b := &Backuper{Jobs: jobs, Config: Config{Image: "img", BackupPVC: "pvc"}}
if err := b.BackupThenRestore(context.Background(), "survival", "usr-1", "bk-1", "/backups/a.tar.gz"); err != nil {
t.Fatalf("BackupThenRestore: %v", err)
}
if len(jobs.got) != 1 {
t.Fatalf("created %d jobs, want 1", len(jobs.got))
}
p := jobs.got[0]
if p.RestoreRef != "/backups/a.tar.gz" || p.RestoreBackupID != "bk-1" || p.FormerOwner != "usr-1" {
t.Errorf("params = %+v", p)
}
if !strings.HasPrefix(p.JobName, BackupJobName("survival")+"-") {
t.Errorf("JobName %q", p.JobName)
}
}
+39 -5
View File
@@ -20,6 +20,16 @@ const (
managedByValue = "felis-backup"
componentValue = "world-backup"
// The restore chain a safety snapshot carries (internal/maintenance keeps the
// canonical copies; maintenance_test pins these against them).
labelThenRestore = "felis.lolicon.best/then-restore"
thenRestorePending = "pending"
annotationRestoreRef = "felis.lolicon.best/restore-ref"
annotationRestoreBackupID = "felis.lolicon.best/restore-backup-id"
// ReasonPreRestore is the world_backups reason of a safety snapshot.
ReasonPreRestore = "pre_restore"
worldVolume = "world"
backupVolume = "backup"
configVolume = "config"
@@ -55,6 +65,14 @@ type JobParams struct {
RunAsGroup int64
FSGroup int64
// RestoreRef / RestoreBackupID make this backup the safety snapshot in front
// of a restore: the Job is labelled as a pending chain and names the backup
// felis-api restores once the snapshot succeeds (internal/maintenance). The
// snapshot is recorded as ReasonPreRestore, and its prune spares the backup
// the restore will extract.
RestoreRef string
RestoreBackupID string
TTLAfterFinished time.Duration
}
@@ -106,6 +124,9 @@ func BackupJob(p JobParams) (*batchv1.Job, error) {
if p.ConfigSecret == "" {
return nil, fmt.Errorf("backup: config secret name is required")
}
if p.RestoreRef != "" && p.RestoreBackupID == "" {
return nil, fmt.Errorf("backup: a chained restore needs the backup id")
}
limits, err := resourceLimits(p.CPULimit, p.MemLimit)
if err != nil {
return nil, err
@@ -129,6 +150,9 @@ func BackupJob(p JobParams) (*batchv1.Job, error) {
if p.FormerOwner != "" {
args = append(args, "--former-owner", p.FormerOwner)
}
if p.RestoreRef != "" {
args = append(args, "--reason", ReasonPreRestore, "--protect", p.RestoreBackupID)
}
container := corev1.Container{
Name: "backup",
@@ -167,12 +191,22 @@ func BackupJob(p JobParams) (*batchv1.Job, error) {
if name == "" {
name = BackupJobName(p.Server)
}
meta := metav1.ObjectMeta{
Name: name,
Namespace: p.Namespace,
Labels: backupLabels(p),
}
if p.RestoreRef != "" {
// On the Job only: felis-api settles the chain by patching this label, and
// the pods never need it.
meta.Labels[labelThenRestore] = thenRestorePending
meta.Annotations = map[string]string{
annotationRestoreRef: p.RestoreRef,
annotationRestoreBackupID: p.RestoreBackupID,
}
}
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: p.Namespace,
Labels: backupLabels(p),
},
ObjectMeta: meta,
Spec: batchv1.JobSpec{
// One shot: a wedged archive must not loop. The TTL GCs the finished Job
// so a later backup of the same server is not blocked forever by a stale
+39
View File
@@ -199,6 +199,44 @@ func TestBackupJobArgsCarryServerAndOwner(t *testing.T) {
}
}
// A safety snapshot records itself as pre_restore, spares the backup the chained
// restore extracts from its prune, and carries the chain on the Job (only there:
// felis-api settles it by patching the Job's label).
func TestBackupJobCarriesTheRestoreChain(t *testing.T) {
p := sampleJobParams()
p.RestoreRef, p.RestoreBackupID = "/backups/survival/a.tar.gz", "bk-1"
job, err := BackupJob(p)
if err != nil {
t.Fatalf("BackupJob: %v", err)
}
args := job.Spec.Template.Spec.Containers[0].Args
if !argsContain(args, "--reason", ReasonPreRestore) || !argsContain(args, "--protect", "bk-1") {
t.Errorf("args = %v, want --reason %s --protect bk-1", args, ReasonPreRestore)
}
if job.Labels[labelThenRestore] != thenRestorePending {
t.Errorf("job labels = %v, want %s=%s", job.Labels, labelThenRestore, thenRestorePending)
}
if _, ok := job.Spec.Template.Labels[labelThenRestore]; ok {
t.Errorf("pod template carries the chain label: %v", job.Spec.Template.Labels)
}
if job.Annotations[annotationRestoreRef] != p.RestoreRef || job.Annotations[annotationRestoreBackupID] != "bk-1" {
t.Errorf("job annotations = %v", job.Annotations)
}
plain, err := BackupJob(sampleJobParams())
if err != nil {
t.Fatalf("BackupJob(plain): %v", err)
}
if _, ok := plain.Labels[labelThenRestore]; ok || len(plain.Annotations) != 0 {
t.Errorf("a plain backup carries a chain: labels %v annotations %v", plain.Labels, plain.Annotations)
}
for _, a := range plain.Spec.Template.Spec.Containers[0].Args {
if a == "--reason" || a == "--protect" {
t.Errorf("a plain backup passes %s: %v", a, plain.Spec.Template.Spec.Containers[0].Args)
}
}
}
func TestBackupJobRejectsMissingInputs(t *testing.T) {
for _, tc := range []struct {
name string
@@ -208,6 +246,7 @@ func TestBackupJobRejectsMissingInputs(t *testing.T) {
{"no world pvc", func(p *JobParams) { p.WorldPVC = "" }},
{"no backup pvc", func(p *JobParams) { p.BackupPVC = "" }},
{"no config secret", func(p *JobParams) { p.ConfigSecret = "" }},
{"chain without backup id", func(p *JobParams) { p.RestoreRef = "/backups/a.tar.gz" }},
} {
t.Run(tc.name, func(t *testing.T) {
p := sampleJobParams()