Unverified Commit 89a01347 authored by Lemon-miaow's avatar Lemon-miaow
Browse files

feat(backup): 归档先写 .partial 再 fsync 改名并记录 sha256，删除原件前回读校验，reaper... 

feat(backup): 归档先写 .partial 再 fsync 改名并记录 sha256，删除原件前回读校验，reaper 抽样巡检与清扫半截归档，保留权限与 mtime，面板标出损坏与已校验
parent 489eff44

cmd/felis/backup.go

+10 −1
Changes for cmd/felis/backup.go: 10 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ import ( 
	"flag"

	"fmt"

	"io"

	"strings"

	"time"



	"felis.lolicon.best/internal/backup"
@@ -91,11 +92,16 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int { 
		return 1

	}



	ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))

	a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))

	if err != nil {

		fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)

		return 1

	}

	ref, size := a.Ref, a.Size

	if len(a.Skipped) > 0 {

		fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",

			len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))

	}



	drv, err := store.Open(ctx, cfg.Database.URL)

	if err != nil {
@@ -112,6 +118,9 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int { 
		SizeBytes:   size,

		Reason:      *reason,

		ExpiresAt:   time.Now().Add(rcfg.ManualRetention),



		SHA256:         a.SHA256,

		SkippedEntries: len(a.Skipped),

	}

	st := reaper.NewPGStore(drv.DB())

	if err := st.InsertBackup(ctx, rec); err != nil {

cmd/felis/reaper.go

+12 −2
Changes for cmd/felis/reaper.go: 12 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -138,14 +138,24 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int { 
// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived

// is kept, and without this nobody would learn that it is never reaped.

func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {

	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d\n",

	fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",

		sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull,

		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed)

		sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,

		sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)

	if !sum.Failed() {

		return 0

	}

	if sum.Skipped > 0 || sum.ExpireFailed > 0 {

		fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",

			sum.Skipped, sum.StoreFull, sum.ExpireFailed)

	}

	if sum.Corrupt > 0 {

		fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)

	}

	if sum.VerifyFailed > 0 || sum.SweepFailed {

		fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",

			sum.VerifyFailed, !sum.SweepFailed)

	}

	return 1

}

cmd/felis/reaper_test.go

+4 −0
Changes for cmd/felis/reaper_test.go: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -30,6 +30,10 @@ func TestReportReaperRunFailsTheJob(t *testing.T) { 
		{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},

		{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},

		{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},

		{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},

		{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},

		{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},

		{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},

	} {

		var out, errb bytes.Buffer

		if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {

cmd/felis/restore_test.go

+2 −2
Changes for cmd/felis/restore_test.go: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup 
		BackupRoot: backupRoot,

		Resolve:    func(string) (string, error) { return srcDir, nil },

	}

	got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")

	got, err := ar.Archive(context.Background(), "survival", "world-survival-0")

	if err != nil {

		t.Fatalf("Archive: %v", err)

	}

	return string(got), backupRoot

	return string(got.Ref), backupRoot

}



// The restore subcommand must extract the archived world into the target world

docs/openapi.yaml

+11 −1
Changes for docs/openapi.yaml: 11 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -368,6 +368,16 @@ components: 
        status: { type: string }

        created_at: { type: string, format: date-time }

        expires_at: { type: string, format: date-time }

        corrupt:

          type: boolean

          description: The archive failed a read-back (a checksum, gzip or tar error) and cannot be restored. Omitted when false.

        verified_at:

          type: string

          format: date-time

          description: The archive's last read-back that matched. Omitted until the first.

        skipped_entries:

          type: integer

          description: World entries the archive could not hold (symbolic links, devices, sockets). Omitted when zero.



    Build:

      type: object
@@ -3016,7 +3026,7 @@ paths: 
            application/json:

              schema: { $ref: '#/components/schemas/Error' }

        '409':

          description: Server is not stopped (not_stopped), a restore, backup or file write already holds its world volume (maintenance_in_progress), or a restore of another backup is still running (restore_in_progress).

          description: Server is not stopped (not_stopped), a restore, backup or file write already holds its world volume (maintenance_in_progress), a restore of another backup is still running (restore_in_progress), or the chosen backup failed a read-back (backup_corrupt).

          content:

            application/json:

              schema: { $ref: '#/components/schemas/Error' }