Unverified Commit 5c58104e authored by Lemon-miaow's avatar Lemon-miaow
Browse files

fix(offsite): DB 包先同步,每个对象按大小单独限时,大归档传不完不再拖住整轮同步

parent 3f041714
Loading
Loading
Loading
Loading
+7 −1
Changes for cmd/felis/offsite.go: 7 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -257,8 +257,14 @@ type offsiteSources struct {
	uploadsDir, uploadsPVC string
}

// offsiteRunLimit backstops one sync pass. Each upload has its own deadline,
// scaled to its size (internal/offsite), so a pass over a big archive may run
// for hours; the timer starts no second pass while one runs, and the unit's
// TimeoutStartSec sits above this.
const offsiteRunLimit = 23 * time.Hour

func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log io.Writer) (offsite.Result, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute)
	ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit)
	defer cancel()
	checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
	err := env.bucket.Check(checkCtx)
+1 −1
Changes for deploy/bootstrap.sh: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -4512,7 +4512,7 @@ Wants=network-online.target
Type=oneshot
EnvironmentFile=${OFFSITE_ENV}
ExecStart=${HOST_BIN} offsite sync -config ${STATE_DIR}/felis.host.toml -env-file ${OFFSITE_ENV} -db-dir ${FELIS_DB_BACKUP_DIR} -backup-pvc "${FELIS_BACKUP_PVC}"
TimeoutStartSec=55min
TimeoutStartSec=24h
Nice=10
IOSchedulingClass=idle
PrivateTmp=yes
+1 −1
Changes for deploy/bootstrap_test.sh: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -2031,7 +2031,7 @@ timer="$(cat "$odir/felis-offsite.timer")"
expect "the unit loads the secrets" "EnvironmentFile=$odir/offsite.env" "$unit"
expect "the unit syncs from the host config" \
  "ExecStart=fakefelis offsite sync -config $odir/felis.host.toml -env-file $odir/offsite.env -db-dir /var/lib/felis/db-backups -backup-pvc \"felis-backups\"" "$unit"
expect "a run ends before the next hour's" "TimeoutStartSec=55min" "$unit"
expect "a pass over a big archive is not cut off at the hour" "TimeoutStartSec=24h" "$unit"
expect "the copy runs hourly" "OnCalendar=hourly" "$timer"
expect "a missed run catches up at boot" "Persistent=true" "$timer"
expect "the timer is enabled" "SYSTEMCTL: enable --now felis-offsite.timer" "$out"
+7 −0
Changes for docs/troubleshooting.md: 7 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -2250,6 +2250,13 @@ What runs:
  `registry/index/<stamp>.json.fenc`, `uploads/blobs/<sha256>.fenc` and
  `uploads/index/<stamp>.json.fenc`: AES-256-GCM in 64 KiB segments, so
  truncation, reordering and a wrong key are all refused on the way back.
- A pass sends the database bundles first, then world archives, images and
  uploads. Each object has its own time limit: 10 minutes plus its size at
  512 KiB/s (about 6 hours for 10 GiB). An archive the uplink cannot send in
  that time fails alone, stays pending and is tried again next pass; the rest
  of the pass still goes. A pass over a big archive can run for hours; the
  timer starts no second one meanwhile. An upload cut off restarts from the
  beginning of that object. [GO-TESTED: `internal/offsite`]
- The reaper deletes an idle world only after its archive is in the bucket
  (§10).
- The watchdog mails the owners when no sync has completed for 12 hours
+3 −1
Changes for internal/offsite/images.go: 3 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -532,7 +532,9 @@ func (s *Syncer) putBytes(ctx context.Context, key string, plain []byte) error {
	if err := Encrypt(&sealed, bytes.NewReader(plain), s.Key); err != nil {
		return err
	}
	return s.Bucket.Put(ctx, key, &sealed, int64(sealed.Len()))
	putCtx, cancel := context.WithTimeout(ctx, s.uploadBudget(int64(len(plain))))
	defer cancel()
	return s.Bucket.Put(putCtx, key, &sealed, int64(sealed.Len()))
}

func (s *Syncer) putImageIndex(ctx context.Context, stamp string, x *ImageIndex) error {
Loading