Loading cmd/felis/offsite.go +7 −1 Changes for cmd/felis/offsite.go: 7 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -257,8 +257,14 @@ type offsiteSources struct { uploadsDir, uploadsPVC string } // offsiteRunLimit backstops one sync pass. Each upload has its own deadline, // scaled to its size (internal/offsite), so a pass over a big archive may run // for hours; the timer starts no second pass while one runs, and the unit's // TimeoutStartSec sits above this. const offsiteRunLimit = 23 * time.Hour func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log io.Writer) (offsite.Result, error) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute) ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit) defer cancel() checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second) err := env.bucket.Check(checkCtx) Loading deploy/bootstrap.sh +1 −1 Changes for deploy/bootstrap.sh: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -4512,7 +4512,7 @@ Wants=network-online.target Type=oneshot EnvironmentFile=${OFFSITE_ENV} ExecStart=${HOST_BIN} offsite sync -config ${STATE_DIR}/felis.host.toml -env-file ${OFFSITE_ENV} -db-dir ${FELIS_DB_BACKUP_DIR} -backup-pvc "${FELIS_BACKUP_PVC}" TimeoutStartSec=55min TimeoutStartSec=24h Nice=10 IOSchedulingClass=idle PrivateTmp=yes Loading deploy/bootstrap_test.sh +1 −1 Changes for deploy/bootstrap_test.sh: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -2031,7 +2031,7 @@ timer="$(cat "$odir/felis-offsite.timer")" expect "the unit loads the secrets" "EnvironmentFile=$odir/offsite.env" "$unit" expect "the unit syncs from the host config" \ "ExecStart=fakefelis offsite sync -config $odir/felis.host.toml -env-file $odir/offsite.env -db-dir /var/lib/felis/db-backups -backup-pvc \"felis-backups\"" "$unit" expect "a run ends before the next hour's" "TimeoutStartSec=55min" "$unit" expect "a pass over a big archive is not cut off at the hour" "TimeoutStartSec=24h" "$unit" expect "the copy runs hourly" "OnCalendar=hourly" "$timer" expect "a missed run catches up at boot" "Persistent=true" "$timer" expect "the timer is enabled" "SYSTEMCTL: enable --now felis-offsite.timer" "$out" Loading docs/troubleshooting.md +7 −0 Changes for docs/troubleshooting.md: 7 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -2250,6 +2250,13 @@ What runs: `registry/index/<stamp>.json.fenc`, `uploads/blobs/<sha256>.fenc` and `uploads/index/<stamp>.json.fenc`: AES-256-GCM in 64 KiB segments, so truncation, reordering and a wrong key are all refused on the way back. - A pass sends the database bundles first, then world archives, images and uploads. Each object has its own time limit: 10 minutes plus its size at 512 KiB/s (about 6 hours for 10 GiB). An archive the uplink cannot send in that time fails alone, stays pending and is tried again next pass; the rest of the pass still goes. A pass over a big archive can run for hours; the timer starts no second one meanwhile. An upload cut off restarts from the beginning of that object. [GO-TESTED: `internal/offsite`] - The reaper deletes an idle world only after its archive is in the bucket (§10). - The watchdog mails the owners when no sync has completed for 12 hours Loading internal/offsite/images.go +3 −1 Changes for internal/offsite/images.go: 3 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -532,7 +532,9 @@ func (s *Syncer) putBytes(ctx context.Context, key string, plain []byte) error { if err := Encrypt(&sealed, bytes.NewReader(plain), s.Key); err != nil { return err } return s.Bucket.Put(ctx, key, &sealed, int64(sealed.Len())) putCtx, cancel := context.WithTimeout(ctx, s.uploadBudget(int64(len(plain)))) defer cancel() return s.Bucket.Put(putCtx, key, &sealed, int64(sealed.Len())) } func (s *Syncer) putImageIndex(ctx context.Context, stamp string, x *ImageIndex) error { Loading Loading
cmd/felis/offsite.go +7 −1 Changes for cmd/felis/offsite.go: 7 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -257,8 +257,14 @@ type offsiteSources struct { uploadsDir, uploadsPVC string } // offsiteRunLimit backstops one sync pass. Each upload has its own deadline, // scaled to its size (internal/offsite), so a pass over a big archive may run // for hours; the timer starts no second pass while one runs, and the unit's // TimeoutStartSec sits above this. const offsiteRunLimit = 23 * time.Hour func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log io.Writer) (offsite.Result, error) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute) ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit) defer cancel() checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second) err := env.bucket.Check(checkCtx) Loading
deploy/bootstrap.sh +1 −1 Changes for deploy/bootstrap.sh: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -4512,7 +4512,7 @@ Wants=network-online.target Type=oneshot EnvironmentFile=${OFFSITE_ENV} ExecStart=${HOST_BIN} offsite sync -config ${STATE_DIR}/felis.host.toml -env-file ${OFFSITE_ENV} -db-dir ${FELIS_DB_BACKUP_DIR} -backup-pvc "${FELIS_BACKUP_PVC}" TimeoutStartSec=55min TimeoutStartSec=24h Nice=10 IOSchedulingClass=idle PrivateTmp=yes Loading
deploy/bootstrap_test.sh +1 −1 Changes for deploy/bootstrap_test.sh: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -2031,7 +2031,7 @@ timer="$(cat "$odir/felis-offsite.timer")" expect "the unit loads the secrets" "EnvironmentFile=$odir/offsite.env" "$unit" expect "the unit syncs from the host config" \ "ExecStart=fakefelis offsite sync -config $odir/felis.host.toml -env-file $odir/offsite.env -db-dir /var/lib/felis/db-backups -backup-pvc \"felis-backups\"" "$unit" expect "a run ends before the next hour's" "TimeoutStartSec=55min" "$unit" expect "a pass over a big archive is not cut off at the hour" "TimeoutStartSec=24h" "$unit" expect "the copy runs hourly" "OnCalendar=hourly" "$timer" expect "a missed run catches up at boot" "Persistent=true" "$timer" expect "the timer is enabled" "SYSTEMCTL: enable --now felis-offsite.timer" "$out" Loading
docs/troubleshooting.md +7 −0 Changes for docs/troubleshooting.md: 7 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -2250,6 +2250,13 @@ What runs: `registry/index/<stamp>.json.fenc`, `uploads/blobs/<sha256>.fenc` and `uploads/index/<stamp>.json.fenc`: AES-256-GCM in 64 KiB segments, so truncation, reordering and a wrong key are all refused on the way back. - A pass sends the database bundles first, then world archives, images and uploads. Each object has its own time limit: 10 minutes plus its size at 512 KiB/s (about 6 hours for 10 GiB). An archive the uplink cannot send in that time fails alone, stays pending and is tried again next pass; the rest of the pass still goes. A pass over a big archive can run for hours; the timer starts no second one meanwhile. An upload cut off restarts from the beginning of that object. [GO-TESTED: `internal/offsite`] - The reaper deletes an idle world only after its archive is in the bucket (§10). - The watchdog mails the owners when no sync has completed for 12 hours Loading
internal/offsite/images.go +3 −1 Changes for internal/offsite/images.go: 3 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -532,7 +532,9 @@ func (s *Syncer) putBytes(ctx context.Context, key string, plain []byte) error { if err := Encrypt(&sealed, bytes.NewReader(plain), s.Key); err != nil { return err } return s.Bucket.Put(ctx, key, &sealed, int64(sealed.Len())) putCtx, cancel := context.WithTimeout(ctx, s.uploadBudget(int64(len(plain)))) defer cancel() return s.Bucket.Put(putCtx, key, &sealed, int64(sealed.Len())) } func (s *Syncer) putImageIndex(ctx context.Context, stamp string, x *ImageIndex) error { Loading