Unverified Commit c9e5e2fe authored by Lemon-miaow's avatar Lemon-miaow
Browse files

fix(offsite): 桶内记录密钥指纹,密钥不符时 sync 拒绝写入和清理,安装时大声提示

parent fa467338
Loading
Loading
Loading
Loading
+81 −8
Changes for cmd/felis/offsite.go: 81 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -33,12 +33,17 @@ const offsiteUsage = `usage:
  felis offsite fetch-worlds [-config path] [-archive-dir dir]
  felis offsite fetch-images [-config path] [-registry host:port] [-at version]
  felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
  felis offsite check-key    [-config path]
  felis offsite keygen

Every verb but keygen reads the bucket credentials and the encryption key from
the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
-env-file (default /etc/felis/offsite.env).

check-key tells whether the key is the one the bucket's objects are sealed
with, writing nothing; it exits 3 when they are sealed with another key, and
sync then refuses to write or prune anything in the bucket.
`

// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
@@ -74,6 +79,8 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int {
		return offsiteFetchImages(fs, rest, stdout, stderr)
	case "fetch-uploads":
		return offsiteFetchUploads(fs, rest, stdout, stderr)
	case "check-key":
		return offsiteCheckKey(fs, rest, stdout, stderr)
	case "keygen":
		k, err := offsite.NewKey()
		if err != nil {
@@ -218,12 +225,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
		registry:   offsiteRegistryEndpoint(*registry, cfg.Registry),
		uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
	}, stderr)
	st.Result = res
	if err != nil {
		st.LastError = err.Error()
	} else {
		st.LastSuccess = st.LastAttempt
	}
	recordRun(&st, res, err)
	if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
	}
@@ -246,6 +248,17 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
	return 0
}

// recordRun puts one pass's outcome into its status record.
func recordRun(st *offsite.Status, res offsite.Result, err error) {
	st.Result = res
	if err != nil {
		st.LastError = err.Error()
		st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
	} else {
		st.LastSuccess = st.LastAttempt
	}
}

// offsiteSources is where one sync pass reads from: the world archive volume
// (archiveDir, or the backupPVC's directory), the bundle directory, the
// registry's loopback endpoint and the uploads volume (uploadsDir, or the
@@ -438,6 +451,10 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
	if st.LastError != "" {
		fmt.Fprintf(stdout, "last error:   %s\n", st.LastError)
	}
	if st.KeyMismatch {
		fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
		return 1
	}
	r := st.Result
	fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
@@ -552,6 +569,62 @@ func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles [
	}
}

func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
	if err := fs.Parse(args); err != nil {
		return 2
	}
	_, env, err := loadOffsite(*cfgPath, *envFile)
	if err != nil {
		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
		return 1
	}
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()
	if err := env.bucket.Check(ctx); err != nil {
		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
		return 1
	}
	return checkKey(ctx, env.bucket, env.key, stdout, stderr)
}

// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when
// the bucket's objects are sealed with another one, 1 when it cannot tell.
func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int {
	fit, err := offsite.CheckKey(ctx, b, key)
	if err != nil {
		fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
		if errors.Is(err, offsite.ErrKeyMismatch) {
			return 3
		}
		return 1
	}
	id := offsite.KeyID(key)
	switch fit {
	case offsite.KeyRecorded:
		fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id)
	case offsite.KeyOpens:
		fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id)
	case offsite.KeyUnused:
		fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id)
	}
	return 0
}

// keyHint explains an object the key cannot open when the bucket records
// another key's id, "" otherwise.
func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {
	if !errors.Is(err, offsite.ErrAuth) {
		return ""
	}
	id, ierr := offsite.BucketKeyID(ctx, b)
	if ierr != nil || id == "" || id == offsite.KeyID(key) {
		return ""
	}
	return fmt.Sprintf("\n  the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key))
}

func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
@@ -603,7 +676,7 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string,
	if name == "latest" {
		var err error
		if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
			fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
			fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
			return 1
		}
	}
@@ -617,7 +690,7 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string,
	}
	dst := filepath.Join(dir, name)
	if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
		fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
		return 1
	}
	m, err := dbbackup.Verify(dst)
+115 −0
Changes for cmd/felis/offsite_test.go: 115 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ import (
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
@@ -285,6 +286,120 @@ func TestOffsiteFetchDB(t *testing.T) {
	})
}

func TestOffsiteCheckKey(t *testing.T) {
	newKey := func() []byte {
		raw, _ := offsite.NewKey()
		k, _ := offsite.ParseKey(raw)
		return k
	}
	key, other := newKey(), newKey()
	marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} }
	unmarked := func(k []byte) mapBucket {
		b := mapBucket{}
		putBundle(t, b, k, 0, nil)
		return b
	}
	for _, tc := range []struct {
		what   string
		bucket mapBucket
		code   int
		says   []string
	}{
		{"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}},
		{"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}},
		{"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}},
		{"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}},
		{"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}},
		{"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}},
	} {
		t.Run(tc.what, func(t *testing.T) {
			before := len(tc.bucket)
			var out, errb bytes.Buffer
			code := checkKey(context.Background(), tc.bucket, key, &out, &errb)
			if code != tc.code {
				t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String())
			}
			said := out.String() + errb.String()
			for _, s := range tc.says {
				if !strings.Contains(said, s) {
					t.Errorf("output lacks %q: %s", s, said)
				}
			}
			if len(tc.bucket) != before {
				t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before)
			}
		})
	}

	// fetch-db names both ids when the bucket records another key.
	b := marked(other)
	name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3})
	for _, arg := range []string{"latest", name} {
		var out, errb bytes.Buffer
		if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 ||
			!strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) {
			t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String())
		}
	}
	// A bundle the bucket lacks is not the key's fault.
	var missOut, missErr bytes.Buffer
	if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") {
		t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String())
	}
	// A bundle damaged under the recorded key gets no such hint.
	b = marked(key)
	name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
	b[offsite.DBKey(name)][60] ^= 1
	var out, errb bytes.Buffer
	if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") {
		t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String())
	}
}

func TestRecordRunMarksAKeyMismatch(t *testing.T) {
	t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
	for _, tc := range []struct {
		err      error
		mismatch bool
		success  bool
	}{
		{nil, false, true},
		{errors.New("list worlds/ in the bucket: connection reset"), false, false},
		{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false},
	} {
		st := offsite.Status{LastAttempt: t0}
		recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err)
		if st.KeyMismatch != tc.mismatch || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 {
			t.Errorf("err %v: status %+v", tc.err, st)
		}
	}
}

func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
	dir := t.TempDir()
	cfg := filepath.Join(dir, "felis.toml")
	writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
	statusFile := filepath.Join(dir, "status.json")
	st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"}
	status := func() (int, string) {
		t.Helper()
		if err := offsite.WriteStatus(statusFile, st); err != nil {
			t.Fatal(err)
		}
		var out, errb bytes.Buffer
		code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
		return code, out.String() + errb.String()
	}
	if code, out := status(); code != 0 || strings.Contains(out, "refused") {
		t.Fatalf("a recent success: exit %d\n%s", code, out)
	}
	st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true
	code, out := status()
	if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") {
		t.Fatalf("a refused run: exit %d\n%s", code, out)
	}
}

func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
	rawKey, _ := offsite.NewKey()
	key, _ := offsite.ParseKey(rawKey)
+38 −7
Changes for deploy/bootstrap.sh: 38 added lines, 7 removed lines.
Original line number Diff line number Diff line
@@ -4541,8 +4541,9 @@ quiet_watchdog() {
}

# The hourly off-site copy. The bucket is checked now, in the install, so wrong
# credentials or an unreachable endpoint show up here; the first copy itself runs in the
# background, since a host with many archives can take a long while to upload them.
# credentials, an unreachable endpoint or a key other than the one its objects are sealed
# with show up here; the first copy itself runs in the background, since a host with many
# archives can take a long while to upload them.
# With no bucket configured the timer is removed (the operator deleted [offsite]) and the
# install says loudly that every backup is on this machine only.
install_offsite_timer() {
@@ -4585,12 +4586,33 @@ WantedBy=timers.target
EOF
  systemctl daemon-reload
  systemctl enable --now felis-offsite.timer
  if "$HOST_BIN" offsite list -config "${STATE_DIR}/felis.host.toml" -env-file "$OFFSITE_ENV" >/dev/null; then
  local rc=0
  "$HOST_BIN" offsite check-key -config "${STATE_DIR}/felis.host.toml" -env-file "$OFFSITE_ENV" >/dev/null || rc=$?
  case "$rc" in
    0)
      systemctl start --no-block felis-offsite.service
      ok "off-site copy: hourly to the [offsite] bucket, first copy started (sudo felis offsite status; journalctl -u felis-offsite)"
  else
    warn "the [offsite] bucket did not answer (error above); nothing is copied off this machine until it does: fix ${OFFSITE_ENV} or [offsite] in ${STATE_DIR}/felis.host.toml, then sudo systemctl start felis-offsite.service"
      ;;
    3)
      # The timer stays: every run is refused (and reported by the watchdog) until the
      # key is fixed, and the first run after that needs no re-run of the installer.
      OFFSITE_KEY_MISMATCH=1
      warn "================================================================================"
      warn "The [offsite] bucket's objects are sealed with another key than the one in"
      warn "${OFFSITE_ENV} (felis offsite check-key, above). Nothing is copied off this"
      warn "machine, and nothing in the bucket is written or pruned, until the keys match."
      if [ "${OFFSITE_KEY_NEW:-0}" = 1 ]; then
        warn "This run generated that key: neither FELIS_OFFSITE_KEY nor ${OFFSITE_ENV} had one."
      fi
      warn "Set FELIS_OFFSITE_KEY in ${OFFSITE_ENV} to the key the bucket was written with and run"
      warn "sudo systemctl start felis-offsite.service, or give [offsite] an empty bucket or prefix"
      warn "and re-run the installer (docs/troubleshooting.md §16)."
      warn "================================================================================"
      ;;
    *)
      warn "the [offsite] bucket did not answer (error above); nothing is copied off this machine until it does: fix ${OFFSITE_ENV} or [offsite] in ${STATE_DIR}/felis.host.toml, then sudo systemctl start felis-offsite.service"
      ;;
  esac
}

# summary_offsite is the installer's last word on where the backups live.
@@ -4601,6 +4623,13 @@ summary_offsite() {
    warn "FELIS_OFFSITE_ACCESS_KEY and FELIS_OFFSITE_SECRET_KEY and re-run (docs/troubleshooting.md §16)."
    return 0
  fi
  if [ "${OFFSITE_KEY_MISMATCH:-0}" = 1 ]; then
    # A key the bucket refuses is no key to store; this run's is in OFFSITE_ENV if the
    # operator moves to an empty bucket instead.
    warn "OFF-SITE COPY STOPPED: the bucket's objects are sealed with another key (see above)."
    warn "Every backup is on this machine only until ${OFFSITE_ENV} holds that key (sudo felis offsite check-key)."
    return 0
  fi
  if [ "${OFFSITE_KEY_NEW:-0}" = 1 ]; then
    echo
    warn "================================================================================"
@@ -4789,10 +4818,12 @@ EOF
# replace the file's (rotating the bucket credentials is a re-run); the encryption key is
# generated when neither has one. A different key than the file's is refused: every object
# already in the bucket is sealed with the old one, and swapping it would make them
# unreadable without a word.
# unreadable without a word. Whether the bucket's objects agree with the key is checked once
# the binary is installed (install_offsite_timer).
configure_offsite() {
  OFFSITE_ENABLED=0
  OFFSITE_KEY_NEW=0
  OFFSITE_KEY_MISMATCH=0
  offsite_enabled || return 0
  OFFSITE_ENABLED=1
  local env_ak="${FELIS_OFFSITE_ACCESS_KEY:-}" env_sk="${FELIS_OFFSITE_SECRET_KEY:-}" env_key="${FELIS_OFFSITE_KEY:-}"
@@ -4806,7 +4837,7 @@ configure_offsite() {
  FELIS_OFFSITE_ACCESS_KEY="${env_ak:-$FELIS_OFFSITE_ACCESS_KEY}"
  FELIS_OFFSITE_SECRET_KEY="${env_sk:-$FELIS_OFFSITE_SECRET_KEY}"
  if [ -n "$env_key" ] && [ -n "$file_key" ] && [ "$env_key" != "$file_key" ]; then
    die "FELIS_OFFSITE_KEY differs from the key in ${OFFSITE_ENV}; the objects already in the bucket are sealed with that one. Unset FELIS_OFFSITE_KEY to keep it (docs/troubleshooting.md §16)"
    die "FELIS_OFFSITE_KEY differs from the key in ${OFFSITE_ENV}, which sealed what this host copied to the bucket. Unset FELIS_OFFSITE_KEY to keep it; when felis offsite check-key says the bucket's objects are sealed with another key, set FELIS_OFFSITE_KEY in ${OFFSITE_ENV} by hand instead (docs/troubleshooting.md §16)"
  fi
  FELIS_OFFSITE_KEY="${env_key:-$file_key}"
  if [ -z "$FELIS_OFFSITE_ACCESS_KEY" ] || [ -z "$FELIS_OFFSITE_SECRET_KEY" ]; then
+25 −2
Changes for deploy/bootstrap_test.sh: 25 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -1997,7 +1997,8 @@ run_offsite() { # script; runs with the off-site functions sourced
    die() { printf "DIE: %s\n" "$*"; exit 1; }
    log() { printf "LOG: %s\n" "$*"; }; ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; }
    systemctl() { printf "SYSTEMCTL: %s\n" "$*" >&2; }
    fakefelis() { printf "RUN: %s\n" "$*" >&2; [ -z "${CHECK_FAILS:-}" ] || { echo "bucket: access denied" >&2; return 1; }; }
    fakefelis() { printf "RUN: %s\n" "$*" >&2; [ -z "${CHECK_FAILS:-}" ] || { echo "bucket: access denied" >&2; return 1; }
      [ -z "${KEY_MISMATCH:-}" ] || { echo "the bucket records key id 1111111111111111, this key is 2222222222222222" >&2; return 3; }; }
    FELIS_OFFSITE_ENDPOINT="${FELIS_OFFSITE_ENDPOINT:-}" FELIS_OFFSITE_BUCKET="${FELIS_OFFSITE_BUCKET:-}"
    FELIS_OFFSITE_REGION="${FELIS_OFFSITE_REGION:-}" FELIS_OFFSITE_PREFIX="${FELIS_OFFSITE_PREFIX:-}"
    FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}"
@@ -2108,7 +2109,7 @@ expect "a pass over a big archive is not cut off at the hour" "TimeoutStartSec=2
expect "the copy runs hourly" "OnCalendar=hourly" "$timer"
expect "a missed run catches up at boot" "Persistent=true" "$timer"
expect "the timer is enabled" "SYSTEMCTL: enable --now felis-offsite.timer" "$out"
expect "the bucket is checked during the install" "RUN: offsite list -config $odir/felis.host.toml -env-file $odir/offsite.env" "$out"
expect "the bucket and its key are checked during the install" "RUN: offsite check-key -config $odir/felis.host.toml -env-file $odir/offsite.env" "$out"
expect "the first copy runs in the background" "SYSTEMCTL: start --no-block felis-offsite.service" "$out"

out="$(OFFSITE_ENABLED=1 CHECK_FAILS=1 run_offsite install_offsite_timer)"
@@ -2119,6 +2120,28 @@ case "$out" in
  *) echo "PASS a failed check does not start the copy" ;;
esac

# A reinstall that lost offsite.env generates a fresh key; a bucket sealed with the old one
# must stop the install's copy and say what to do, not read as an unreachable bucket.
out="$(OFFSITE_ENABLED=1 OFFSITE_KEY_NEW=1 FELIS_OFFSITE_KEY=NEWKEY KEY_MISMATCH=1 run_offsite 'install_offsite_timer; summary_offsite')"
expect "a key mismatch shows both ids" "the bucket records key id 1111111111111111, this key is 2222222222222222" "$out"
expect "a key mismatch is a loud warning" "WARN: The [offsite] bucket's objects are sealed with another key than the one in" "$out"
expect "a key mismatch says the key was generated by this run" "WARN: This run generated that key" "$out"
expect "a key mismatch says how to fix it" "WARN: Set FELIS_OFFSITE_KEY in $odir/offsite.env to the key the bucket was written with" "$out"
expect "the summary says the copy stopped" "WARN: OFF-SITE COPY STOPPED" "$out"
expect "the timer stays for the run after the fix" "SYSTEMCTL: enable --now felis-offsite.timer" "$out"
case "$out" in
  *"start --no-block"* | *"did not answer"* | *"Store it NOW"* | *NEWKEY*)
    echo "FAIL a key mismatch started the copy, read as an unreachable bucket or showed the refused key: $out"; fails=$((fails + 1)) ;;
  *) echo "PASS a key mismatch neither starts the copy nor asks to store the refused key" ;;
esac
out="$(OFFSITE_ENABLED=1 KEY_MISMATCH=1 run_offsite install_offsite_timer)"
case "$out" in
  *"This run generated"*) echo "FAIL a kept key was called generated"; fails=$((fails + 1)) ;;
  *) echo "PASS a kept key that mismatches is not called generated" ;;
esac
out="$(OFFSITE_ENABLED=1 OFFSITE_KEY_NEW=1 FELIS_OFFSITE_KEY=NEWKEY run_offsite 'install_offsite_timer; summary_offsite')"
expect "a key the bucket takes is shown once" "WARN:     FELIS_OFFSITE_KEY=NEWKEY" "$out"

out="$(OFFSITE_ENABLED=0 run_offsite 'systemctl() { printf "SYSTEMCTL: %s\n" "$*" >> "$STATE_DIR/systemctl.log"; }; install_offsite_timer')"
expect "removing [offsite] disables the timer" "SYSTEMCTL: disable --now felis-offsite.timer" "$(cat "$odir/systemctl.log")"
expect "removing [offsite] says so" "WARN: no [offsite] bucket is configured any more" "$out"
+19 −1
Changes for docs/troubleshooting.md: 19 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -2148,7 +2148,8 @@ host yourself, plus the off-site encryption key if the copy is in the bucket.
   checks it (`felis db verify`) and names it, with when it was taken, the
   release and schema that took it, and how many accounts and servers its
   database holds. Read those before going on. A wrong key fails with
   `object does not decrypt with this key` and writes nothing. For a copy you
   `object does not decrypt with this key` (naming the bucket's key id and this
   key's, once the bucket records one) and writes nothing. For a copy you
   made yourself, check it with `sha256sum -c felis-db-....tar.sha256`.

   `latest` is the newest bundle, unless that one holds no servers and at
@@ -2266,6 +2267,21 @@ host that has a bucket, and removed `felis-offsite.timer` as they did; a re-run
of the current installer puts it back. `systemctl list-timers felis-offsite.timer`
shows whether the timer is there. [SH-TESTED] [VM-TESTED: a re-run that lost the race]

The bucket records which key sealed it: `felis-key-id`, the one object kept in
the clear, holds the key's id (the `key id:` of `felis offsite status`), which
names the key without revealing it. The first sync writes it; a bucket from
before it was recorded is judged by whether the key opens its newest objects.
The installer runs `felis offsite check-key`, and every sync checks before
anything else. A key other than the bucket's (a reinstall that lost
`offsite.env` and was given no `FELIS_OFFSITE_KEY` generates a new one) stops
the sync before it copies or prunes anything, the installer ends with
`OFF-SITE COPY STOPPED`, `status` exits 1 and the watchdog mails the owners
at once. Set `FELIS_OFFSITE_KEY` in `/etc/felis/offsite.env` to the bucket's
key and `sudo systemctl start felis-offsite.service`, or give `[offsite]` an
empty bucket or prefix and re-run the installer.
[GO-TESTED: `TestSyncRefusesAnotherKeysBucket`, `TestCheckKey`] [SH-TESTED]
[VM-TESTED: MinIO, the other key's sync refused with the bucket unchanged, check-key 0/3, fetch-db naming both ids]

What runs:

- **`felis-offsite.timer`** runs `felis offsite sync` hourly (plus up to
@@ -2306,6 +2322,7 @@ What runs:
  `registry/index/<stamp>.json.fenc`, `uploads/blobs/<sha256>.fenc` and
  `uploads/index/<stamp>.json.fenc`: AES-256-GCM in 64 KiB segments, so
  truncation, reordering and a wrong key are all refused on the way back.
  `felis-key-id` next to them holds the key's id in the clear.
- A pass sends the database bundles first, then world archives, images and
  uploads. Each object has its own time limit: 10 minutes plus its size at
  512 KiB/s (about 6 hours for 10 GiB). An archive the uplink cannot send in
@@ -2323,6 +2340,7 @@ Checking it:
```
sudo felis offsite status        # last run, errors, what the bucket holds, what waits
sudo felis offsite list          # the bundles, image lists and upload lists in the bucket, newest first
sudo felis offsite check-key     # whether offsite.env holds the key the bucket was sealed with
sudo journalctl -u felis-offsite -n 50 --no-pager
sudo systemctl start felis-offsite.service   # run one now
```
Loading