diff --git a/cmd/felis/offsite.go b/cmd/felis/offsite.go index 7f15b4b..5863ad2 100644 --- a/cmd/felis/offsite.go +++ b/cmd/felis/offsite.go @@ -33,12 +33,17 @@ const offsiteUsage = `usage: felis offsite fetch-worlds [-config path] [-archive-dir dir] felis offsite fetch-images [-config path] [-registry host:port] [-at version] felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version] + felis offsite check-key [-config path] felis offsite keygen Every verb but keygen reads the bucket credentials and the encryption key from the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY, FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from -env-file (default /etc/felis/offsite.env). + +check-key tells whether the key is the one the bucket's objects are sealed +with, writing nothing; it exits 3 when they are sealed with another key, and +sync then refuses to write or prune anything in the bucket. ` // defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the @@ -74,6 +79,8 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int { return offsiteFetchImages(fs, rest, stdout, stderr) case "fetch-uploads": return offsiteFetchUploads(fs, rest, stdout, stderr) + case "check-key": + return offsiteCheckKey(fs, rest, stdout, stderr) case "keygen": k, err := offsite.NewKey() if err != nil { @@ -218,12 +225,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int registry: offsiteRegistryEndpoint(*registry, cfg.Registry), uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC, }, stderr) - st.Result = res - if err != nil { - st.LastError = err.Error() - } else { - st.LastSuccess = st.LastAttempt - } + recordRun(&st, res, err) if werr := offsite.WriteStatus(*statusFile, st); werr != nil { fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr) } @@ -246,6 +248,17 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int return 0 } +// recordRun puts one pass's outcome into its status record. +func recordRun(st *offsite.Status, res offsite.Result, err error) { + st.Result = res + if err != nil { + st.LastError = err.Error() + st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch) + } else { + st.LastSuccess = st.LastAttempt + } +} + // offsiteSources is where one sync pass reads from: the world archive volume // (archiveDir, or the backupPVC's directory), the bundle directory, the // registry's loopback endpoint and the uploads volume (uploadsDir, or the @@ -438,6 +451,10 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in if st.LastError != "" { fmt.Fprintf(stdout, "last error: %s\n", st.LastError) } + if st.KeyMismatch { + fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile) + return 1 + } r := st.Result fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n", r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB)) @@ -552,6 +569,62 @@ func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles [ } } +func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int { + cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml") + envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set") + if err := fs.Parse(args); err != nil { + return 2 + } + _, env, err := loadOffsite(*cfgPath, *envFile) + if err != nil { + fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err) + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := env.bucket.Check(ctx); err != nil { + fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err) + return 1 + } + return checkKey(ctx, env.bucket, env.key, stdout, stderr) +} + +// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when +// the bucket's objects are sealed with another one, 1 when it cannot tell. +func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int { + fit, err := offsite.CheckKey(ctx, b, key) + if err != nil { + fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err) + if errors.Is(err, offsite.ErrKeyMismatch) { + return 3 + } + return 1 + } + id := offsite.KeyID(key) + switch fit { + case offsite.KeyRecorded: + fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id) + case offsite.KeyOpens: + fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id) + case offsite.KeyUnused: + fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id) + } + return 0 +} + +// keyHint explains an object the key cannot open when the bucket records +// another key's id, "" otherwise. +func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string { + if !errors.Is(err, offsite.ErrAuth) { + return "" + } + id, ierr := offsite.BucketKeyID(ctx, b) + if ierr != nil || id == "" || id == offsite.KeyID(key) { + return "" + } + return fmt.Sprintf("\n the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key)) +} + func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int { cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead") envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set") @@ -603,7 +676,7 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, if name == "latest" { var err error if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil { - fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err) + fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err)) return 1 } } @@ -617,7 +690,7 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string, } dst := filepath.Join(dir, name) if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil { - fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err) + fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err)) return 1 } m, err := dbbackup.Verify(dst) diff --git a/cmd/felis/offsite_test.go b/cmd/felis/offsite_test.go index 314d31f..001b523 100644 --- a/cmd/felis/offsite_test.go +++ b/cmd/felis/offsite_test.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" @@ -285,6 +286,120 @@ func TestOffsiteFetchDB(t *testing.T) { }) } +func TestOffsiteCheckKey(t *testing.T) { + newKey := func() []byte { + raw, _ := offsite.NewKey() + k, _ := offsite.ParseKey(raw) + return k + } + key, other := newKey(), newKey() + marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} } + unmarked := func(k []byte) mapBucket { + b := mapBucket{} + putBundle(t, b, k, 0, nil) + return b + } + for _, tc := range []struct { + what string + bucket mapBucket + code int + says []string + }{ + {"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}}, + {"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}}, + {"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}}, + {"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}}, + {"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}}, + {"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}}, + } { + t.Run(tc.what, func(t *testing.T) { + before := len(tc.bucket) + var out, errb bytes.Buffer + code := checkKey(context.Background(), tc.bucket, key, &out, &errb) + if code != tc.code { + t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String()) + } + said := out.String() + errb.String() + for _, s := range tc.says { + if !strings.Contains(said, s) { + t.Errorf("output lacks %q: %s", s, said) + } + } + if len(tc.bucket) != before { + t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before) + } + }) + } + + // fetch-db names both ids when the bucket records another key. + b := marked(other) + name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3}) + for _, arg := range []string{"latest", name} { + var out, errb bytes.Buffer + if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 || + !strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) { + t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String()) + } + } + // A bundle the bucket lacks is not the key's fault. + var missOut, missErr bytes.Buffer + if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") { + t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String()) + } + // A bundle damaged under the recorded key gets no such hint. + b = marked(key) + name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3}) + b[offsite.DBKey(name)][60] ^= 1 + var out, errb bytes.Buffer + if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") { + t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String()) + } +} + +func TestRecordRunMarksAKeyMismatch(t *testing.T) { + t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + err error + mismatch bool + success bool + }{ + {nil, false, true}, + {errors.New("list worlds/ in the bucket: connection reset"), false, false}, + {fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false}, + } { + st := offsite.Status{LastAttempt: t0} + recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err) + if st.KeyMismatch != tc.mismatch || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 { + t.Errorf("err %v: status %+v", tc.err, st) + } + } +} + +func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) { + dir := t.TempDir() + cfg := filepath.Join(dir, "felis.toml") + writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600) + statusFile := filepath.Join(dir, "status.json") + st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"} + status := func() (int, string) { + t.Helper() + if err := offsite.WriteStatus(statusFile, st); err != nil { + t.Fatal(err) + } + var out, errb bytes.Buffer + code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb) + return code, out.String() + errb.String() + } + if code, out := status(); code != 0 || strings.Contains(out, "refused") { + t.Fatalf("a recent success: exit %d\n%s", code, out) + } + st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true + code, out := status() + if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") { + t.Fatalf("a refused run: exit %d\n%s", code, out) + } +} + func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) { rawKey, _ := offsite.NewKey() key, _ := offsite.ParseKey(rawKey) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 0347fde..b47a183 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -4541,8 +4541,9 @@ quiet_watchdog() { } # The hourly off-site copy. The bucket is checked now, in the install, so wrong -# credentials or an unreachable endpoint show up here; the first copy itself runs in the -# background, since a host with many archives can take a long while to upload them. +# credentials, an unreachable endpoint or a key other than the one its objects are sealed +# with show up here; the first copy itself runs in the background, since a host with many +# archives can take a long while to upload them. # With no bucket configured the timer is removed (the operator deleted [offsite]) and the # install says loudly that every backup is on this machine only. install_offsite_timer() { @@ -4585,12 +4586,33 @@ WantedBy=timers.target EOF systemctl daemon-reload systemctl enable --now felis-offsite.timer - if "$HOST_BIN" offsite list -config "${STATE_DIR}/felis.host.toml" -env-file "$OFFSITE_ENV" >/dev/null; then - systemctl start --no-block felis-offsite.service - ok "off-site copy: hourly to the [offsite] bucket, first copy started (sudo felis offsite status; journalctl -u felis-offsite)" - else - warn "the [offsite] bucket did not answer (error above); nothing is copied off this machine until it does: fix ${OFFSITE_ENV} or [offsite] in ${STATE_DIR}/felis.host.toml, then sudo systemctl start felis-offsite.service" - fi + local rc=0 + "$HOST_BIN" offsite check-key -config "${STATE_DIR}/felis.host.toml" -env-file "$OFFSITE_ENV" >/dev/null || rc=$? + case "$rc" in + 0) + systemctl start --no-block felis-offsite.service + ok "off-site copy: hourly to the [offsite] bucket, first copy started (sudo felis offsite status; journalctl -u felis-offsite)" + ;; + 3) + # The timer stays: every run is refused (and reported by the watchdog) until the + # key is fixed, and the first run after that needs no re-run of the installer. + OFFSITE_KEY_MISMATCH=1 + warn "================================================================================" + warn "The [offsite] bucket's objects are sealed with another key than the one in" + warn "${OFFSITE_ENV} (felis offsite check-key, above). Nothing is copied off this" + warn "machine, and nothing in the bucket is written or pruned, until the keys match." + if [ "${OFFSITE_KEY_NEW:-0}" = 1 ]; then + warn "This run generated that key: neither FELIS_OFFSITE_KEY nor ${OFFSITE_ENV} had one." + fi + warn "Set FELIS_OFFSITE_KEY in ${OFFSITE_ENV} to the key the bucket was written with and run" + warn "sudo systemctl start felis-offsite.service, or give [offsite] an empty bucket or prefix" + warn "and re-run the installer (docs/troubleshooting.md §16)." + warn "================================================================================" + ;; + *) + warn "the [offsite] bucket did not answer (error above); nothing is copied off this machine until it does: fix ${OFFSITE_ENV} or [offsite] in ${STATE_DIR}/felis.host.toml, then sudo systemctl start felis-offsite.service" + ;; + esac } # summary_offsite is the installer's last word on where the backups live. @@ -4601,6 +4623,13 @@ summary_offsite() { warn "FELIS_OFFSITE_ACCESS_KEY and FELIS_OFFSITE_SECRET_KEY and re-run (docs/troubleshooting.md §16)." return 0 fi + if [ "${OFFSITE_KEY_MISMATCH:-0}" = 1 ]; then + # A key the bucket refuses is no key to store; this run's is in OFFSITE_ENV if the + # operator moves to an empty bucket instead. + warn "OFF-SITE COPY STOPPED: the bucket's objects are sealed with another key (see above)." + warn "Every backup is on this machine only until ${OFFSITE_ENV} holds that key (sudo felis offsite check-key)." + return 0 + fi if [ "${OFFSITE_KEY_NEW:-0}" = 1 ]; then echo warn "================================================================================" @@ -4789,10 +4818,12 @@ EOF # replace the file's (rotating the bucket credentials is a re-run); the encryption key is # generated when neither has one. A different key than the file's is refused: every object # already in the bucket is sealed with the old one, and swapping it would make them -# unreadable without a word. +# unreadable without a word. Whether the bucket's objects agree with the key is checked once +# the binary is installed (install_offsite_timer). configure_offsite() { OFFSITE_ENABLED=0 OFFSITE_KEY_NEW=0 + OFFSITE_KEY_MISMATCH=0 offsite_enabled || return 0 OFFSITE_ENABLED=1 local env_ak="${FELIS_OFFSITE_ACCESS_KEY:-}" env_sk="${FELIS_OFFSITE_SECRET_KEY:-}" env_key="${FELIS_OFFSITE_KEY:-}" @@ -4806,7 +4837,7 @@ configure_offsite() { FELIS_OFFSITE_ACCESS_KEY="${env_ak:-$FELIS_OFFSITE_ACCESS_KEY}" FELIS_OFFSITE_SECRET_KEY="${env_sk:-$FELIS_OFFSITE_SECRET_KEY}" if [ -n "$env_key" ] && [ -n "$file_key" ] && [ "$env_key" != "$file_key" ]; then - die "FELIS_OFFSITE_KEY differs from the key in ${OFFSITE_ENV}; the objects already in the bucket are sealed with that one. Unset FELIS_OFFSITE_KEY to keep it (docs/troubleshooting.md §16)" + die "FELIS_OFFSITE_KEY differs from the key in ${OFFSITE_ENV}, which sealed what this host copied to the bucket. Unset FELIS_OFFSITE_KEY to keep it; when felis offsite check-key says the bucket's objects are sealed with another key, set FELIS_OFFSITE_KEY in ${OFFSITE_ENV} by hand instead (docs/troubleshooting.md §16)" fi FELIS_OFFSITE_KEY="${env_key:-$file_key}" if [ -z "$FELIS_OFFSITE_ACCESS_KEY" ] || [ -z "$FELIS_OFFSITE_SECRET_KEY" ]; then diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index 97fe94e..b2802e8 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1997,7 +1997,8 @@ run_offsite() { # script; runs with the off-site functions sourced die() { printf "DIE: %s\n" "$*"; exit 1; } log() { printf "LOG: %s\n" "$*"; }; ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; } systemctl() { printf "SYSTEMCTL: %s\n" "$*" >&2; } - fakefelis() { printf "RUN: %s\n" "$*" >&2; [ -z "${CHECK_FAILS:-}" ] || { echo "bucket: access denied" >&2; return 1; }; } + fakefelis() { printf "RUN: %s\n" "$*" >&2; [ -z "${CHECK_FAILS:-}" ] || { echo "bucket: access denied" >&2; return 1; } + [ -z "${KEY_MISMATCH:-}" ] || { echo "the bucket records key id 1111111111111111, this key is 2222222222222222" >&2; return 3; }; } FELIS_OFFSITE_ENDPOINT="${FELIS_OFFSITE_ENDPOINT:-}" FELIS_OFFSITE_BUCKET="${FELIS_OFFSITE_BUCKET:-}" FELIS_OFFSITE_REGION="${FELIS_OFFSITE_REGION:-}" FELIS_OFFSITE_PREFIX="${FELIS_OFFSITE_PREFIX:-}" FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}" @@ -2108,7 +2109,7 @@ expect "a pass over a big archive is not cut off at the hour" "TimeoutStartSec=2 expect "the copy runs hourly" "OnCalendar=hourly" "$timer" expect "a missed run catches up at boot" "Persistent=true" "$timer" expect "the timer is enabled" "SYSTEMCTL: enable --now felis-offsite.timer" "$out" -expect "the bucket is checked during the install" "RUN: offsite list -config $odir/felis.host.toml -env-file $odir/offsite.env" "$out" +expect "the bucket and its key are checked during the install" "RUN: offsite check-key -config $odir/felis.host.toml -env-file $odir/offsite.env" "$out" expect "the first copy runs in the background" "SYSTEMCTL: start --no-block felis-offsite.service" "$out" out="$(OFFSITE_ENABLED=1 CHECK_FAILS=1 run_offsite install_offsite_timer)" @@ -2119,6 +2120,28 @@ case "$out" in *) echo "PASS a failed check does not start the copy" ;; esac +# A reinstall that lost offsite.env generates a fresh key; a bucket sealed with the old one +# must stop the install's copy and say what to do, not read as an unreachable bucket. +out="$(OFFSITE_ENABLED=1 OFFSITE_KEY_NEW=1 FELIS_OFFSITE_KEY=NEWKEY KEY_MISMATCH=1 run_offsite 'install_offsite_timer; summary_offsite')" +expect "a key mismatch shows both ids" "the bucket records key id 1111111111111111, this key is 2222222222222222" "$out" +expect "a key mismatch is a loud warning" "WARN: The [offsite] bucket's objects are sealed with another key than the one in" "$out" +expect "a key mismatch says the key was generated by this run" "WARN: This run generated that key" "$out" +expect "a key mismatch says how to fix it" "WARN: Set FELIS_OFFSITE_KEY in $odir/offsite.env to the key the bucket was written with" "$out" +expect "the summary says the copy stopped" "WARN: OFF-SITE COPY STOPPED" "$out" +expect "the timer stays for the run after the fix" "SYSTEMCTL: enable --now felis-offsite.timer" "$out" +case "$out" in + *"start --no-block"* | *"did not answer"* | *"Store it NOW"* | *NEWKEY*) + echo "FAIL a key mismatch started the copy, read as an unreachable bucket or showed the refused key: $out"; fails=$((fails + 1)) ;; + *) echo "PASS a key mismatch neither starts the copy nor asks to store the refused key" ;; +esac +out="$(OFFSITE_ENABLED=1 KEY_MISMATCH=1 run_offsite install_offsite_timer)" +case "$out" in + *"This run generated"*) echo "FAIL a kept key was called generated"; fails=$((fails + 1)) ;; + *) echo "PASS a kept key that mismatches is not called generated" ;; +esac +out="$(OFFSITE_ENABLED=1 OFFSITE_KEY_NEW=1 FELIS_OFFSITE_KEY=NEWKEY run_offsite 'install_offsite_timer; summary_offsite')" +expect "a key the bucket takes is shown once" "WARN: FELIS_OFFSITE_KEY=NEWKEY" "$out" + out="$(OFFSITE_ENABLED=0 run_offsite 'systemctl() { printf "SYSTEMCTL: %s\n" "$*" >> "$STATE_DIR/systemctl.log"; }; install_offsite_timer')" expect "removing [offsite] disables the timer" "SYSTEMCTL: disable --now felis-offsite.timer" "$(cat "$odir/systemctl.log")" expect "removing [offsite] says so" "WARN: no [offsite] bucket is configured any more" "$out" diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index fc2174f..cf9e34b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2148,7 +2148,8 @@ host yourself, plus the off-site encryption key if the copy is in the bucket. checks it (`felis db verify`) and names it, with when it was taken, the release and schema that took it, and how many accounts and servers its database holds. Read those before going on. A wrong key fails with - `object does not decrypt with this key` and writes nothing. For a copy you + `object does not decrypt with this key` (naming the bucket's key id and this + key's, once the bucket records one) and writes nothing. For a copy you made yourself, check it with `sha256sum -c felis-db-....tar.sha256`. `latest` is the newest bundle, unless that one holds no servers and at @@ -2266,6 +2267,21 @@ host that has a bucket, and removed `felis-offsite.timer` as they did; a re-run of the current installer puts it back. `systemctl list-timers felis-offsite.timer` shows whether the timer is there. [SH-TESTED] [VM-TESTED: a re-run that lost the race] +The bucket records which key sealed it: `felis-key-id`, the one object kept in +the clear, holds the key's id (the `key id:` of `felis offsite status`), which +names the key without revealing it. The first sync writes it; a bucket from +before it was recorded is judged by whether the key opens its newest objects. +The installer runs `felis offsite check-key`, and every sync checks before +anything else. A key other than the bucket's (a reinstall that lost +`offsite.env` and was given no `FELIS_OFFSITE_KEY` generates a new one) stops +the sync before it copies or prunes anything, the installer ends with +`OFF-SITE COPY STOPPED`, `status` exits 1 and the watchdog mails the owners +at once. Set `FELIS_OFFSITE_KEY` in `/etc/felis/offsite.env` to the bucket's +key and `sudo systemctl start felis-offsite.service`, or give `[offsite]` an +empty bucket or prefix and re-run the installer. +[GO-TESTED: `TestSyncRefusesAnotherKeysBucket`, `TestCheckKey`] [SH-TESTED] +[VM-TESTED: MinIO, the other key's sync refused with the bucket unchanged, check-key 0/3, fetch-db naming both ids] + What runs: - **`felis-offsite.timer`** runs `felis offsite sync` hourly (plus up to @@ -2306,6 +2322,7 @@ What runs: `registry/index/.json.fenc`, `uploads/blobs/.fenc` and `uploads/index/.json.fenc`: AES-256-GCM in 64 KiB segments, so truncation, reordering and a wrong key are all refused on the way back. + `felis-key-id` next to them holds the key's id in the clear. - A pass sends the database bundles first, then world archives, images and uploads. Each object has its own time limit: 10 minutes plus its size at 512 KiB/s (about 6 hours for 10 GiB). An archive the uplink cannot send in @@ -2323,6 +2340,7 @@ Checking it: ``` sudo felis offsite status # last run, errors, what the bucket holds, what waits sudo felis offsite list # the bundles, image lists and upload lists in the bucket, newest first +sudo felis offsite check-key # whether offsite.env holds the key the bucket was sealed with sudo journalctl -u felis-offsite -n 50 --no-pager sudo systemctl start felis-offsite.service # run one now ``` diff --git a/internal/offsite/keymark.go b/internal/offsite/keymark.go new file mode 100644 index 0000000..6c4fa16 --- /dev/null +++ b/internal/offsite/keymark.go @@ -0,0 +1,152 @@ +package offsite + +import ( + "context" + "errors" + "fmt" + "io" + "regexp" + "sort" + "strings" +) + +// keyMark is the one object the bucket holds in the clear: the KeyID of the +// key every other object is sealed with. The id names the key without +// revealing it, so a host holding another key (a reinstall that generated a +// fresh one, an offsite.env from elsewhere) is refused before it writes an +// object or prunes one the right key still opens. +const keyMark = "felis-key-id" + +// ErrKeyMismatch is a bucket whose objects are sealed with another key. +var ErrKeyMismatch = errors.New("offsite: the bucket's objects are sealed with another key") + +const keyMismatchFix = "set FELIS_OFFSITE_KEY in /etc/felis/offsite.env to the key they were sealed with, or point [offsite] at an empty bucket or prefix (docs/troubleshooting.md §16)" + +var keyIDPattern = regexp.MustCompile(`^[0-9a-f]{16}$`) + +// KeyFit is how CheckKey matched a key to a bucket. The zero value is no match: +// what CheckKey returns with an error. +type KeyFit int + +const ( + // KeyRecorded: the bucket records this key's id. + KeyRecorded KeyFit = iota + 1 + // KeyOpens: the bucket records no id, and the key opens its newest objects. + KeyOpens + // KeyUnused: the bucket holds no sealed object yet. + KeyUnused +) + +// A bucket without a recorded id is judged by its newest sealed objects: the +// key must open the first segment of one of them. keyRefusals objects that +// refuse the key decide a mismatch; at most keyTries are read, so a run of +// damaged objects cannot stall the check. +const ( + keyRefusals = 3 + keyTries = 10 +) + +// BucketKeyID reads the key id the bucket records, "" when it records none. +func BucketKeyID(ctx context.Context, b Bucket) (string, error) { + rc, err := b.Get(ctx, keyMark) + if errors.Is(err, ErrNotFound) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("read %s: %w", keyMark, err) + } + defer rc.Close() + raw, err := io.ReadAll(io.LimitReader(rc, 256)) + if err != nil { + return "", fmt.Errorf("read %s: %w", keyMark, err) + } + id := strings.TrimSpace(string(raw)) + if !keyIDPattern.MatchString(id) { + return "", fmt.Errorf("offsite: %s in the bucket is not a key id Felis wrote", keyMark) + } + return id, nil +} + +// errOpened stops Decrypt once the first segment has authenticated. +var errOpened = errors.New("offsite: first segment opened") + +type firstSegment struct{} + +func (firstSegment) Write([]byte) (int, error) { return 0, errOpened } + +// CheckKey tells whether key is the one the bucket's objects are sealed with. +// It writes nothing; a mismatch is ErrKeyMismatch. +func CheckKey(ctx context.Context, b Bucket, key []byte) (KeyFit, error) { + mine := KeyID(key) + id, err := BucketKeyID(ctx, b) + if err != nil { + return 0, err + } + if id != "" { + if id != mine { + return 0, fmt.Errorf("%w: the bucket records key id %s, this key is %s; %s", ErrKeyMismatch, id, mine, keyMismatchFix) + } + return KeyRecorded, nil + } + var sealed []Object + for _, dir := range []string{dbDir, worldsDir, registryDir, uploadsDir} { + objs, err := b.List(ctx, dir) + if err != nil { + return 0, fmt.Errorf("list %s: %w", dir, err) + } + for _, o := range objs { + if strings.HasSuffix(o.Key, objExt) { + sealed = append(sealed, o) + } + } + } + if len(sealed) == 0 { + return KeyUnused, nil + } + sort.Slice(sealed, func(i, j int) bool { return sealed[i].Modified.After(sealed[j].Modified) }) + var refused []string + var unjudged error + for i, o := range sealed { + if i == keyTries || len(refused) == keyRefusals { + break + } + rc, err := b.Get(ctx, o.Key) + if errors.Is(err, ErrNotFound) { + continue // pruned since the listing + } + if err != nil { + return 0, fmt.Errorf("%s: %w", o.Key, err) + } + err = Decrypt(firstSegment{}, rc, key) + rc.Close() + switch { + case err == nil || errors.Is(err, errOpened): + return KeyOpens, nil + case errors.Is(err, ErrAuth): + refused = append(refused, o.Key) + case unjudged == nil: + unjudged = fmt.Errorf("%s: %w", o.Key, err) + } + } + if len(refused) > 0 { + return 0, fmt.Errorf("%w: this key (key id %s) opens none of %s; %s", ErrKeyMismatch, mine, strings.Join(refused, ", "), keyMismatchFix) + } + if unjudged != nil { + return 0, fmt.Errorf("offsite: cannot tell which key sealed the bucket's objects: %w", unjudged) + } + return KeyUnused, nil +} + +// ClaimKey is CheckKey, then records key's id in a bucket that has none, so +// that every later check reads the id. +func ClaimKey(ctx context.Context, b Bucket, key []byte) error { + fit, err := CheckKey(ctx, b, key) + if err != nil || fit == KeyRecorded { + return err + } + id := KeyID(key) + "\n" + if err := b.Put(ctx, keyMark, strings.NewReader(id), int64(len(id))); err != nil { + return fmt.Errorf("record the key id in %s: %w", keyMark, err) + } + return nil +} diff --git a/internal/offsite/keymark_test.go b/internal/offsite/keymark_test.go new file mode 100644 index 0000000..b831b5e --- /dev/null +++ b/internal/offsite/keymark_test.go @@ -0,0 +1,248 @@ +package offsite + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +// putAt stores data under key as modified at t0 plus min minutes. +func putAt(b *memBucket, key string, data []byte, min int) { + if b.modified == nil { + b.modified = map[string]time.Time{} + } + b.objs[key] = data + b.modified[key] = now.Add(time.Duration(min) * time.Minute) +} + +func TestCheckKey(t *testing.T) { + key, other := testKey(t), testKey(t) + big := make([]byte, 3*segmentSize) + tailDamaged := seal(t, big, key) + tailDamaged[len(tailDamaged)-1] ^= 1 + for _, tc := range []struct { + what string + bucket func(b *memBucket) + fit KeyFit + err string // "" for none + named []string // what a refusal names + skip []string // what it must leave out + }{ + {what: "an empty bucket", bucket: func(b *memBucket) {}, fit: KeyUnused}, + {what: "nothing Felis sealed", bucket: func(b *memBucket) { putAt(b, "db/notes.txt", []byte("hi"), 0) }, fit: KeyUnused}, + { + what: "the recorded id", fit: KeyRecorded, + bucket: func(b *memBucket) { + b.objs[keyMark] = []byte(KeyID(key) + "\n") + putAt(b, "worlds/a.tar.gz.fenc", seal(t, []byte("a"), other), 0) // the id is the judge + }, + }, + { + what: "another recorded id", + bucket: func(b *memBucket) { b.objs[keyMark] = []byte(KeyID(other) + "\n") }, + err: "sealed with another key", named: []string{KeyID(other), KeyID(key), "offsite.env"}, + }, + { + what: "a marker Felis did not write", + bucket: func(b *memBucket) { b.objs[keyMark] = []byte("hello") }, + err: "not a key id Felis wrote", + }, + { + what: "an unmarked bucket the key opens", fit: KeyOpens, + bucket: func(b *memBucket) { + putAt(b, "db/felis-db-20260101T030000Z-daily.tar.fenc", seal(t, []byte("old"), other), 0) + putAt(b, "registry/blobs/aa.fenc", seal(t, []byte("new"), key), 10) + }, + }, + { + what: "only the first segment is read", fit: KeyOpens, + bucket: func(b *memBucket) { putAt(b, "uploads/blobs/bb.fenc", tailDamaged, 0) }, + }, + { + what: "an unmarked bucket sealed with another key", + bucket: func(b *memBucket) { + for i, k := range []string{"worlds/1.fenc", "worlds/2.fenc", "db/3.fenc", "uploads/index/4.json.fenc"} { + putAt(b, k, seal(t, []byte(k), other), i) + } + }, + err: "sealed with another key", named: []string{"uploads/index/4.json.fenc", "db/3.fenc", "worlds/2.fenc", KeyID(key)}, skip: []string{"worlds/1.fenc"}, + }, + { + what: "two refusals, then an object the key opens", fit: KeyOpens, + bucket: func(b *memBucket) { + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), key), 0) + putAt(b, "worlds/2.fenc", seal(t, []byte("2"), other), 1) + putAt(b, "worlds/3.fenc", seal(t, []byte("3"), other), 2) + }, + }, + { + what: "damaged objects are passed over", fit: KeyOpens, + bucket: func(b *memBucket) { + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), key), 0) + putAt(b, "worlds/2.fenc", []byte("partial"), 1) + }, + }, + { + what: "a refusal among damaged objects", + bucket: func(b *memBucket) { + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), other), 0) + putAt(b, "worlds/2.fenc", []byte("partial"), 1) + }, + err: "sealed with another key", named: []string{"worlds/1.fenc"}, + }, + { + what: "nothing it can judge", + bucket: func(b *memBucket) { + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), key), 0) + for i := range keyTries { + putAt(b, fmt.Sprintf("worlds/junk-%d.fenc", i), []byte("partial"), 1+i) + } + }, + err: "cannot tell which key", named: []string{"worlds/junk-"}, + }, + } { + t.Run(tc.what, func(t *testing.T) { + b := newMemBucket() + tc.bucket(b) + fit, err := CheckKey(context.Background(), b, key) + if b.puts != 0 { + t.Errorf("CheckKey wrote %d objects", b.puts) + } + if tc.err == "" { + if err != nil || fit != tc.fit { + t.Fatalf("CheckKey = %v, %v; want %v", fit, err, tc.fit) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Fatalf("CheckKey = %v, %v; want an error saying %q", fit, err, tc.err) + } + if mismatch := tc.err == "sealed with another key"; errors.Is(err, ErrKeyMismatch) != mismatch { + t.Errorf("errors.Is(err, ErrKeyMismatch) = %v, want %v: %v", !mismatch, mismatch, err) + } + for _, s := range tc.named { + if !strings.Contains(err.Error(), s) { + t.Errorf("error lacks %q: %v", s, err) + } + } + for _, s := range tc.skip { + if strings.Contains(err.Error(), s) { + t.Errorf("error names %q: %v", s, err) + } + } + }) + } + + // A bucket that does not answer is not a mismatch. + b := newMemBucket() + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), other), 0) + down := errors.New("connection reset") + b.getErr = map[string]error{"worlds/1.fenc": down} + if _, err := CheckKey(context.Background(), b, key); !errors.Is(err, down) || errors.Is(err, ErrKeyMismatch) { + t.Errorf("unreachable object: err = %v, want the read error", err) + } + b.getErr = map[string]error{keyMark: down} + if _, err := CheckKey(context.Background(), b, key); !errors.Is(err, down) || errors.Is(err, ErrKeyMismatch) { + t.Errorf("unreachable marker: err = %v, want the read error", err) + } + + // An object pruned between the listing and the read is passed over. + b = newMemBucket() + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), key), 0) + putAt(b, "db/2.fenc", seal(t, []byte("2"), key), 1) + b.getErr = map[string]error{"db/2.fenc": fmt.Errorf("%w: db/2.fenc", ErrNotFound)} + if fit, err := CheckKey(context.Background(), b, key); fit != KeyOpens || err != nil { + t.Errorf("an object gone since the listing: CheckKey = %v, %v; want KeyOpens", fit, err) + } +} + +func TestClaimKey(t *testing.T) { + key, other := testKey(t), testKey(t) + marker := func(b *memBucket) string { return string(b.objs[keyMark]) } + + b := newMemBucket() + if err := ClaimKey(context.Background(), b, key); err != nil { + t.Fatal(err) + } + if marker(b) != KeyID(key)+"\n" { + t.Fatalf("empty bucket: marker = %q, want %s", marker(b), KeyID(key)) + } + if err := ClaimKey(context.Background(), b, key); err != nil || b.puts != 1 { + t.Errorf("second claim: err %v, puts %d; want the marker written once", err, b.puts) + } + if fit, err := CheckKey(context.Background(), b, key); fit != KeyRecorded || err != nil { + t.Errorf("after the claim: CheckKey = %v, %v", fit, err) + } + if err := ClaimKey(context.Background(), b, other); !errors.Is(err, ErrKeyMismatch) || marker(b) != KeyID(key)+"\n" { + t.Errorf("another key: err %v, marker %q; want a refusal that leaves the marker", err, marker(b)) + } + + b = newMemBucket() + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), key), 0) + if err := ClaimKey(context.Background(), b, key); err != nil || marker(b) != KeyID(key)+"\n" { + t.Errorf("unmarked bucket the key opens: err %v, marker %q", err, marker(b)) + } + + b = newMemBucket() + putAt(b, "worlds/1.fenc", seal(t, []byte("1"), other), 0) + if err := ClaimKey(context.Background(), b, key); !errors.Is(err, ErrKeyMismatch) || b.puts != 0 { + t.Errorf("unmarked bucket under another key: err %v, puts %d; want a refusal that writes nothing", err, b.puts) + } +} + +// TestSyncRefusesAnotherKeysBucket: a host whose key does not match the +// bucket's objects neither copies into it nor prunes what only the right key +// opens. +func TestSyncRefusesAnotherKeysBucket(t *testing.T) { + for _, marked := range []bool{true, false} { + t.Run(fmt.Sprintf("marked=%v", marked), func(t *testing.T) { + cat := &fakeCatalog{rows: []*row{ + {WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"}, + }} + s, b := newSyncer(t, cat) + other := testKey(t) + delete(b.objs, keyMark) + if marked { + b.objs[keyMark] = []byte(KeyID(other) + "\n") + } + writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 100) + writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50) + for i, name := range []string{"felis-db-20260901T030000Z-daily.tar", "felis-db-20260902T030000Z-daily.tar", "felis-db-20260903T030000Z-daily.tar"} { + putAt(b, DBKey(name), seal(t, []byte(name), other), i) + } + before := len(b.objs) + + _, err := s.Run(context.Background()) + if !errors.Is(err, ErrKeyMismatch) { + t.Fatalf("Run error = %v, want ErrKeyMismatch", err) + } + if len(b.started) != 0 || len(b.removed) != 0 || len(b.objs) != before { + t.Errorf("the refused run began puts %v and removed %v", b.started, b.removed) + } + if !cat.rows[0].offsite.IsZero() { + t.Error("the refused run recorded alpha as copied") + } + }) + } +} + +// TestSyncRecordsTheKey: the first run into a bucket records its key before +// the first upload. +func TestSyncRecordsTheKey(t *testing.T) { + s, b := newSyncer(t, &fakeCatalog{}) + delete(b.objs, keyMark) + writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50) + if _, err := s.Run(context.Background()); err != nil { + t.Fatal(err) + } + if !bytes.Equal(b.objs[keyMark], []byte(KeyID(s.Key)+"\n")) { + t.Fatalf("marker = %q, want %s", b.objs[keyMark], KeyID(s.Key)) + } + if len(b.started) != 2 || b.started[0] != keyMark { + t.Errorf("puts began in the order %v, want the marker, then the bundle", b.started) + } +} diff --git a/internal/offsite/offsite_test.go b/internal/offsite/offsite_test.go index 84088d9..80d2d4b 100644 --- a/internal/offsite/offsite_test.go +++ b/internal/offsite/offsite_test.go @@ -122,6 +122,10 @@ type memBucket struct { stall map[string]bool // started lists every Put in the order it began. started []string + // modified is what List reports as each key's modification time. + modified map[string]time.Time + // getErr fails Get for a key the way an unreachable bucket would. + getErr map[string]error } func newMemBucket() *memBucket { @@ -157,6 +161,9 @@ func (b *memBucket) Put(ctx context.Context, key string, r io.Reader, size int64 func (b *memBucket) Get(_ context.Context, key string) (io.ReadCloser, error) { b.mu.Lock() defer b.mu.Unlock() + if err := b.getErr[key]; err != nil { + return nil, err + } data, ok := b.objs[key] if !ok { return nil, fmt.Errorf("%w: %s", ErrNotFound, key) @@ -170,7 +177,7 @@ func (b *memBucket) List(_ context.Context, prefix string) ([]Object, error) { var out []Object for k, v := range b.objs { if strings.HasPrefix(k, prefix) { - out = append(out, Object{Key: k, Size: int64(len(v))}) + out = append(out, Object{Key: k, Size: int64(len(v)), Modified: b.modified[k]}) } } sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) @@ -245,11 +252,15 @@ func writeFile(t *testing.T, dir, name string, size int) []byte { var now = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) +// newSyncer syncs into a bucket that already records its key, as every bucket +// does after its first run. func newSyncer(t *testing.T, cat *fakeCatalog) (*Syncer, *memBucket) { t.Helper() b := newMemBucket() + key := testKey(t) + b.objs[keyMark] = []byte(KeyID(key) + "\n") return &Syncer{ - Bucket: b, Catalog: cat, Key: testKey(t), + Bucket: b, Catalog: cat, Key: key, ArchiveDir: t.TempDir(), DBDir: t.TempDir(), DBKeep: 2, Now: func() time.Time { return now }, }, b @@ -393,7 +404,7 @@ func TestSyncDBKeepsNewest(t *testing.T) { keys = append(keys, k) } sort.Strings(keys) - want := "db/felis-db-20260923T030000Z-manual.tar.fenc db/felis-db-20260924T030000Z-daily.tar.fenc db/notes.txt" + want := "db/felis-db-20260923T030000Z-manual.tar.fenc db/felis-db-20260924T030000Z-daily.tar.fenc db/notes.txt " + keyMark if strings.Join(keys, " ") != want { t.Fatalf("bucket = %v, want %s", keys, want) } diff --git a/internal/offsite/status.go b/internal/offsite/status.go index 81568d6..3a6b174 100644 --- a/internal/offsite/status.go +++ b/internal/offsite/status.go @@ -28,6 +28,10 @@ type Status struct { Prefix string `json:"prefix,omitempty"` KeyID string `json:"key_id"` Result Result `json:"result"` + // KeyMismatch is a run refused because the bucket's objects are sealed + // with another key (ErrKeyMismatch): no later run copies anything until + // the key is fixed, so the watchdog reports it at once. + KeyMismatch bool `json:"key_mismatch,omitempty"` } // ReadStatus reads the status file. A missing file is (nil, nil): no sync has diff --git a/internal/offsite/sync.go b/internal/offsite/sync.go index 3b4d5c5..d468d62 100644 --- a/internal/offsite/sync.go +++ b/internal/offsite/sync.go @@ -200,6 +200,11 @@ func (s *Syncer) Run(ctx context.Context) (Result, error) { s.logf("%s", msg) } + // Before anything is written or pruned: objects sealed with another key + // are copies only that key opens, and DBKeep would prune them. + if err := ClaimKey(ctx, s.Bucket, s.Key); err != nil { + return res, err + } remoteWorlds, err := s.listSizes(ctx, worldsDir) if err != nil { return res, fmt.Errorf("list %s in the bucket: %w", worldsDir, err) diff --git a/internal/watchdog/probes.go b/internal/watchdog/probes.go index fce910f..a2dc404 100644 --- a/internal/watchdog/probes.go +++ b/internal/watchdog/probes.go @@ -360,6 +360,14 @@ func OffsiteFinding(statusFile string, now time.Time) *Finding { Hint: hint, } } + if st != nil && st.KeyMismatch { + return &Finding{ + Key: "offsite", Severity: Warning, For: backupFor, + Summary: "异地备份已停止:桶里的对象是用另一把密钥加密的,本机不往桶里写入也不清理任何对象(" + st.LastError + ")", + SummaryEN: "the off-site copy has stopped: the bucket's objects are sealed with another key, and this host writes and prunes nothing there (" + st.LastError + ")", + Hint: "`sudo felis offsite check-key`; set FELIS_OFFSITE_KEY in /etc/felis/offsite.env to the key the bucket was written with (docs/troubleshooting.md §16)", + } + } if st != nil && !st.LastSuccess.IsZero() && now.Sub(st.LastSuccess) <= offsite.StaleAfter { return nil } diff --git a/internal/watchdog/probes_test.go b/internal/watchdog/probes_test.go index 72d7f04..26f9fa8 100644 --- a/internal/watchdog/probes_test.go +++ b/internal/watchdog/probes_test.go @@ -183,6 +183,11 @@ func TestOffsiteFinding(t *testing.T) { if f := OffsiteFinding(path, t0); f != nil { t.Fatalf("a success within %s reported: %+v", offsite.StaleAfter, f) } + // A key mismatch stops every later run too: reported without waiting out StaleAfter. + write(offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-2 * time.Hour), LastError: "sealed with another key", KeyMismatch: true}) + if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "has stopped") || !strings.Contains(f.SummaryEN, "(sealed with another key)") || !strings.Contains(f.Hint, "check-key") { + t.Fatalf("key mismatch: %+v", f) + } } func TestMemoryFinding(t *testing.T) {