FelisMC/Felis
2
0 Fork 0
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

fix(offsite): 桶内记录密钥指纹，密钥不符时 sync 拒绝写入和清理，安装时大声提示

Browse Source
This commit is contained in:
Lemon-miaow committed 2026-09-27 07:21:25 +08:00
1 parent fa46733859
commit c9e5e2fe3e
12 files changed
+717 -24
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/felis/offsite.go
+81 -8
View File
@@ -33,12 +33,17 @@ const offsiteUsage = `usage:
felis offsite fetch-worlds [-config path] [-archive-dir dir]
felis offsite fetch-images [-config path] [-registry host:port] [-at version]
felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
felis offsite check-key [-config path]
felis offsite keygen
Every verb but keygen reads the bucket credentials and the encryption key from
the variables [offsite] names (default FELIS_OFFSITE_ACCESS_KEY,
FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
-env-file (default /etc/felis/offsite.env).
check-key tells whether the key is the one the bucket's objects are sealed
with, writing nothing; it exits 3 when they are sealed with another key, and
sync then refuses to write or prune anything in the bucket.
`
// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
@@ -74,6 +79,8 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int {
return offsiteFetchImages(fs, rest, stdout, stderr)
case "fetch-uploads":
return offsiteFetchUploads(fs, rest, stdout, stderr)
case "check-key":
return offsiteCheckKey(fs, rest, stdout, stderr)
case "keygen":
k, err := offsite.NewKey()
if err != nil {
@@ -218,12 +225,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
registry: offsiteRegistryEndpoint(*registry, cfg.Registry),
uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC,
}, stderr)
st.Result = res
if err != nil {
st.LastError = err.Error()
} else {
st.LastSuccess = st.LastAttempt
}
recordRun(&st, res, err)
if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
}
@@ -246,6 +248,17 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
return 0
}
// recordRun puts one pass's outcome into its status record.
func recordRun(st *offsite.Status, res offsite.Result, err error) {
st.Result = res
if err != nil {
st.LastError = err.Error()
st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
} else {
st.LastSuccess = st.LastAttempt
}
}
// offsiteSources is where one sync pass reads from: the world archive volume
// (archiveDir, or the backupPVC's directory), the bundle directory, the
// registry's loopback endpoint and the uploads volume (uploadsDir, or the
@@ -438,6 +451,10 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
if st.LastError != "" {
fmt.Fprintf(stdout, "last error: %s\n", st.LastError)
}
if st.KeyMismatch {
fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
return 1
}
r := st.Result
fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
@@ -552,6 +569,62 @@ func printDBBundles(ctx context.Context, b offsite.Bucket, key []byte, bundles [
}
}
func offsiteCheckKey(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
if err := fs.Parse(args); err != nil {
return 2
}
_, env, err := loadOffsite(*cfgPath, *envFile)
if err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
if err := env.bucket.Check(ctx); err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
return 1
}
return checkKey(ctx, env.bucket, env.key, stdout, stderr)
}
// checkKey is check-key once the bucket is open: 0 when the key fits, 3 when
// the bucket's objects are sealed with another one, 1 when it cannot tell.
func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr io.Writer) int {
fit, err := offsite.CheckKey(ctx, b, key)
if err != nil {
fmt.Fprintf(stderr, "felis offsite check-key: %v\n", err)
if errors.Is(err, offsite.ErrKeyMismatch) {
return 3
}
return 1
}
id := offsite.KeyID(key)
switch fit {
case offsite.KeyRecorded:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket records key id %s, this key's\n", id)
case offsite.KeyOpens:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket's newest objects open with this key (key id %s); the next sync records it\n", id)
case offsite.KeyUnused:
fmt.Fprintf(stdout, "felis offsite check-key: the bucket holds no sealed object yet; the first sync records key id %s\n", id)
}
return 0
}
// keyHint explains an object the key cannot open when the bucket records
// another key's id, "" otherwise.
func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {
if !errors.Is(err, offsite.ErrAuth) {
return ""
}
id, ierr := offsite.BucketKeyID(ctx, b)
if ierr != nil || id == "" || id == offsite.KeyID(key) {
return ""
}
return fmt.Sprintf("\n the bucket records key id %s, and this key is %s: set FELIS_OFFSITE_KEY to the key the bucket was written with", id, offsite.KeyID(key))
}
func offsiteFetchDB(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml; on a host with no install yet, give -endpoint and -bucket instead")
envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
@@ -603,7 +676,7 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string,
if name == "latest" {
var err error
if name, _, err = offsite.ChooseDB(ctx, b, key); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
return 1
}
}
@@ -617,7 +690,7 @@ func fetchDB(ctx context.Context, b offsite.Bucket, key []byte, arg, dir string,
}
dst := filepath.Join(dir, name)
if err := offsite.FetchObject(ctx, b, key, offsite.DBKey(name), dst, 0o600); err != nil {
fmt.Fprintf(stderr, "felis offsite fetch-db: %v\n", err)
fmt.Fprintf(stderr, "felis offsite fetch-db: %v%s\n", err, keyHint(ctx, b, key, err))
return 1
}
m, err := dbbackup.Verify(dst)
cmd/felis/offsite_test.go
+115
View File
@@ -8,6 +8,7 @@ import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
@@ -285,6 +286,120 @@ func TestOffsiteFetchDB(t *testing.T) {
})
}
func TestOffsiteCheckKey(t *testing.T) {
newKey := func() []byte {
raw, _ := offsite.NewKey()
k, _ := offsite.ParseKey(raw)
return k
}
key, other := newKey(), newKey()
marked := func(k []byte) mapBucket { return mapBucket{"felis-key-id": []byte(offsite.KeyID(k) + "\n")} }
unmarked := func(k []byte) mapBucket {
b := mapBucket{}
putBundle(t, b, k, 0, nil)
return b
}
for _, tc := range []struct {
what string
bucket mapBucket
code int
says []string
}{
{"the recorded key", marked(key), 0, []string{"records key id " + offsite.KeyID(key)}},
{"an unmarked bucket the key opens", unmarked(key), 0, []string{"open with this key", "the next sync records it"}},
{"an empty bucket", mapBucket{}, 0, []string{"no sealed object yet", "records key id " + offsite.KeyID(key)}},
{"another recorded key", marked(other), 3, []string{offsite.KeyID(other), offsite.KeyID(key), "FELIS_OFFSITE_KEY"}},
{"an unmarked bucket under another key", unmarked(other), 3, []string{"opens none of db/felis-db-"}},
{"a marker Felis did not write", mapBucket{"felis-key-id": []byte("hello")}, 1, []string{"not a key id"}},
} {
t.Run(tc.what, func(t *testing.T) {
before := len(tc.bucket)
var out, errb bytes.Buffer
code := checkKey(context.Background(), tc.bucket, key, &out, &errb)
if code != tc.code {
t.Fatalf("exit %d, want %d; stdout %q, stderr %q", code, tc.code, out.String(), errb.String())
}
said := out.String() + errb.String()
for _, s := range tc.says {
if !strings.Contains(said, s) {
t.Errorf("output lacks %q: %s", s, said)
}
}
if len(tc.bucket) != before {
t.Errorf("check-key wrote to the bucket: %d objects, had %d", len(tc.bucket), before)
}
})
}
// fetch-db names both ids when the bucket records another key.
b := marked(other)
name := putBundle(t, b, other, 0, &dbbackup.Counts{Users: 5, Servers: 3})
for _, arg := range []string{"latest", name} {
var out, errb bytes.Buffer
if code := fetchDB(context.Background(), b, key, arg, t.TempDir(), fetchT0, &out, &errb); code != 1 ||
!strings.Contains(errb.String(), "the bucket records key id "+offsite.KeyID(other)+", and this key is "+offsite.KeyID(key)) {
t.Errorf("fetch-db %s under another key: exit %d, stderr %q", arg, code, errb.String())
}
}
// A bundle the bucket lacks is not the key's fault.
var missOut, missErr bytes.Buffer
if code := fetchDB(context.Background(), b, key, "felis-db-20200101T000000Z-daily.tar", t.TempDir(), fetchT0, &missOut, &missErr); code != 1 || strings.Contains(missErr.String(), "records key id") {
t.Errorf("missing bundle: exit %d, stderr %q", code, missErr.String())
}
// A bundle damaged under the recorded key gets no such hint.
b = marked(key)
name = putBundle(t, b, key, 0, &dbbackup.Counts{Users: 5, Servers: 3})
b[offsite.DBKey(name)][60] ^= 1
var out, errb bytes.Buffer
if code := fetchDB(context.Background(), b, key, name, t.TempDir(), fetchT0, &out, &errb); code != 1 || strings.Contains(errb.String(), "records key id") {
t.Errorf("damaged bundle: exit %d, stderr %q", code, errb.String())
}
}
func TestRecordRunMarksAKeyMismatch(t *testing.T) {
t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
for _, tc := range []struct {
err error
mismatch bool
success bool
}{
{nil, false, true},
{errors.New("list worlds/ in the bucket: connection reset"), false, false},
{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false},
} {
st := offsite.Status{LastAttempt: t0}
recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err)
if st.KeyMismatch != tc.mismatch || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 {
t.Errorf("err %v: status %+v", tc.err, st)
}
}
}
func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
dir := t.TempDir()
cfg := filepath.Join(dir, "felis.toml")
writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
statusFile := filepath.Join(dir, "status.json")
st := offsite.Status{LastAttempt: time.Now().Add(-time.Minute), LastSuccess: time.Now().Add(-time.Hour), KeyID: "0123456789abcdef"}
status := func() (int, string) {
t.Helper()
if err := offsite.WriteStatus(statusFile, st); err != nil {
t.Fatal(err)
}
var out, errb bytes.Buffer
code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
return code, out.String() + errb.String()
}
if code, out := status(); code != 0 || strings.Contains(out, "refused") {
t.Fatalf("a recent success: exit %d\n%s", code, out)
}
st.LastError, st.KeyMismatch = "offsite: the bucket's objects are sealed with another key", true
code, out := status()
if code != 1 || !strings.Contains(out, "The last run was refused") || !strings.Contains(out, "key id 0123456789abcdef") || strings.Contains(out, "bucket holds:") {
t.Fatalf("a refused run: exit %d\n%s", code, out)
}
}
func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
rawKey, _ := offsite.NewKey()
key, _ := offsite.ParseKey(rawKey)