feat(offsite): 异地副本加入 registry 用户镜像，按摘要加密去重，索引按版本保留 14 天，新增 fetch-images 回推恢复

cmd/felis/mirrortools.go

@@ -14,7 +14,6 @@ import (
 
 	"felis.lolicon.best/internal/build"
 	"felis.lolicon.best/internal/imagepush"
-	"felis.lolicon.best/internal/registrygate"
 )
 
 // defaultBuildToolsStatus is where mirror-build-tools records its last run; the
@@ -49,16 +48,9 @@ func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
 		return 2
 	}
-	if err := loadEnvFile(*secrets); err != nil {
-		fmt.Fprintf(stderr, "felis mirror-build-tools: read %s: %v\n", *secrets, err)
-		return 1
-	}
-	user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
-	if pass == "" {
-		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
-	}
-	if pass == "" {
-		fmt.Fprintln(stderr, "felis mirror-build-tools: no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
 		return 2
 	}
 

cmd/felis/offsite.go

@@ -24,12 +24,14 @@ import (
 )
 
 const offsiteUsage = `usage:
-  felis offsite sync         [-config path] [-archive-dir dir] [-db-dir dir] [-status-file path]
+  felis offsite sync         [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port|off]
+                             [-status-file path]
   felis offsite status       [-config path] [-status-file path]
   felis offsite list         [-config path]
   felis offsite fetch-db     [-config path | -endpoint url -bucket name [-region r] [-prefix p]]
                              [-dir dir] latest|<bundle>
   felis offsite fetch-worlds [-config path] [-archive-dir dir]
+  felis offsite fetch-images [-config path] [-registry host:port] [-at version]
   felis offsite keygen
 
 Every verb but keygen reads the bucket credentials and the encryption key from
@@ -43,7 +45,8 @@ FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
 const defaultOffsiteEnvFile = "/etc/felis/offsite.env"
 
 // cmdOffsite implements `felis offsite`: the off-site copy of the world
-// archives and the database bundles (internal/offsite). felis-offsite.timer
+// archives, the database bundles and the registry's user images
+// (internal/offsite). felis-offsite.timer
 // runs `sync` hourly on the host; the fetch verbs are the way back after the
 // node is lost (docs/troubleshooting.md §16).
 func cmdOffsite(args []string, stdout, stderr io.Writer) int {
@@ -66,6 +69,8 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int {
 		return offsiteFetchDB(fs, rest, stdout, stderr)
 	case "fetch-worlds":
 		return offsiteFetchWorlds(fs, rest, stdout, stderr)
+	case "fetch-images":
+		return offsiteFetchImages(fs, rest, stdout, stderr)
 	case "keygen":
 		k, err := offsite.NewKey()
 		if err != nil {
@@ -186,6 +191,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
 	archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
 	backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
 	dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
+	registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
 	statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
 	if err := fs.Parse(args); err != nil {
 		return 2
@@ -202,7 +208,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
 	if prev, _ := offsite.ReadStatus(*statusFile); prev != nil {
 		st.LastSuccess = prev.LastSuccess
 	}
-	res, err := runOffsiteSync(cfg, env, *archiveDir, *backupPVC, *dbDir, stderr)
+	res, err := runOffsiteSync(cfg, env, *archiveDir, *backupPVC, *dbDir, offsiteRegistryEndpoint(*registry, cfg.Registry), stderr)
 	st.Result = res
 	if err != nil {
 		st.LastError = err.Error()
@@ -212,12 +218,16 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
 	if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
 		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
 	}
-	fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; bucket holds %d worlds (%s) and %d bundles\n",
+	fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s)\n",
 		res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
-		res.DBUploaded, res.DBPruned, res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB)
+		res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
+		res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes))
 	for _, m := range res.WorldsMissing {
 		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
 	}
+	for _, m := range res.ImagesIncomplete {
+		fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
+	}
 	if err != nil {
 		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
 		return 1
@@ -225,7 +235,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
 	return 0
 }
 
-func runOffsiteSync(cfg *config.Config, env *offsiteEnv, archiveDir, backupPVC, dbDir string, log io.Writer) (offsite.Result, error) {
+func runOffsiteSync(cfg *config.Config, env *offsiteEnv, archiveDir, backupPVC, dbDir, registry string, log io.Writer) (offsite.Result, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute)
 	defer cancel()
 	checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
@@ -250,6 +260,9 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, archiveDir, backupPVC,
 		Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
 		ArchiveDir: archiveDir, DBDir: dbDir, DBKeep: env.cfg.DBKeep, Log: log,
 	}
+	if registry != "" {
+		s.Images = newRegistryImages(registry)
+	}
 	return s.Run(ctx)
 }
 
@@ -347,7 +360,7 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
 		return 1
 	}
 	if !cfg.Offsite.Enabled() {
-		fmt.Fprintln(stdout, "off-site copy: not configured. World archives and database bundles exist on this machine only.")
+		fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles and user images exist on this machine only.")
 		fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
 		return 1
 	}
@@ -380,10 +393,19 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
 	r := st.Result
 	fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
 		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
+	if r.ImageIndex != "" {
+		fmt.Fprintf(stdout, "images:       %d in %d repositories (%s), registry index %s\n",
+			r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
+	} else {
+		fmt.Fprintln(stdout, "images:       not copied (no in-cluster registry, or no sync has reached it yet)")
+	}
 	fmt.Fprintf(stdout, "waiting:      %d world archives not yet copied\n", r.WorldsPending)
 	for _, m := range r.WorldsMissing {
 		fmt.Fprintf(stdout, "missing:      %s is recorded but not on the volume\n", m)
 	}
+	for _, m := range r.ImagesIncomplete {
+		fmt.Fprintf(stdout, "not whole:    %s\n", m)
+	}
 	if st.LastSuccess.IsZero() || now.Sub(st.LastSuccess) > offsite.StaleAfter {
 		fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
 		return 1
@@ -434,6 +456,20 @@ func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
 		total += w.Size
 	}
 	fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
+	versions, err := offsite.ImageIndexes(ctx, env.bucket)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
+	for i := len(versions) - 1; i >= 0; i-- {
+		x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
+		if err != nil {
+			fmt.Fprintf(stdout, "  %s  unreadable: %v\n", versions[i], err)
+			continue
+		}
+		fmt.Fprintf(stdout, "  %s  %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
+	}
 	return 0
 }
 

cmd/felis/offsite_images.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
+	"felis.lolicon.best/internal/offsite"
+	"felis.lolicon.best/internal/registrygate"
+	"felis.lolicon.best/internal/registryprune"
+)
+
+// registryImages is the platform registry as the off-site copy sees it: read
+// anonymously through the gate (the catalog, its manifest index, manifests and
+// blobs) and, for a restore, written as the platform principal. Both go through
+// the node's loopback hostPort, the way the installer pushes.
+type registryImages struct {
+	host   string
+	index  *registryprune.Client
+	source *imagepush.Source
+	pusher *imagepush.Pusher
+}
+
+func newRegistryImages(endpoint string) *registryImages {
+	return &registryImages{
+		host:   endpoint,
+		index:  &registryprune.Client{Endpoint: "http://" + endpoint},
+		source: &imagepush.Source{Scheme: "http"},
+	}
+}
+
+func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
+	return r.index.Repositories(ctx)
+}
+
+func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
+	idx, err := r.index.Index(ctx, repo)
+	if err != nil {
+		return nil, nil, err
+	}
+	digests := make([]string, 0, len(idx.Revisions))
+	for _, rev := range idx.Revisions {
+		digests = append(digests, rev.Digest)
+	}
+	return digests, idx.Tags, nil
+}
+
+func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
+	body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
+	return body, mt, registryGone(err)
+}
+
+func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
+	rc, err := r.source.Blob(ctx, r.host, repo, digest)
+	return rc, registryGone(err)
+}
+
+func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
+	return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
+}
+
+func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
+	_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
+	return err
+}
+
+// registryGone marks a 404 as a manifest or blob the registry no longer holds.
+func registryGone(err error) error {
+	var se *imagepush.StatusError
+	if errors.As(err, &se) && se.Code == http.StatusNotFound {
+		return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
+	}
+	return err
+}
+
+// offsiteRegistryEndpoint is where the host reaches the registry whose images
+// the off-site copy covers: flag when given ("off" for none), otherwise the
+// loopback hostPort of the in-cluster registry [registry] url names. A
+// registry outside the cluster is not this install's to copy.
+func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
+	switch flag {
+	case "off":
+		return ""
+	case "":
+	default:
+		return flag
+	}
+	host, _, _ := strings.Cut(reg.URL, "/")
+	name, _, _ := strings.Cut(host, ":")
+	if strings.HasSuffix(name, ".svc") || strings.HasSuffix(name, ".svc.cluster.local") {
+		return loopbackEndpoint(host)
+	}
+	return ""
+}
+
+// registryWriteCredential is the credential a host-side push uses:
+// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
+// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
+func registryWriteCredential(secrets string) (string, string, error) {
+	if err := loadEnvFile(secrets); err != nil {
+		return "", "", fmt.Errorf("read %s: %w", secrets, err)
+	}
+	user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
+	if pass == "" {
+		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
+	}
+	if pass == "" {
+		return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
+	}
+	return user, pass, nil
+}
+
+func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
+	cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
+	envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
+	registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
+	secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
+	at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	cfg, env, err := loadOffsite(*cfgPath, *envFile)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
+	if endpoint == "" {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
+		return 2
+	}
+	user, pass, err := registryWriteCredential(*secrets)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 2
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
+	defer cancel()
+	stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
+		return 1
+	}
+	fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
+		stamp, len(idx.Repositories), idx.Images(), endpoint)
+	target := newRegistryImages(endpoint)
+	target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
+	res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
+	fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
+		res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
+	for _, f := range res.Failures {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
+		return 1
+	}
+	return 0
+}

cmd/felis/offsite_test.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
 	"felis.lolicon.best/internal/config"
+	"felis.lolicon.best/internal/imagepush"
 	"felis.lolicon.best/internal/offsite"
 )
 
@@ -94,3 +96,27 @@ func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
 		t.Fatalf("exit %d: %s", code, errb.String())
 	}
 }
+
+func TestOffsiteRegistryEndpoint(t *testing.T) {
+	for _, tc := range []struct{ flag, url, want string }{
+		{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
+		{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
+		{"", "ghcr.io/acme", ""},
+		{"", "", ""},
+		{"off", "registry.felis.svc:5000", ""},
+		{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
+	} {
+		if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
+			t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
+		}
+	}
+}
+
+func TestRegistryGoneMarksNotFound(t *testing.T) {
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("404 = %v, want ErrImageGone", err)
+	}
+	if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
+		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
+	}
+}

cmd/felis/run.go

@@ -13,7 +13,7 @@ Usage:
 Commands:
   migrate up        Apply embedded database migrations under an advisory lock (snapshots the database first)
   db                Back up, verify, list and restore the control-plane database (backup|restore|verify|list|check)
-  offsite           Copy world archives and database bundles to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|keygen)
+  offsite           Copy world archives, database bundles and user images to an off-site bucket, and fetch them back (sync|status|list|fetch-db|fetch-worlds|fetch-images|keygen)
   operator          Run the MinecraftServer controller-manager
   api               Run the felis-api HTTP server
   nano              Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)