feat(offsite): 异地副本加入 registry 用户镜像，按摘要加密去重，索引按版本保留 14 天，新增 fetch-images 回推恢复 (e3ac9cd5) · Commits · FelisMC / Felis · GitLab

cmd/felis/mirrortools.go

+3 −11

Changes for cmd/felis/mirrortools.go: 3 added lines, 11 removed lines.
Original line number	Diff line number	Diff line
		@@ -14,7 +14,6 @@ import (

		"felis.lolicon.best/internal/build"
		"felis.lolicon.best/internal/imagepush"
		"felis.lolicon.best/internal/registrygate"
		)

		// defaultBuildToolsStatus is where mirror-build-tools records its last run; the
		@@ -49,16 +48,9 @@ func cmdMirrorBuildTools(args []string, stdout, stderr io.Writer) int {
		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
		return 2
		}
		if err := loadEnvFile(*secrets); err != nil {
		fmt.Fprintf(stderr, "felis mirror-build-tools: read %s: %v\n", *secrets, err)
		return 1
		}
		user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
		if pass == "" {
		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
		}
		if pass == "" {
		fmt.Fprintln(stderr, "felis mirror-build-tools: no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
		user, pass, err := registryWriteCredential(*secrets)
		if err != nil {
		fmt.Fprintf(stderr, "felis mirror-build-tools: %v\n", err)
		return 2
		}

cmd/felis/offsite.go

+43 −7

Changes for cmd/felis/offsite.go: 43 added lines, 7 removed lines.
Original line number	Diff line number	Diff line
		@@ -24,12 +24,14 @@ import (
		)

		const offsiteUsage = `usage:
		felis offsite sync [-config path] [-archive-dir dir] [-db-dir dir] [-status-file path]
		felis offsite sync [-config path] [-archive-dir dir] [-db-dir dir] [-registry host:port\|off]
		[-status-file path]
		felis offsite status [-config path] [-status-file path]
		felis offsite list [-config path]
		felis offsite fetch-db [-config path \| -endpoint url -bucket name [-region r] [-prefix p]]
		[-dir dir] latest\|<bundle>
		felis offsite fetch-worlds [-config path] [-archive-dir dir]
		felis offsite fetch-images [-config path] [-registry host:port] [-at version]
		felis offsite keygen

		Every verb but keygen reads the bucket credentials and the encryption key from
		@@ -43,7 +45,8 @@ FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
		const defaultOffsiteEnvFile = "/etc/felis/offsite.env"

		// cmdOffsite implements `felis offsite`: the off-site copy of the world
		// archives and the database bundles (internal/offsite). felis-offsite.timer
		// archives, the database bundles and the registry's user images
		// (internal/offsite). felis-offsite.timer
		// runs `sync` hourly on the host; the fetch verbs are the way back after the
		// node is lost (docs/troubleshooting.md §16).
		func cmdOffsite(args []string, stdout, stderr io.Writer) int {
		@@ -66,6 +69,8 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int {
		return offsiteFetchDB(fs, rest, stdout, stderr)
		case "fetch-worlds":
		return offsiteFetchWorlds(fs, rest, stdout, stderr)
		case "fetch-images":
		return offsiteFetchImages(fs, rest, stdout, stderr)
		case "keygen":
		k, err := offsite.NewKey()
		if err != nil {
		@@ -186,6 +191,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
		archiveDir := fs.String("archive-dir", "", "host directory of the world archive volume (default: resolved from the backup PVC through the cluster)")
		backupPVC := fs.String("backup-pvc", "felis-backups", `the world archive PVC, in the [k8s] namespace ("" when backups are off)`)
		dbDir := fs.String("db-dir", dbbackup.DefaultDir, `database bundle directory ("" copies no bundles)`)
		registry := fs.String("registry", "", `host[:port] of the registry whose user images are copied (default: the in-cluster registry's loopback hostPort; "off" copies none)`)
		statusFile := fs.String("status-file", offsite.DefaultStatusFile, "where the result of this run is recorded for the watchdog and `status`")
		if err := fs.Parse(args); err != nil {
		return 2
		@@ -202,7 +208,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
		if prev, _ := offsite.ReadStatus(*statusFile); prev != nil {
		st.LastSuccess = prev.LastSuccess
		}
		res, err := runOffsiteSync(cfg, env, archiveDir, backupPVC, *dbDir, stderr)
		res, err := runOffsiteSync(cfg, env, archiveDir, backupPVC, dbDir, offsiteRegistryEndpoint(registry, cfg.Registry), stderr)
		st.Result = res
		if err != nil {
		st.LastError = err.Error()
		@@ -212,12 +218,16 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
		if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
		}
		fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; bucket holds %d worlds (%s) and %d bundles\n",
		fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s)\n",
		res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
		res.DBUploaded, res.DBPruned, res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB)
		res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
		res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes))
		for _, m := range res.WorldsMissing {
		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
		}
		for _, m := range res.ImagesIncomplete {
		fmt.Fprintf(stderr, "felis offsite sync: registry image not whole: %s\n", m)
		}
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
		return 1
		@@ -225,7 +235,7 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
		return 0
		}

		func runOffsiteSync(cfg config.Config, env offsiteEnv, archiveDir, backupPVC, dbDir string, log io.Writer) (offsite.Result, error) {
		func runOffsiteSync(cfg config.Config, env offsiteEnv, archiveDir, backupPVC, dbDir, registry string, log io.Writer) (offsite.Result, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 50*time.Minute)
		defer cancel()
		checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
		@@ -250,6 +260,9 @@ func runOffsiteSync(cfg config.Config, env offsiteEnv, archiveDir, backupPVC,
		Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
		ArchiveDir: archiveDir, DBDir: dbDir, DBKeep: env.cfg.DBKeep, Log: log,
		}
		if registry != "" {
		s.Images = newRegistryImages(registry)
		}
		return s.Run(ctx)
		}

		@@ -347,7 +360,7 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
		return 1
		}
		if !cfg.Offsite.Enabled() {
		fmt.Fprintln(stdout, "off-site copy: not configured. World archives and database bundles exist on this machine only.")
		fmt.Fprintln(stdout, "off-site copy: not configured. World archives, database bundles and user images exist on this machine only.")
		fmt.Fprintln(stdout, "See docs/troubleshooting.md §16, \"Keep a copy somewhere else\".")
		return 1
		}
		@@ -380,10 +393,19 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
		r := st.Result
		fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
		if r.ImageIndex != "" {
		fmt.Fprintf(stdout, "images: %d in %d repositories (%s), registry index %s\n",
		r.Images, r.ImageRepos, offsite.HumanBytes(r.RemoteImageBytes), r.ImageIndex)
		} else {
		fmt.Fprintln(stdout, "images: not copied (no in-cluster registry, or no sync has reached it yet)")
		}
		fmt.Fprintf(stdout, "waiting: %d world archives not yet copied\n", r.WorldsPending)
		for _, m := range r.WorldsMissing {
		fmt.Fprintf(stdout, "missing: %s is recorded but not on the volume\n", m)
		}
		for _, m := range r.ImagesIncomplete {
		fmt.Fprintf(stdout, "not whole: %s\n", m)
		}
		if st.LastSuccess.IsZero() \|\| now.Sub(st.LastSuccess) > offsite.StaleAfter {
		fmt.Fprintf(stdout, "\nThe last successful sync is older than %s: journalctl -u felis-offsite -n 50\n", dbbackup.Age(offsite.StaleAfter))
		return 1
		@@ -434,6 +456,20 @@ func printOffsiteList(env *offsiteEnv, stdout, stderr io.Writer) int {
		total += w.Size
		}
		fmt.Fprintf(stdout, "world archives: %d (%s)\n", len(worlds), offsite.HumanBytes(total))
		versions, err := offsite.ImageIndexes(ctx, env.bucket)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite list: %v\n", err)
		return 1
		}
		fmt.Fprintf(stdout, "registry index versions (%d, newest first; restore one with fetch-images -at):\n", len(versions))
		for i := len(versions) - 1; i >= 0; i-- {
		x, err := offsite.LoadImageIndex(ctx, env.bucket, env.key, versions[i])
		if err != nil {
		fmt.Fprintf(stdout, " %s unreadable: %v\n", versions[i], err)
		continue
		}
		fmt.Fprintf(stdout, " %s %d images in %d repositories\n", versions[i], x.Images(), len(x.Repositories))
		}
		return 0
		}

cmd/felis/offsite_images.go

0 → 100644

+167 −0

Changes for cmd/felis/offsite_images.go: 167 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"context"
		"errors"
		"flag"
		"fmt"
		"io"
		"net/http"
		"os"
		"strings"
		"time"

		"felis.lolicon.best/internal/config"
		"felis.lolicon.best/internal/imagepush"
		"felis.lolicon.best/internal/offsite"
		"felis.lolicon.best/internal/registrygate"
		"felis.lolicon.best/internal/registryprune"
		)

		// registryImages is the platform registry as the off-site copy sees it: read
		// anonymously through the gate (the catalog, its manifest index, manifests and
		// blobs) and, for a restore, written as the platform principal. Both go through
		// the node's loopback hostPort, the way the installer pushes.
		type registryImages struct {
		host string
		index *registryprune.Client
		source *imagepush.Source
		pusher *imagepush.Pusher
		}

		func newRegistryImages(endpoint string) *registryImages {
		return &registryImages{
		host: endpoint,
		index: &registryprune.Client{Endpoint: "http://" + endpoint},
		source: &imagepush.Source{Scheme: "http"},
		}
		}

		func (r *registryImages) Repositories(ctx context.Context) ([]string, error) {
		return r.index.Repositories(ctx)
		}

		func (r *registryImages) Revisions(ctx context.Context, repo string) ([]string, map[string]string, error) {
		idx, err := r.index.Index(ctx, repo)
		if err != nil {
		return nil, nil, err
		}
		digests := make([]string, 0, len(idx.Revisions))
		for _, rev := range idx.Revisions {
		digests = append(digests, rev.Digest)
		}
		return digests, idx.Tags, nil
		}

		func (r *registryImages) Manifest(ctx context.Context, repo, digest string) ([]byte, string, error) {
		body, mt, err := r.source.Manifest(ctx, r.host, repo, digest)
		return body, mt, registryGone(err)
		}

		func (r *registryImages) Blob(ctx context.Context, repo, digest string) (io.ReadCloser, error) {
		rc, err := r.source.Blob(ctx, r.host, repo, digest)
		return rc, registryGone(err)
		}

		func (r *registryImages) PutBlob(ctx context.Context, repo, digest string, size int64, open func() (io.ReadCloser, error)) error {
		return r.pusher.UploadBlob(ctx, r.host, repo, digest, size, open)
		}

		func (r *registryImages) PutManifest(ctx context.Context, repo, reference, mediaType string, body []byte) error {
		_, err := r.pusher.PutManifest(ctx, r.host, repo, reference, mediaType, body)
		return err
		}

		// registryGone marks a 404 as a manifest or blob the registry no longer holds.
		func registryGone(err error) error {
		var se *imagepush.StatusError
		if errors.As(err, &se) && se.Code == http.StatusNotFound {
		return fmt.Errorf("%w: %v", offsite.ErrImageGone, err)
		}
		return err
		}

		// offsiteRegistryEndpoint is where the host reaches the registry whose images
		// the off-site copy covers: flag when given ("off" for none), otherwise the
		// loopback hostPort of the in-cluster registry [registry] url names. A
		// registry outside the cluster is not this install's to copy.
		func offsiteRegistryEndpoint(flag string, reg config.RegistryConfig) string {
		switch flag {
		case "off":
		return ""
		case "":
		default:
		return flag
		}
		host, _, _ := strings.Cut(reg.URL, "/")
		name, _, _ := strings.Cut(host, ":")
		if strings.HasSuffix(name, ".svc") \|\| strings.HasSuffix(name, ".svc.cluster.local") {
		return loopbackEndpoint(host)
		}
		return ""
		}

		// registryWriteCredential is the credential a host-side push uses:
		// FELIS_REGISTRY_USERNAME/PASSWORD when set, otherwise the platform principal
		// with REGISTRY_PLATFORM_TOKEN from the installer's secrets file.
		func registryWriteCredential(secrets string) (string, string, error) {
		if err := loadEnvFile(secrets); err != nil {
		return "", "", fmt.Errorf("read %s: %w", secrets, err)
		}
		user, pass := os.Getenv("FELIS_REGISTRY_USERNAME"), os.Getenv("FELIS_REGISTRY_PASSWORD")
		if pass == "" {
		user, pass = registrygate.PrincipalPlatform, os.Getenv("REGISTRY_PLATFORM_TOKEN")
		}
		if pass == "" {
		return "", "", errors.New("no registry credential: set FELIS_REGISTRY_PASSWORD or run as root on the node (REGISTRY_PLATFORM_TOKEN in /etc/felis/secrets.env)")
		}
		return user, pass, nil
		}

		func offsiteFetchImages(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
		cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml (the host copy)")
		envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
		registry := fs.String("registry", "", "host[:port] of the registry to push into (default: the loopback hostPort of the in-cluster registry)")
		secrets := fs.String("secrets-env", "/etc/felis/secrets.env", "installer secrets file holding REGISTRY_PLATFORM_TOKEN, read when FELIS_REGISTRY_PASSWORD is unset")
		at := fs.String("at", "", "registry index version to restore (default: the newest; `felis offsite list` shows them)")
		if err := fs.Parse(args); err != nil {
		return 2
		}
		cfg, env, err := loadOffsite(cfgPath, envFile)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
		return 1
		}
		endpoint := offsiteRegistryEndpoint(*registry, cfg.Registry)
		if endpoint == "" {
		fmt.Fprintf(stderr, "felis offsite fetch-images: [registry] url %q is not the in-cluster registry; pass -registry host:port\n", cfg.Registry.URL)
		return 2
		}
		user, pass, err := registryWriteCredential(*secrets)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
		return 2
		}
		ctx, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
		defer cancel()
		stamp, idx, err := offsite.ChooseImageIndex(ctx, env.bucket, env.key, *at)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-images: %v\n", err)
		return 1
		}
		fmt.Fprintf(stdout, "felis offsite fetch-images: restoring registry index %s (%d repositories, %d images) into %s\n",
		stamp, len(idx.Repositories), idx.Images(), endpoint)
		target := newRegistryImages(endpoint)
		target.pusher = &imagepush.Pusher{Scheme: "http", Username: user, Password: pass}
		res, err := offsite.FetchImages(ctx, env.bucket, env.key, idx, target, stderr)
		fmt.Fprintf(stdout, "felis offsite fetch-images: %d of %d repositories restored, %d images, %d tags, %d blobs pushed (%s)\n",
		res.Repositories, len(idx.Repositories), res.Manifests, res.Tags, res.BlobsPushed, offsite.HumanBytes(res.BytesPushed))
		for _, f := range res.Failures {
		fmt.Fprintf(stderr, "felis offsite fetch-images: %s\n", f)
		}
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite fetch-images: %v (a second run pushes only what is still missing)\n", err)
		return 1
		}
		return 0
		}

cmd/felis/offsite_test.go

+26 −0

Changes for cmd/felis/offsite_test.go: 26 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -2,12 +2,14 @@ package main

		import (
		"bytes"
		"errors"
		"os"
		"path/filepath"
		"strings"
		"testing"

		"felis.lolicon.best/internal/config"
		"felis.lolicon.best/internal/imagepush"
		"felis.lolicon.best/internal/offsite"
		)

		@@ -94,3 +96,27 @@ func TestOffsiteFetchDBRejectsOddNames(t *testing.T) {
		t.Fatalf("exit %d: %s", code, errb.String())
		}
		}

		func TestOffsiteRegistryEndpoint(t *testing.T) {
		for _, tc := range []struct{ flag, url, want string }{
		{"", "registry.felis.svc:5000", "127.0.0.1:5000"},
		{"", "registry.felis.svc.cluster.local:5001", "127.0.0.1:5001"},
		{"", "ghcr.io/acme", ""},
		{"", "", ""},
		{"off", "registry.felis.svc:5000", ""},
		{"10.0.0.5:5000", "ghcr.io/acme", "10.0.0.5:5000"},
		} {
		if got := offsiteRegistryEndpoint(tc.flag, config.RegistryConfig{URL: tc.url}); got != tc.want {
		t.Errorf("offsiteRegistryEndpoint(%q, %q) = %q, want %q", tc.flag, tc.url, got, tc.want)
		}
		}
		}

		func TestRegistryGoneMarksNotFound(t *testing.T) {
		if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 404}); !errors.Is(err, offsite.ErrImageGone) {
		t.Fatalf("404 = %v, want ErrImageGone", err)
		}
		if err := registryGone(&imagepush.StatusError{Op: "get blob", Code: 503}); errors.Is(err, offsite.ErrImageGone) {
		t.Fatalf("503 = %v, want it kept an ordinary failure", err)
		}
		}

cmd/felis/run.go

+1 −1

Changes for cmd/felis/run.go: 1 added line, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -13,7 +13,7 @@ Usage:
		Commands:
		migrate up Apply embedded database migrations under an advisory lock (snapshots the database first)
		db Back up, verify, list and restore the control-plane database (backup\|restore\|verify\|list\|check)
		offsite Copy world archives and database bundles to an off-site bucket, and fetch them back (sync\|status\|list\|fetch-db\|fetch-worlds\|keygen)
		offsite Copy world archives, database bundles and user images to an off-site bucket, and fetch them back (sync\|status\|list\|fetch-db\|fetch-worlds\|fetch-images\|keygen)
		operator Run the MinecraftServer controller-manager
		api Run the felis-api HTTP server
		nano Run the Felis-nano hasJoined multiplexer (multi-Yggdrasil, no control plane)