Unverified Commit 151c9d2e authored by Lemon-miaow's avatar Lemon-miaow
Browse files

feat(registry): api 定期删除无引用 manifest，gate 提供 manifest 索引

parent 05e8c64e

cmd/felis/api.go

+80 −0
Changes for cmd/felis/api.go: 80 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ import ( 
	"flag"

	"fmt"

	"io"

	"log/slog"

	"net/http"

	"os"

	"regexp"
@@ -26,6 +27,7 @@ import ( 
	"felis.lolicon.best/internal/passkey"

	"felis.lolicon.best/internal/platform"

	"felis.lolicon.best/internal/reaper"

	"felis.lolicon.best/internal/registryprune"

	"felis.lolicon.best/internal/restore"

	"felis.lolicon.best/internal/store"

	"felis.lolicon.best/internal/submit"
@@ -399,6 +401,10 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 
	// reconciles it, but this loop converges builds nobody is polling.

	go reconcileBuilds(ctx, builder, stderr)



	if pruner := registryPruner(cfg, builder.Store, a.Cluster, stderr); pruner != nil {

		go pruner.Loop(ctx, registryPruneInterval)

	}



	select {

	case <-ctx.Done():

		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -613,6 +619,80 @@ func reconcileBuilds(ctx context.Context, b *build.Builder, stderr io.Writer) { 
	}

}



// registryPruneInterval spaces the registry pruner's runs. The registry-gc

// sidecar sweeps once a day, so pruning more often only changes which sweep frees

// a layer.

const registryPruneInterval = 6 * time.Hour



// registryPruner deletes the registry manifests nothing references

// (internal/registryprune); the registry-gc sidecar frees their layers on its next

// sweep. It acts as the gate's prune principal, whose token the api Deployment

// injects from felis-registry-auth. Without the token the registry only grows,

// which is said once here.

func registryPruner(cfg *config.Config, store imageRefStore, servers serverLister, stderr io.Writer) *registryprune.Pruner {

	if cfg.Registry.URL == "" {

		return nil

	}

	token := os.Getenv(platform.RegistryPruneTokenEnv)

	if token == "" {

		fmt.Fprintf(stderr, "felis api: registry pruner disabled (%s unset) — images nothing uses are never deleted from the registry\n", platform.RegistryPruneTokenEnv)

		return nil

	}

	static := []string{

		os.Getenv("FELIS_IMAGE"),

		cfg.Registry.KanikoImage, cfg.Registry.TrivyImage,

		cfg.Registry.TrivyDBRepository, cfg.Registry.TrivyJavaDBRepository,

	}

	return &registryprune.Pruner{

		Registry: &registryprune.Client{Endpoint: "http://" + cfg.Registry.URL, Token: token},

		Host:     cfg.Registry.URL,

		Refs: func(ctx context.Context) ([]string, error) {

			return inUseImageRefs(ctx, store, servers, static)

		},

		Log: slog.New(slog.NewTextHandler(stderr, nil)),

	}

}



type imageRefStore interface {

	ListImages(ctx context.Context) ([]build.Image, error)

	ListUnfinishedBuilds(ctx context.Context) ([]build.Build, error)

}



type serverLister interface {

	ListServers(ctx context.Context) ([]api.ServerInfo, error)

}



// inUseImageRefs lists every image reference the platform still depends on: the

// whitelist (disabled rows too, an admin may enable them again), every server's

// spec, builds still running, and the images the control plane and the build

// Jobs run. Any source failing fails the whole list, so the pruner never decides

// on a partial view.

func inUseImageRefs(ctx context.Context, store imageRefStore, servers serverLister, static []string) ([]string, error) {

	refs := append([]string(nil), static...)

	images, err := store.ListImages(ctx)

	if err != nil {

		return nil, fmt.Errorf("image whitelist: %w", err)

	}

	for _, img := range images {

		refs = append(refs, img.ImageRef)

	}

	srvs, err := servers.ListServers(ctx)

	if err != nil {

		return nil, fmt.Errorf("servers: %w", err)

	}

	for _, s := range srvs {

		refs = append(refs, s.Image)

	}

	builds, err := store.ListUnfinishedBuilds(ctx)

	if err != nil {

		return nil, fmt.Errorf("running builds: %w", err)

	}

	for _, b := range builds {

		refs = append(refs, b.ImageRef)

	}

	return refs, nil

}



// mailLimit turns smtp.max_per_hour into the API's install-wide mail bucket:

// the hourly cap as the refill rate, with a quarter of it (at least 5) allowed

// at once so a burst of real sign-ins is not queued behind the average.

cmd/felis/api_test.go

+49 −0
Changes for cmd/felis/api_test.go: 49 added lines, 0 removed lines.
Original line number Diff line number Diff line 
package main



import (

	"context"

	"errors"

	"fmt"

	"net/http"

	"testing"



	"felis.lolicon.best/internal/api"

	"felis.lolicon.best/internal/build"

	"felis.lolicon.best/internal/config"

)
@@ -91,3 +96,47 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) { 
		t.Errorf("ReadTimeout = %v, want 0 (unset) so a slow SSE attach is not capped", srv.ReadTimeout)

	}

}



type fakeRefStore struct {

	images []build.Image

	builds []build.Build

	err    error

}



func (f fakeRefStore) ListImages(context.Context) ([]build.Image, error) { return f.images, f.err }

func (f fakeRefStore) ListUnfinishedBuilds(context.Context) ([]build.Build, error) {

	return f.builds, nil

}



type fakeServers []api.ServerInfo



func (f fakeServers) ListServers(context.Context) ([]api.ServerInfo, error) { return f, nil }



// The registry pruner deletes whatever this list does not name, so every source of

// a reference has to be in it, and a failing source must fail the list.

func TestInUseImageRefsCoversEverySource(t *testing.T) {

	const reg = "registry.felis.svc:5000/"

	store := fakeRefStore{

		images: []build.Image{{ImageRef: reg + "modpacks/pack:*"}, {ImageRef: reg + "felis/paper:demo"}},

		builds: []build.Build{{ImageRef: reg + "user-uploads/sub-9:latest"}},

	}

	servers := fakeServers{{Name: "s1", Image: reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1)}}

	got, err := inUseImageRefs(context.Background(), store, servers, []string{reg + "felis/felis:b60"})

	if err != nil {

		t.Fatal(err)

	}

	want := []string{

		reg + "felis/felis:b60",

		reg + "modpacks/pack:*", reg + "felis/paper:demo",

		reg + "felis/paper:demo@sha256:" + fmt.Sprintf("%064d", 1),

		reg + "user-uploads/sub-9:latest",

	}

	if fmt.Sprint(got) != fmt.Sprint(want) {

		t.Fatalf("refs = %v\nwant %v", got, want)

	}



	store.err = errors.New("db down")

	if _, err := inUseImageRefs(context.Background(), store, servers, nil); err == nil {

		t.Fatal("a failing whitelist read produced a reference list")

	}

}

cmd/felis/registrygate.go

+2 −0
Changes for cmd/felis/registrygate.go: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ func cmdRegistryGate(args []string, _, stderr io.Writer) int { 
	maintListen := fs.String("maint-listen", "", "loopback address for the GC sidecar's read-only handshake (empty disables it)")

	maintDir := fs.String("maint-dir", "", "directory that keeps an open read-only window across a gate restart")

	quiet := fs.Duration("maint-quiet", registrygate.DefaultQuiet, "how long writes must be idle before a read-only window is granted")

	dataDir := fs.String("data-dir", "", "the registry's storage root, mounted read-only, for the manifest index (empty disables it)")

	if err := fs.Parse(args); err != nil {

		return 2

	}
@@ -70,6 +71,7 @@ func cmdRegistryGate(args []string, _, stderr io.Writer) int { 


	gate := registrygate.New(target, tokens, log)

	gate.SetQuiet(*quiet)

	gate.DataDir = *dataDir

	if *maintDir != "" {

		if err := gate.SetMaintenanceState(registrygate.MaintStatePath(*maintDir)); err != nil {

			// A corrupt file must not keep the registry from serving pulls.

deploy/bootstrap.sh

+6 −1
Changes for deploy/bootstrap.sh: 6 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -411,10 +411,12 @@ apply_registry_secrets() { 
  remember_temp "$dir"

  printf '%s' "$REGISTRY_PLATFORM_TOKEN" > "${dir}/platform"

  printf '%s' "$REGISTRY_BUILD_TOKEN" > "${dir}/build"

  printf '%s' "$REGISTRY_PRUNE_TOKEN" > "${dir}/prune"

  printf '%s' build > "${dir}/username"

  kube -n "$CONTROL_NS" create secret generic felis-registry-auth \

    --from-file=platform="${dir}/platform" \

    --from-file=build="${dir}/build" \

    --from-file=prune="${dir}/prune" \

    --dry-run=client -o yaml | kube apply -f -

  kube -n "$BUILD_NS" create secret generic felis-registry-push \

    --from-file=username="${dir}/username" \
@@ -2210,9 +2212,11 @@ load_or_make_secrets() { 
  # Registry write credentials, one per principal the registry gate knows

  # (internal/registrygate): platform pushes the installer's own images and the

  # Trivy DB mirrors, build is what a build Job's push container presents and may

  # never write under felis/ or mirror/. Reads stay anonymous.

  # never write under felis/ or mirror/, prune is felis-api deleting manifests

  # nothing references (internal/registryprune). Reads stay anonymous.

  REGISTRY_PLATFORM_TOKEN="${REGISTRY_PLATFORM_TOKEN:-$(openssl rand -hex 32)}"

  REGISTRY_BUILD_TOKEN="${REGISTRY_BUILD_TOKEN:-$(openssl rand -hex 32)}"

  REGISTRY_PRUNE_TOKEN="${REGISTRY_PRUNE_TOKEN:-$(openssl rand -hex 32)}"

  (

    umask 077

    cat > "$SECRETS_ENV" <<EOF
@@ -2222,6 +2226,7 @@ SESSION_SECRET=${SESSION_SECRET} 
FORWARDING_SECRET=${FORWARDING_SECRET}

REGISTRY_PLATFORM_TOKEN=${REGISTRY_PLATFORM_TOKEN}

REGISTRY_BUILD_TOKEN=${REGISTRY_BUILD_TOKEN}

REGISTRY_PRUNE_TOKEN=${REGISTRY_PRUNE_TOKEN}

EOF

  )

  chmod 0600 "$SECRETS_ENV"

internal/naming/naming.go

+2 −2
Changes for internal/naming/naming.go: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -72,8 +72,8 @@ const ( 
)



// Registry write credentials (internal/registrygate). The registry namespace holds

// RegistryAuthSecretName with one key per principal (platform, build), mounted into

// the gate sidecar. The build namespace holds RegistryPushSecretName with the build

// RegistryAuthSecretName with one key per principal (platform, build, prune),

// mounted into the gate sidecar; felis-api reads the prune key into env. The build namespace holds RegistryPushSecretName with the build

// principal's username/password, read only by a build Job's push container. Both

// are provisioned out-of-band by deploy/bootstrap.sh.

const (