Loading cmd/felis/offsite.go +1 −0 Changes for cmd/felis/offsite.go: 1 added line, 0 removed lines. Original line number Diff line number Diff line Loading @@ -294,6 +294,7 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log } if src.registry != "" { s.Images = newRegistryImages(src.registry) s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL) } return s.Run(ctx) } Loading cmd/felis/offsite_images.go +46 −0 Changes for cmd/felis/offsite_images.go: 46 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -2,15 +2,18 @@ package main import ( "context" "database/sql" "errors" "flag" "fmt" "io" "net/http" "os" "slices" "strings" "time" "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/config" "felis.lolicon.best/internal/imagepush" "felis.lolicon.best/internal/offsite" Loading Loading @@ -72,6 +75,49 @@ func (r *registryImages) PutManifest(ctx context.Context, repo, reference, media return err } // imagePins lists, per repository of the registry refs spell as host, the // digests the MinecraftServers' specs and the image whitelist pin: what a // restored database and its servers will ask the registry for. func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) { return func(ctx context.Context) (map[string][]string, error) { cl, err := buildSystemServerClient() if err != nil { return nil, fmt.Errorf("reach the cluster: %w", err) } var servers v1alpha1.MinecraftServerList if err := cl.List(ctx, &servers); err != nil { return nil, fmt.Errorf("list MinecraftServers: %w", err) } refs := make([]string, 0, len(servers.Items)) for _, s := range servers.Items { refs = append(refs, s.Spec.Image) } rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`) if err != nil { return nil, fmt.Errorf("read the image whitelist: %w", err) } defer rows.Close() for rows.Next() { var ref string if err := rows.Scan(&ref); err != nil { return nil, fmt.Errorf("read the image whitelist: %w", err) } refs = append(refs, ref) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("read the image whitelist: %w", err) } pins := map[string][]string{} for _, ref := range refs { repo, _, digest, ok := registryprune.ParseRef(ref, host) if ok && digest != "" && !slices.Contains(pins[repo], digest) { pins[repo] = append(pins[repo], digest) } } return pins, nil } } // registryGone marks a 404 as a manifest or blob the registry no longer holds. func registryGone(err error) error { var se *imagepush.StatusError Loading docs/troubleshooting.md +8 −3 Changes for docs/troubleshooting.md: 8 added lines, 3 removed lines. Original line number Diff line number Diff line Loading @@ -1792,9 +1792,14 @@ What runs: the right size is recorded without being sent again, so a run cut short resumes. [GO-TESTED: `internal/offsite`] - The same run copies the user images in the platform registry: every repository outside `felis/` and `mirror/` (the installer pushes those again), each manifest the registry's index lists and every layer it names, read through the loopback hostPort. A layer shared by many images is stored once. repository outside `felis/` and `mirror/`, each manifest the registry's index lists and every layer it names, read through the loopback hostPort. A layer shared by many images is stored once. The installer pushes `felis/` and `mirror/` again on a new host, but at new digests, so from those the run copies only the revisions a MinecraftServer or a whitelist entry pins by digest (a server created from the platform's Paper image, for one), without their tags; a restore puts them back by digest and leaves the installer's tags alone. When the set changed, a new version of the image list is written; versions replaced more than 14 days ago are dropped together with the layers only they named, so an image deleted by mistake stays restorable for two weeks Loading internal/offsite/images.go +32 −12 Changes for internal/offsite/images.go: 32 added lines, 12 removed lines. Original line number Diff line number Diff line package offsite // The off-site copy of the platform registry's user images. The installer // pushes everything under felis/ and mirror/ again on any host, so those are // left out; every other repository holds builds that exist nowhere else: a // lost registry volume would otherwise leave each server pinned to one of them // in ImagePullBackOff until someone rebuilds it. // The off-site copy of the platform registry's user images. Every repository // outside felis/ and mirror/ holds builds that exist nowhere else: a lost // registry volume would otherwise leave each server pinned to one of them in // ImagePullBackOff until someone rebuilds it. The installer pushes felis/ and // mirror/ again on any host, but under the same tags at new digests (a build // is not reproducible, an upstream tag moves on), so from those only the // revisions a server or a whitelist entry pins by digest are copied, without // their tags: the tags belong to whatever the installer pushed last. // // The bucket holds each blob and manifest once, by digest, encrypted like // everything else, plus an index that says which repository holds which Loading Loading @@ -234,19 +237,28 @@ func (s *Syncer) syncImages(ctx context.Context, res *Result, fail func(string, return } sort.Strings(repos) listed := map[string]bool{} pins, pinsKnown := map[string][]string{}, true if s.ImagePins != nil { if pins, err = s.ImagePins(ctx); err != nil { fail("list the images servers and the whitelist pin: %v", err) pinsKnown, c.failed = false, true } } for _, repo := range repos { if reservedRepo(repo) { if reservedRepo(repo) && !pinsKnown { c.carry(repo) continue } if reservedRepo(repo) && len(pins[repo]) == 0 { continue } listed[repo] = true if ctx.Err() != nil { c.fail("stopped before %s: %v", repo, ctx.Err()) c.failed = true c.carry(repo) continue } c.repo(ctx, repo) c.repo(ctx, repo, pins[repo]) } stamp := "" Loading Loading @@ -277,9 +289,10 @@ func (s *Syncer) syncImages(ctx context.Context, res *Result, fail func(string, } } // repo copies one repository into c.next. A repository that cannot be read in // full keeps the entry the previous index had for it. func (c *imageCopy) repo(ctx context.Context, repo string) { // repo copies one repository into c.next; of a reserved one, only the pinned // revisions and none of its tags. A repository that cannot be read in full // keeps the entry the previous index had for it. func (c *imageCopy) repo(ctx context.Context, repo string, pinned []string) { digests, tags, err := c.s.Images.Revisions(ctx, repo) if err != nil { c.fail("list the images of %s: %v", repo, err) Loading @@ -287,6 +300,10 @@ func (c *imageCopy) repo(ctx context.Context, repo string) { c.carry(repo) return } if reservedRepo(repo) { digests = slices.DeleteFunc(digests, func(d string) bool { return !slices.Contains(pinned, d) }) tags = nil } entry := ImageRepo{Tags: map[string]string{}} short := false for _, d := range digests { Loading Loading @@ -732,6 +749,9 @@ func FetchImages(ctx context.Context, b Bucket, key []byte, x *ImageIndex, t Ima pushedBlobs := map[string]bool{} for _, repo := range slices.Sorted(maps.Keys(x.Repositories)) { entry := x.Repositories[repo] if reservedRepo(repo) { entry.Tags = nil // the installer's tags stay where it put them } done := map[string]bool{} var push func(d string) error push = func(d string) error { Loading internal/offsite/images_test.go +62 −0 Changes for internal/offsite/images_test.go: 62 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -280,6 +280,68 @@ func TestSyncImagesCopiesUserImages(t *testing.T) { } } // TestSyncImagesCopiesPinnedPlatformRevisions: of felis/ and mirror/ only the // revisions a pin names are copied, without tags; a restore puts them back by // digest and leaves the tags the installer pushed on the new host alone. When // the pins cannot be read, the previous copy of those revisions is kept and // the run fails. func TestSyncImagesCopiesPinnedPlatformRevisions(t *testing.T) { reg := newFakeRegistry() old := reg.image("felis/paper", "demo", layer(4000)) current := reg.image("felis/paper", "demo", layer(4000)) reg.image("mirror/trivy", "1", layer(2000)) user := reg.image("user/a", "v1", layer(1000)) s, b, clock := newImageSyncer(t, reg) s.ImagePins = func(context.Context) (map[string][]string, error) { return map[string][]string{"felis/paper": {old}, "user/a": {user}}, nil } res, err := s.Run(context.Background()) if err != nil { t.Fatalf("Run: %v", err) } x, err := LoadImageIndex(context.Background(), b, s.Key, res.ImageIndex) if err != nil { t.Fatal(err) } paper, ok := x.Repositories["felis/paper"] if !ok || !slices.Equal(paper.Manifests, []string{old}) || len(paper.Tags) != 0 { t.Fatalf("felis/paper = %+v, want only the pinned %s and no tags", paper, old) } if _, ok := x.Repositories["mirror/trivy"]; ok { t.Fatal("unpinned mirror/trivy was copied") } if got := x.Repositories["user/a"]; got.Tags["v1"] != user { t.Fatalf("user/a = %+v", got) } fresh := newFakeRegistry() rebuilt := fresh.image("felis/paper", "demo", layer(4000)) if _, err := FetchImages(context.Background(), b, s.Key, x, fresh, nil); err != nil { t.Fatalf("FetchImages: %v", err) } if got := fresh.repos["felis/paper"]; got.tags["demo"] != rebuilt || !slices.Contains(got.digests, old) { t.Fatalf("restored felis/paper: tags %v digests %v, want demo=%s and %s present", got.tags, got.digests, rebuilt, old) } if slices.Contains(fresh.repos["felis/paper"].digests, current) { t.Fatal("the unpinned revision was restored") } *clock = clock.Add(time.Hour) s.ImagePins = func(context.Context) (map[string][]string, error) { return nil, errors.New("cluster down") } res, err = s.Run(context.Background()) if err == nil { t.Fatal("Run succeeded without the pins") } x, err = LoadImageIndex(context.Background(), b, s.Key, res.ImageIndex) if err != nil { t.Fatal(err) } if got := x.Repositories["felis/paper"]; !slices.Equal(got.Manifests, []string{old}) { t.Fatalf("without the pins felis/paper = %+v, want the previous copy kept", got) } } // TestSyncImagesRejectsCorruptBlob: bytes that do not hash to the digest never // become that digest's object, the image stays out of the index, and the run // fails so the next one tries again. Loading Loading
cmd/felis/offsite.go +1 −0 Changes for cmd/felis/offsite.go: 1 added line, 0 removed lines. Original line number Diff line number Diff line Loading @@ -294,6 +294,7 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log } if src.registry != "" { s.Images = newRegistryImages(src.registry) s.ImagePins = imagePins(drv.DB(), cfg.Registry.URL) } return s.Run(ctx) } Loading
cmd/felis/offsite_images.go +46 −0 Changes for cmd/felis/offsite_images.go: 46 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -2,15 +2,18 @@ package main import ( "context" "database/sql" "errors" "flag" "fmt" "io" "net/http" "os" "slices" "strings" "time" "felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/config" "felis.lolicon.best/internal/imagepush" "felis.lolicon.best/internal/offsite" Loading Loading @@ -72,6 +75,49 @@ func (r *registryImages) PutManifest(ctx context.Context, repo, reference, media return err } // imagePins lists, per repository of the registry refs spell as host, the // digests the MinecraftServers' specs and the image whitelist pin: what a // restored database and its servers will ask the registry for. func imagePins(db *sql.DB, host string) func(ctx context.Context) (map[string][]string, error) { return func(ctx context.Context) (map[string][]string, error) { cl, err := buildSystemServerClient() if err != nil { return nil, fmt.Errorf("reach the cluster: %w", err) } var servers v1alpha1.MinecraftServerList if err := cl.List(ctx, &servers); err != nil { return nil, fmt.Errorf("list MinecraftServers: %w", err) } refs := make([]string, 0, len(servers.Items)) for _, s := range servers.Items { refs = append(refs, s.Spec.Image) } rows, err := db.QueryContext(ctx, `SELECT image_ref FROM image_whitelist`) if err != nil { return nil, fmt.Errorf("read the image whitelist: %w", err) } defer rows.Close() for rows.Next() { var ref string if err := rows.Scan(&ref); err != nil { return nil, fmt.Errorf("read the image whitelist: %w", err) } refs = append(refs, ref) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("read the image whitelist: %w", err) } pins := map[string][]string{} for _, ref := range refs { repo, _, digest, ok := registryprune.ParseRef(ref, host) if ok && digest != "" && !slices.Contains(pins[repo], digest) { pins[repo] = append(pins[repo], digest) } } return pins, nil } } // registryGone marks a 404 as a manifest or blob the registry no longer holds. func registryGone(err error) error { var se *imagepush.StatusError Loading
docs/troubleshooting.md +8 −3 Changes for docs/troubleshooting.md: 8 added lines, 3 removed lines. Original line number Diff line number Diff line Loading @@ -1792,9 +1792,14 @@ What runs: the right size is recorded without being sent again, so a run cut short resumes. [GO-TESTED: `internal/offsite`] - The same run copies the user images in the platform registry: every repository outside `felis/` and `mirror/` (the installer pushes those again), each manifest the registry's index lists and every layer it names, read through the loopback hostPort. A layer shared by many images is stored once. repository outside `felis/` and `mirror/`, each manifest the registry's index lists and every layer it names, read through the loopback hostPort. A layer shared by many images is stored once. The installer pushes `felis/` and `mirror/` again on a new host, but at new digests, so from those the run copies only the revisions a MinecraftServer or a whitelist entry pins by digest (a server created from the platform's Paper image, for one), without their tags; a restore puts them back by digest and leaves the installer's tags alone. When the set changed, a new version of the image list is written; versions replaced more than 14 days ago are dropped together with the layers only they named, so an image deleted by mistake stays restorable for two weeks Loading
internal/offsite/images.go +32 −12 Changes for internal/offsite/images.go: 32 added lines, 12 removed lines. Original line number Diff line number Diff line package offsite // The off-site copy of the platform registry's user images. The installer // pushes everything under felis/ and mirror/ again on any host, so those are // left out; every other repository holds builds that exist nowhere else: a // lost registry volume would otherwise leave each server pinned to one of them // in ImagePullBackOff until someone rebuilds it. // The off-site copy of the platform registry's user images. Every repository // outside felis/ and mirror/ holds builds that exist nowhere else: a lost // registry volume would otherwise leave each server pinned to one of them in // ImagePullBackOff until someone rebuilds it. The installer pushes felis/ and // mirror/ again on any host, but under the same tags at new digests (a build // is not reproducible, an upstream tag moves on), so from those only the // revisions a server or a whitelist entry pins by digest are copied, without // their tags: the tags belong to whatever the installer pushed last. // // The bucket holds each blob and manifest once, by digest, encrypted like // everything else, plus an index that says which repository holds which Loading Loading @@ -234,19 +237,28 @@ func (s *Syncer) syncImages(ctx context.Context, res *Result, fail func(string, return } sort.Strings(repos) listed := map[string]bool{} pins, pinsKnown := map[string][]string{}, true if s.ImagePins != nil { if pins, err = s.ImagePins(ctx); err != nil { fail("list the images servers and the whitelist pin: %v", err) pinsKnown, c.failed = false, true } } for _, repo := range repos { if reservedRepo(repo) { if reservedRepo(repo) && !pinsKnown { c.carry(repo) continue } if reservedRepo(repo) && len(pins[repo]) == 0 { continue } listed[repo] = true if ctx.Err() != nil { c.fail("stopped before %s: %v", repo, ctx.Err()) c.failed = true c.carry(repo) continue } c.repo(ctx, repo) c.repo(ctx, repo, pins[repo]) } stamp := "" Loading Loading @@ -277,9 +289,10 @@ func (s *Syncer) syncImages(ctx context.Context, res *Result, fail func(string, } } // repo copies one repository into c.next. A repository that cannot be read in // full keeps the entry the previous index had for it. func (c *imageCopy) repo(ctx context.Context, repo string) { // repo copies one repository into c.next; of a reserved one, only the pinned // revisions and none of its tags. A repository that cannot be read in full // keeps the entry the previous index had for it. func (c *imageCopy) repo(ctx context.Context, repo string, pinned []string) { digests, tags, err := c.s.Images.Revisions(ctx, repo) if err != nil { c.fail("list the images of %s: %v", repo, err) Loading @@ -287,6 +300,10 @@ func (c *imageCopy) repo(ctx context.Context, repo string) { c.carry(repo) return } if reservedRepo(repo) { digests = slices.DeleteFunc(digests, func(d string) bool { return !slices.Contains(pinned, d) }) tags = nil } entry := ImageRepo{Tags: map[string]string{}} short := false for _, d := range digests { Loading Loading @@ -732,6 +749,9 @@ func FetchImages(ctx context.Context, b Bucket, key []byte, x *ImageIndex, t Ima pushedBlobs := map[string]bool{} for _, repo := range slices.Sorted(maps.Keys(x.Repositories)) { entry := x.Repositories[repo] if reservedRepo(repo) { entry.Tags = nil // the installer's tags stay where it put them } done := map[string]bool{} var push func(d string) error push = func(d string) error { Loading
internal/offsite/images_test.go +62 −0 Changes for internal/offsite/images_test.go: 62 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -280,6 +280,68 @@ func TestSyncImagesCopiesUserImages(t *testing.T) { } } // TestSyncImagesCopiesPinnedPlatformRevisions: of felis/ and mirror/ only the // revisions a pin names are copied, without tags; a restore puts them back by // digest and leaves the tags the installer pushed on the new host alone. When // the pins cannot be read, the previous copy of those revisions is kept and // the run fails. func TestSyncImagesCopiesPinnedPlatformRevisions(t *testing.T) { reg := newFakeRegistry() old := reg.image("felis/paper", "demo", layer(4000)) current := reg.image("felis/paper", "demo", layer(4000)) reg.image("mirror/trivy", "1", layer(2000)) user := reg.image("user/a", "v1", layer(1000)) s, b, clock := newImageSyncer(t, reg) s.ImagePins = func(context.Context) (map[string][]string, error) { return map[string][]string{"felis/paper": {old}, "user/a": {user}}, nil } res, err := s.Run(context.Background()) if err != nil { t.Fatalf("Run: %v", err) } x, err := LoadImageIndex(context.Background(), b, s.Key, res.ImageIndex) if err != nil { t.Fatal(err) } paper, ok := x.Repositories["felis/paper"] if !ok || !slices.Equal(paper.Manifests, []string{old}) || len(paper.Tags) != 0 { t.Fatalf("felis/paper = %+v, want only the pinned %s and no tags", paper, old) } if _, ok := x.Repositories["mirror/trivy"]; ok { t.Fatal("unpinned mirror/trivy was copied") } if got := x.Repositories["user/a"]; got.Tags["v1"] != user { t.Fatalf("user/a = %+v", got) } fresh := newFakeRegistry() rebuilt := fresh.image("felis/paper", "demo", layer(4000)) if _, err := FetchImages(context.Background(), b, s.Key, x, fresh, nil); err != nil { t.Fatalf("FetchImages: %v", err) } if got := fresh.repos["felis/paper"]; got.tags["demo"] != rebuilt || !slices.Contains(got.digests, old) { t.Fatalf("restored felis/paper: tags %v digests %v, want demo=%s and %s present", got.tags, got.digests, rebuilt, old) } if slices.Contains(fresh.repos["felis/paper"].digests, current) { t.Fatal("the unpinned revision was restored") } *clock = clock.Add(time.Hour) s.ImagePins = func(context.Context) (map[string][]string, error) { return nil, errors.New("cluster down") } res, err = s.Run(context.Background()) if err == nil { t.Fatal("Run succeeded without the pins") } x, err = LoadImageIndex(context.Background(), b, s.Key, res.ImageIndex) if err != nil { t.Fatal(err) } if got := x.Repositories["felis/paper"]; !slices.Equal(got.Manifests, []string{old}) { t.Fatalf("without the pins felis/paper = %+v, want the previous copy kept", got) } } // TestSyncImagesRejectsCorruptBlob: bytes that do not hash to the digest never // become that digest's object, the image stays out of the index, and the run // fails so the next one tries again. Loading