diff --git a/cmd/felis/imagebundle.go b/cmd/felis/imagebundle.go new file mode 100644 index 0000000..7cab363 --- /dev/null +++ b/cmd/felis/imagebundle.go @@ -0,0 +1,109 @@ +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + + "felis.lolicon.best/internal/imagepush" +) + +// cmdImageBundle writes a release's image bundle: one OCI layout tar holding every +// image an install runs, for one platform, plus its listing (one "role name +// manifest-digest config-digest" line per image). deploy/build-release-artifacts.sh +// runs it in CI; deploy/bootstrap.sh imports the tar into k3s's containerd and +// pushes it into the platform registry with push-image --image. +// +// --layout role=name=path an image buildx wrote with --output type=oci +// --pull role=ref a digest-pinned public image, named repository@digest +// +// The tar and the listing are written beside their final paths and renamed into +// place, so a failed run leaves neither behind. +func cmdImageBundle(args []string, _, stderr io.Writer) int { + fs := flag.NewFlagSet("image-bundle", flag.ContinueOnError) + fs.SetOutput(stderr) + platform := fs.String("platform", "", "os/arch the bundle is for, e.g. linux/arm64") + out := fs.String("out", "", "path of the bundle tar to write") + list := fs.String("list", "", "path of the listing to write") + var images []imagepush.BundleImage + fs.Func("layout", "role=name=path of an OCI layout tar (repeatable)", func(v string) error { + role, rest, ok := strings.Cut(v, "=") + name, path, ok2 := strings.Cut(rest, "=") + if !ok || !ok2 || role == "" || name == "" || path == "" { + return fmt.Errorf("want role=name=path, got %q", v) + } + images = append(images, imagepush.BundleImage{Role: role, Name: name, Layout: path}) + return nil + }) + fs.Func("pull", "role=ref of a digest-pinned public image (repeatable)", func(v string) error { + role, ref, ok := strings.Cut(v, "=") + if !ok || role == "" || !strings.Contains(ref, "@sha256:") { + return fmt.Errorf("want role=ref with ref pinned by digest, got %q", v) + } + images = append(images, imagepush.BundleImage{Role: role, Name: imagepush.PinnedName(ref), Source: ref}) + return nil + }) + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if *platform == "" || *out == "" || *list == "" || len(images) == 0 { + fmt.Fprintln(stderr, "felis image-bundle: --platform, --out, --list and at least one --layout or --pull are required") + return 2 + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := writeImageBundle(ctx, &imagepush.Source{Platform: *platform}, images, *out, *list); err != nil { + fmt.Fprintf(stderr, "felis image-bundle: %v\n", err) + return 1 + } + return 0 +} + +func writeImageBundle(ctx context.Context, s *imagepush.Source, images []imagepush.BundleImage, out, list string) (err error) { + tmpOut, tmpList := out+".tmp", list+".tmp" + defer func() { + if err != nil { + os.Remove(tmpOut) + os.Remove(tmpList) + } + }() + f, err := os.Create(tmpOut) + if err != nil { + return err + } + w := bufio.NewWriterSize(f, 1<<20) + entries, err := imagepush.WriteBundle(ctx, s, images, w) + if err == nil { + err = w.Flush() + } + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + var b strings.Builder + for _, e := range entries { + fmt.Fprintf(&b, "%s %s %s %s\n", e.Role, e.Name, e.Digest, e.Config) + } + if err := os.WriteFile(tmpList, []byte(b.String()), 0o644); err != nil { + return err + } + if err := os.Rename(tmpOut, out); err != nil { + return err + } + return os.Rename(tmpList, list) +} diff --git a/cmd/felis/imagebundle_test.go b/cmd/felis/imagebundle_test.go new file mode 100644 index 0000000..d1b7441 --- /dev/null +++ b/cmd/felis/imagebundle_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTestLayout writes an OCI layout tar holding one linux/arch image with one +// layer, the way buildx --output type=oci leaves a single-platform build, and +// returns its path with the manifest and config digests. +func writeTestLayout(t *testing.T, dir, arch, layer string) (path, manifestDigest, configDigest string) { + t.Helper() + blobs := map[string][]byte{} + add := func(b []byte) (string, int) { + sum := sha256.Sum256(b) + d := "sha256:" + hex.EncodeToString(sum[:]) + blobs[d] = b + return d, len(b) + } + cfgDigest, cfgSize := add([]byte(`{"architecture":"` + arch + `","os":"linux","rootfs":{"type":"layers"}}`)) + layerDigest, layerSize := add([]byte(layer)) + manifest := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",`+ + `"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":%q,"size":%d},`+ + `"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":%q,"size":%d}]}`, + cfgDigest, cfgSize, layerDigest, layerSize) + mDigest, mSize := add([]byte(manifest)) + index, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, + "manifests": []map[string]any{{ + "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": mDigest, "size": mSize, + }}, + }) + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + put := func(name string, b []byte) { + tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(b)), Typeflag: tar.TypeReg}) + tw.Write(b) + } + put("oci-layout", []byte(`{"imageLayoutVersion":"1.0.0"}`)) + put("index.json", index) + for d, b := range blobs { + put("blobs/sha256/"+strings.TrimPrefix(d, "sha256:"), b) + } + tw.Close() + path = filepath.Join(dir, arch+"-"+layer+".tar") + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return path, mDigest, cfgDigest +} + +func TestImageBundleWritesTheListingTheInstallerReads(t *testing.T) { + dir := t.TempDir() + limbo, limboDigest, limboConfig := writeTestLayout(t, dir, "arm64", "limbo") + lobby, lobbyDigest, lobbyConfig := writeTestLayout(t, dir, "arm64", "lobby") + out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt") + var stderr bytes.Buffer + code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list, + "--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo, + "--layout", "lobby=registry.felis.svc:5000/felis/lobby:demo=" + lobby, + }, nil, &stderr) + if code != 0 { + t.Fatalf("image-bundle = %d: %s", code, stderr.String()) + } + // deploy/bootstrap.sh reads this with `read -r role name digest config`. + got, _ := os.ReadFile(list) + want := "limbo registry.felis.svc:5000/felis/limbo:demo " + limboDigest + " " + limboConfig + "\n" + + "lobby registry.felis.svc:5000/felis/lobby:demo " + lobbyDigest + " " + lobbyConfig + "\n" + if string(got) != want { + t.Errorf("listing:\n%s\nwant:\n%s", got, want) + } + if fi, err := os.Stat(out); err != nil || fi.Size() == 0 { + t.Errorf("bundle: %v", err) + } + if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 { + t.Errorf("left behind %v", leftovers) + } +} + +func TestImageBundleLeavesNothingWhenAnImageIsRefused(t *testing.T) { + dir := t.TempDir() + amd, _, _ := writeTestLayout(t, dir, "amd64", "limbo") + out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt") + // The pair an earlier run wrote stays as it was: a listing beside a bundle it + // does not describe would have the installer look for images that are not there. + os.WriteFile(out, []byte("old bundle"), 0o644) + os.WriteFile(list, []byte("old listing\n"), 0o644) + var stderr bytes.Buffer + code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list, + "--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + amd}, nil, &stderr) + if code != 1 || !strings.Contains(stderr.String(), "is a linux/amd64 image") { + t.Fatalf("image-bundle = %d: %s", code, stderr.String()) + } + if got, _ := os.ReadFile(out); string(got) != "old bundle" { + t.Errorf("bundle was replaced with %d bytes", len(got)) + } + if got, _ := os.ReadFile(list); string(got) != "old listing\n" { + t.Errorf("listing was replaced with %q", got) + } + if leftovers, _ := filepath.Glob(filepath.Join(dir, "*.tmp")); len(leftovers) != 0 { + t.Errorf("left behind %v", leftovers) + } +} + +func TestPushImageReadsABundleByImageName(t *testing.T) { + dir := t.TempDir() + limbo, _, _ := writeTestLayout(t, dir, "arm64", "limbo") + out, list := filepath.Join(dir, "images.tar"), filepath.Join(dir, "images.txt") + if code := cmdImageBundle([]string{"--platform", "linux/arm64", "--out", out, "--list", list, + "--layout", "limbo=registry.felis.svc:5000/felis/limbo:demo=" + limbo}, nil, &bytes.Buffer{}); code != 0 { + t.Fatal("image-bundle failed") + } + t.Setenv("FELIS_REGISTRY_USERNAME", "platform") + t.Setenv("FELIS_REGISTRY_PASSWORD", "x") + // The name is looked up in the bundle's index before the registry is contacted, + // so a name the bundle lacks fails here with what it does hold. + var stderr bytes.Buffer + code := cmdPushImage([]string{"--tar", out, "--image", "registry.felis.svc:5000/felis/lobby:demo", + "--ref", "127.0.0.1:1/felis/lobby:demo"}, &bytes.Buffer{}, &stderr) + if code != 1 || !strings.Contains(stderr.String(), "holds no image named registry.felis.svc:5000/felis/lobby:demo (it holds: registry.felis.svc:5000/felis/limbo:demo") { + t.Fatalf("push-image = %d: %s", code, stderr.String()) + } +} diff --git a/cmd/felis/registrygate.go b/cmd/felis/registrygate.go index d8089f4..f69fd16 100644 --- a/cmd/felis/registrygate.go +++ b/cmd/felis/registrygate.go @@ -131,7 +131,8 @@ func loopbackAddr(addr string) bool { func cmdPushImage(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("push-image", flag.ContinueOnError) fs.SetOutput(stderr) - tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path") + tarPath := fs.String("tar", "", "image tarball Kaniko wrote with --tar-path, or with --image an OCI layout tar") + image := fs.String("image", "", "push the image this name (io.containerd.image.name) marks in the OCI layout tar --tar, e.g. a release's image bundle") ref := fs.String("ref", "", "host/repository:tag to publish it as") scheme := fs.String("scheme", "http", "registry scheme: http for the in-cluster registry, https otherwise") if err := fs.Parse(args); err != nil { @@ -154,7 +155,13 @@ func cmdPushImage(args []string, stdout, stderr io.Writer) int { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() p := &imagepush.Pusher{Scheme: *scheme, Username: user, Password: pass, Log: stderr} - digest, err := p.Push(ctx, *tarPath, *ref) + var digest string + var err error + if *image != "" { + digest, err = p.PushLayout(ctx, *tarPath, *image, *ref) + } else { + digest, err = p.Push(ctx, *tarPath, *ref) + } if err != nil { fmt.Fprintf(stderr, "felis push-image: %v\n", err) return 1 diff --git a/cmd/felis/run.go b/cmd/felis/run.go index f8ff00e..a4b5be7 100644 --- a/cmd/felis/run.go +++ b/cmd/felis/run.go @@ -78,6 +78,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{ "init-forwarding": cmdInitForwarding, "init-volume": cmdInitVolume, "pin-images": cmdPinImages, + "image-bundle": cmdImageBundle, "version": cmdVersion, "update": cmdUpdate, "watchdog": cmdWatchdog, diff --git a/cmd/felis/run_test.go b/cmd/felis/run_test.go index e5319b3..019ffef 100644 --- a/cmd/felis/run_test.go +++ b/cmd/felis/run_test.go @@ -43,7 +43,7 @@ func TestRunUnknownCommand(t *testing.T) { // decision rather than an oversight. var undocumentedCommands = map[string]bool{ "bootstrap-assets": true, "init-forwarding": true, "init-volume": true, - "pin-images": true, + "pin-images": true, "image-bundle": true, } // The usage text and the dispatch table must describe the same set of commands. diff --git a/internal/imagepush/bundle.go b/internal/imagepush/bundle.go new file mode 100644 index 0000000..bca6ce5 --- /dev/null +++ b/internal/imagepush/bundle.go @@ -0,0 +1,356 @@ +package imagepush + +import ( + "archive/tar" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "strings" + "time" +) + +// The release bundle: one OCI layout tar per architecture holding every image an +// install runs (the control plane, the login, lobby and Paper images, the registry +// and PostgreSQL), built once by CI. The installer imports it into k3s's containerd +// and pushes it into the platform registry, so a host needs neither Docker nor +// Docker Hub to install. +// +// Every index.json entry points straight at one platform's image manifest: no +// nested index, no platform field, no descriptor whose blobs are missing. ctr's +// importer then imports each entry the same way whatever platform matcher it runs +// with, and the digest a pod pins is the digest containerd holds. + +// BundleImage is one image WriteBundle puts in a bundle. +type BundleImage struct { + // Role is the image's job in the install (felis, limbo, registry, ...), for the + // listing the installer reads. + Role string + // Name is the containerd image name the bundle gives it. + Name string + // Layout is an OCI layout tar (buildx --output type=oci) holding the image. + Layout string + // Source is a digest-pinned reference to read it from a public registry when + // Layout is empty. + Source string +} + +// BundleEntry describes one image of a written bundle. +type BundleEntry struct { + Role, Name string + // Digest is the image manifest's digest; Config its config's, which is the + // image ID docker and CRI report. + Digest, Config string +} + +// bundleTime stamps every tar header, so two runs over the same images write the +// same bytes. +var bundleTime = time.Unix(0, 0).UTC() + +// resolvedImage is a BundleImage narrowed to one platform's manifest. +type resolvedImage struct { + BundleImage + mediaType string + body []byte + m *manifest + config []byte + // open returns one blob's bytes; the writer checks them against the digest. + open func(ctx context.Context, d descriptor) (io.ReadCloser, error) +} + +// WriteBundle writes the images as one OCI layout tar to w, each narrowed to s's +// platform, and returns what it wrote. Every blob is checked against its digest on +// the way through, and every image's config must be for the platform, so a bundle +// cannot carry an image another architecture's build left behind. +func WriteBundle(ctx context.Context, s *Source, images []BundleImage, w io.Writer) ([]BundleEntry, error) { + goos, arch, _ := s.platform() + seenRole, seenName := map[string]bool{}, map[string]bool{} + var resolved []*resolvedImage + for _, img := range images { + if img.Role == "" || img.Name == "" { + return nil, fmt.Errorf("imagepush: bundle image %+v needs a role and a name", img) + } + if seenRole[img.Role] || seenName[img.Name] { + return nil, fmt.Errorf("imagepush: bundle names role %s or image %s twice", img.Role, img.Name) + } + seenRole[img.Role], seenName[img.Name] = true, true + var ( + r *resolvedImage + err error + ) + switch { + case img.Layout != "" && img.Source == "": + r, err = resolveLayoutImage(img, s) + case img.Source != "" && img.Layout == "": + r, err = resolveSourceImage(ctx, img, s) + default: + err = fmt.Errorf("imagepush: bundle image %s needs exactly one of a layout and a source", img.Name) + } + if err != nil { + return nil, err + } + var cfg struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } + if err := json.Unmarshal(r.config, &cfg); err != nil { + return nil, fmt.Errorf("imagepush: %s: config: %w", img.Name, err) + } + if cfg.OS != goos || cfg.Architecture != arch { + return nil, fmt.Errorf("imagepush: %s is a %s/%s image, the bundle is for %s/%s", img.Name, cfg.OS, cfg.Architecture, goos, arch) + } + resolved = append(resolved, r) + } + + tw := tar.NewWriter(w) + written := map[string]int64{} + writeBytes := func(name string, b []byte) error { + if err := tw.WriteHeader(bundleHeader(name, int64(len(b)))); err != nil { + return err + } + _, err := tw.Write(b) + return err + } + if err := writeBytes(layoutMarkerFile, []byte(layoutMarkerContent)); err != nil { + return nil, err + } + for _, dir := range []string{"blobs/", "blobs/sha256/"} { + h := bundleHeader(dir, 0) + h.Typeflag, h.Mode = tar.TypeDir, 0o755 + if err := tw.WriteHeader(h); err != nil { + return nil, err + } + } + var ( + index = layoutIndex{SchemaVersion: 2, MediaType: mediaOCIIndex} + entries []BundleEntry + ) + for _, r := range resolved { + for _, d := range append([]descriptor{r.m.Config}, r.m.Layers...) { + if size, ok := written[d.Digest]; ok { + if size != d.Size { + return nil, fmt.Errorf("imagepush: %s: blob %s is %d bytes here and %d bytes in an earlier image", r.Name, d.Digest, d.Size, size) + } + continue + } + if err := copyBlob(ctx, tw, r, d); err != nil { + return nil, err + } + written[d.Digest] = d.Size + } + digest := digestOf(r.body) + if _, ok := written[digest]; !ok { + if err := writeBytes(blobPath(digest), r.body); err != nil { + return nil, err + } + written[digest] = int64(len(r.body)) + } + desc := layoutDescriptor{MediaType: r.mediaType, Digest: digest, Size: int64(len(r.body))} + for _, name := range bundleNames(r.Name, digest) { + e := desc + e.Annotations = map[string]string{annotationImageName: name} + if tag := refTag(name); tag != "" { + e.Annotations[annotationRefName] = tag + } + index.Manifests = append(index.Manifests, e) + } + entries = append(entries, BundleEntry{Role: r.Role, Name: r.Name, Digest: digest, Config: r.m.Config.Digest}) + } + body, err := json.Marshal(index) + if err != nil { + return nil, err + } + if err := writeBytes(layoutIndexFile, body); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + return entries, nil +} + +func bundleHeader(name string, size int64) *tar.Header { + return &tar.Header{Name: name, Mode: 0o644, Size: size, Typeflag: tar.TypeReg, ModTime: bundleTime} +} + +// bundleNames is every name the bundle gives an image: its own, and for a tagged +// name also repository@digest. Pods run the game images pinned to their digest +// (felis pin-images), and CRI looks a pinned reference up by that second name, so +// with it a pinned pod starts from what the bundle imported instead of pulling. +func bundleNames(name, digest string) []string { + if refTag(name) == "" { + return []string{name} + } + return []string{name, refRepo(name) + "@" + digest} +} + +// refTag is the tag of a host/repository:tag name, or "" for a digest name. +func refTag(name string) string { + if strings.Contains(name, "@") { + return "" + } + if i := strings.LastIndexByte(name, ':'); i > strings.LastIndexByte(name, '/') { + return name[i+1:] + } + return "" +} + +// refRepo is a reference with its tag and digest dropped. +func refRepo(ref string) string { + name, _, _ := strings.Cut(ref, "@") + if i := strings.LastIndexByte(name, ':'); i > strings.LastIndexByte(name, '/') { + return name[:i] + } + return name +} + +// PinnedName is the name containerd lists a digest-pinned reference under once CRI +// pulled it: repository@digest, the tag dropped. It is the name a bundle gives an +// image read from Source, so the pods naming that reference find it. +func PinnedName(ref string) string { + _, digest, _ := strings.Cut(ref, "@") + return refRepo(ref) + "@" + digest +} + +// copyBlob streams one blob into the tar, checking its size and digest. +func copyBlob(ctx context.Context, tw *tar.Writer, r *resolvedImage, d descriptor) error { + rc, err := r.open(ctx, d) + if err != nil { + return fmt.Errorf("imagepush: %s: blob %s: %w", r.Name, d.Digest, err) + } + defer rc.Close() + if err := tw.WriteHeader(bundleHeader(blobPath(d.Digest), d.Size)); err != nil { + return err + } + h := sha256.New() + n, err := io.Copy(io.MultiWriter(tw, h), io.LimitReader(rc, d.Size)) + if err != nil { + return fmt.Errorf("imagepush: %s: blob %s: %w", r.Name, d.Digest, err) + } + // A byte past the descriptor's size is looked for, never written. + if extra, _ := io.CopyN(io.Discard, rc, 1); extra > 0 { + return fmt.Errorf("imagepush: %s: blob %s is longer than its %d bytes", r.Name, d.Digest, d.Size) + } + if got := "sha256:" + hex.EncodeToString(h.Sum(nil)); n != d.Size || got != d.Digest { + return fmt.Errorf("imagepush: %s: blob %s (%d bytes) holds %s (%d bytes)", r.Name, d.Digest, d.Size, got, n) + } + return nil +} + +// resolveLayoutImage finds the platform's image in a buildx OCI layout tar. +func resolveLayoutImage(img BundleImage, s *Source) (*resolvedImage, error) { + idx, err := readLayoutIndex(img.Layout) + if err != nil { + return nil, err + } + entries := idx.Manifests + for depth := 0; depth < 4; depth++ { + d, err := pickLayoutEntry(entries, s) + if err != nil { + return nil, fmt.Errorf("imagepush: %s: %w", img.Layout, err) + } + body, err := readLayoutBlob(img.Layout, d) + if err != nil { + return nil, err + } + if isIndexType(d.MediaType) { + var nested layoutIndex + if err := json.Unmarshal(body, &nested); err != nil { + return nil, fmt.Errorf("imagepush: %s: index %s: %w", img.Layout, d.Digest, err) + } + entries = nested.Manifests + continue + } + if !isManifestType(d.MediaType) { + return nil, fmt.Errorf("imagepush: %s: %s is a %q, want an image", img.Layout, d.Digest, d.MediaType) + } + m, err := parseManifest(body, d.MediaType) + if err != nil { + return nil, fmt.Errorf("imagepush: %s: %w", img.Layout, err) + } + config, err := readLayoutBlob(img.Layout, layoutDescriptor{Digest: m.Config.Digest, Size: m.Config.Size}) + if err != nil { + return nil, err + } + path := img.Layout + return &resolvedImage{BundleImage: img, mediaType: d.MediaType, body: body, m: m, config: config, + open: func(_ context.Context, d descriptor) (io.ReadCloser, error) { + f, entry, err := openEntry(path, blobPath(d.Digest)) + if err != nil { + return nil, err + } + return struct { + io.Reader + io.Closer + }{entry, f}, nil + }}, nil + } + return nil, fmt.Errorf("imagepush: %s: indexes nest too deep", img.Layout) +} + +// pickLayoutEntry chooses the entry for s's platform. A lone entry without a +// platform is taken as it is (its config is checked later); buildx's attestation +// manifests say unknown/unknown and never match. Releases are built for plain +// linux/amd64 and linux/arm64, so a variant is not looked at. +func pickLayoutEntry(entries []layoutDescriptor, s *Source) (layoutDescriptor, error) { + if len(entries) == 1 && entries[0].Platform == nil { + return entries[0], nil + } + wantOS, wantArch, _ := s.platform() + for _, e := range entries { + if e.Platform != nil && e.Platform.OS == wantOS && e.Platform.Architecture == wantArch { + return e, nil + } + } + return layoutDescriptor{}, fmt.Errorf("no %s/%s image among %d entries", wantOS, wantArch, len(entries)) +} + +// resolveSourceImage reads the platform's manifest of a digest-pinned public image. +func resolveSourceImage(ctx context.Context, img BundleImage, s *Source) (*resolvedImage, error) { + sr, err := ParseSourceRef(img.Source) + if err != nil { + return nil, err + } + if sr.Digest == "" { + return nil, fmt.Errorf("imagepush: %s is not pinned by digest; a release bundles only pinned images", img.Source) + } + // manifest checks the bytes against the pinned digest. + body, mt, err := s.manifest(ctx, sr, sr.Digest) + if err != nil { + return nil, fmt.Errorf("imagepush: %s: %w", sr, err) + } + if isIndexType(mt) { + d, err := s.pick(body) + if err != nil { + return nil, fmt.Errorf("imagepush: %s: %w", sr, err) + } + if body, mt, err = s.manifest(ctx, sr, d); err != nil { + return nil, fmt.Errorf("imagepush: %s: %w", sr, err) + } + } + if !isManifestType(mt) { + return nil, fmt.Errorf("imagepush: %s: unsupported manifest type %q", sr, mt) + } + m, err := parseManifest(body, mt) + if err != nil { + return nil, fmt.Errorf("imagepush: %s: %w", sr, err) + } + rc, err := s.blob(ctx, sr, m.Config.Digest) + if err != nil { + return nil, fmt.Errorf("imagepush: %s: config: %w", sr, err) + } + config, err := io.ReadAll(io.LimitReader(rc, maxManifestBytes+1)) + rc.Close() + if err != nil { + return nil, fmt.Errorf("imagepush: %s: config: %w", sr, err) + } + if digestOf(config) != m.Config.Digest || int64(len(config)) != m.Config.Size { + return nil, fmt.Errorf("imagepush: %s: config does not match %s", sr, m.Config.Digest) + } + return &resolvedImage{BundleImage: img, mediaType: mt, body: body, m: m, config: config, + open: func(ctx context.Context, d descriptor) (io.ReadCloser, error) { + return s.blob(ctx, sr, d.Digest) + }}, nil +} diff --git a/internal/imagepush/bundle_test.go b/internal/imagepush/bundle_test.go new file mode 100644 index 0000000..e125907 --- /dev/null +++ b/internal/imagepush/bundle_test.go @@ -0,0 +1,344 @@ +package imagepush + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "felis.lolicon.best/internal/registrygate" +) + +// layoutBuilder writes OCI layout tars the way buildx's oci exporter does. +type layoutBuilder struct { + blobs map[string][]byte + order []string +} + +func newLayoutBuilder() *layoutBuilder { return &layoutBuilder{blobs: map[string][]byte{}} } + +func (b *layoutBuilder) add(body []byte) descriptor { + d := digestOf(body) + if _, ok := b.blobs[d]; !ok { + b.order = append(b.order, d) + } + b.blobs[d] = body + return descriptor{MediaType: mediaOCILayerGz, Size: int64(len(body)), Digest: d} +} + +// image stores one platform's manifest and returns its descriptor. +func (b *layoutBuilder) image(arch string, layers ...string) layoutDescriptor { + cfg := b.add([]byte(`{"architecture":"` + arch + `","os":"linux","rootfs":{"type":"layers"}}`)) + cfg.MediaType = mediaOCIConfig + m := manifest{SchemaVersion: 2, MediaType: mediaOCIManifest, Config: cfg} + for _, l := range layers { + m.Layers = append(m.Layers, b.add([]byte(l))) + } + body, _ := json.Marshal(m) + d := b.add(body) + return layoutDescriptor{MediaType: mediaOCIManifest, Digest: d.Digest, Size: d.Size} +} + +// index stores a nested index over the given entries. +func (b *layoutBuilder) index(entries ...layoutDescriptor) layoutDescriptor { + body, _ := json.Marshal(layoutIndex{SchemaVersion: 2, MediaType: mediaOCIIndex, Manifests: entries}) + d := b.add(body) + return layoutDescriptor{MediaType: mediaOCIIndex, Digest: d.Digest, Size: d.Size} +} + +func (b *layoutBuilder) write(t *testing.T, top ...layoutDescriptor) string { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + add := func(name string, body []byte) { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + tw.Write(body) + } + add(layoutMarkerFile, []byte(layoutMarkerContent)) + idx, _ := json.Marshal(layoutIndex{SchemaVersion: 2, MediaType: mediaOCIIndex, Manifests: top}) + add(layoutIndexFile, idx) + for _, d := range b.order { + add(blobPath(d), b.blobs[d]) + } + tw.Close() + path := filepath.Join(t.TempDir(), "layout.tar") + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +// readTar returns every regular file in a tar, and fails on a name written twice. +func readTar(t *testing.T, r io.Reader) map[string][]byte { + t.Helper() + files := map[string][]byte{} + tr := tar.NewReader(r) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + return files + } + if err != nil { + t.Fatal(err) + } + if h.Typeflag != tar.TypeReg { + continue + } + if _, dup := files[h.Name]; dup { + t.Errorf("%s is written twice", h.Name) + } + files[h.Name], _ = io.ReadAll(tr) + } +} + +func TestBundleCarriesOnePlatformAndPushesByteForByte(t *testing.T) { + // The control-plane image the way buildx leaves it with provenance on: an index + // over the image and an attestation manifest, beside another architecture. + lb := newLayoutBuilder() + arm := lb.image("arm64", "shared base", "felis binary") + amd := lb.image("amd64", "amd64 base", "amd64 binary") + att := lb.image("unknown", "provenance") + att.Annotations = map[string]string{"vnd.docker.reference.type": "attestation-manifest"} + att.Platform = &platformSpec{OS: "unknown", Architecture: "unknown"} + arm.Platform = &platformSpec{OS: "linux", Architecture: "arm64"} + amd.Platform = &platformSpec{OS: "linux", Architecture: "amd64"} + felisLayout := lb.write(t, lb.index(amd, att, arm)) + + // A game image built alone: one manifest, no platform field, sharing a base layer. + lb2 := newLayoutBuilder() + limbo := lb2.image("arm64", "shared base", "limbo jar") + limboLayout := lb2.write(t, limbo) + + src := newFakeSource(t) + idx, srcArm := src.addIndex("2.8.3") + pinned := src.host() + "/tools/thing:2.8.3@" + idx + + var out bytes.Buffer + s := &Source{Scheme: "http", Platform: "linux/arm64"} + entries, err := WriteBundle(context.Background(), s, []BundleImage{ + {Role: "felis", Name: "registry.felis.svc:5000/felis/felis:v1.2.3", Layout: felisLayout}, + {Role: "limbo", Name: "registry.felis.svc:5000/felis/limbo:demo", Layout: limboLayout}, + {Role: "registry", Name: PinnedName(pinned), Source: pinned}, + }, &out) + if err != nil { + t.Fatal(err) + } + + want := map[string]string{"felis": arm.Digest, "limbo": limbo.Digest, "registry": srcArm} + if len(entries) != 3 { + t.Fatalf("entries = %+v", entries) + } + for _, e := range entries { + if e.Digest != want[e.Role] { + t.Errorf("%s: digest %s, want the arm64 manifest %s", e.Role, e.Digest, want[e.Role]) + } + } + if entries[2].Name != src.host()+"/tools/thing@"+idx { + t.Errorf("the pulled image is named %s, want repository@index-digest, the name CRI looks the pinned ref up by", entries[2].Name) + } + + files := readTar(t, bytes.NewReader(out.Bytes())) + if string(files[layoutMarkerFile]) != layoutMarkerContent { + t.Errorf("oci-layout = %q", files[layoutMarkerFile]) + } + var index layoutIndex + if err := json.Unmarshal(files[layoutIndexFile], &index); err != nil { + t.Fatal(err) + } + names := map[string]layoutDescriptor{} + for _, m := range index.Manifests { + if m.Platform != nil || m.MediaType != mediaOCIManifest { + t.Errorf("index entry %+v must point straight at a manifest, with no platform", m) + } + names[m.Annotations[annotationImageName]] = m + } + for name, digest := range map[string]string{ + "registry.felis.svc:5000/felis/felis:v1.2.3": arm.Digest, + "registry.felis.svc:5000/felis/felis@" + arm.Digest: arm.Digest, + "registry.felis.svc:5000/felis/limbo:demo": limbo.Digest, + "registry.felis.svc:5000/felis/limbo@" + limbo.Digest: limbo.Digest, + src.host() + "/tools/thing@" + idx: srcArm, + } { + if names[name].Digest != digest { + t.Errorf("%s -> %q, want %s", name, names[name].Digest, digest) + } + } + if len(names) != 5 { + t.Errorf("index names %d images, want 5: %v", len(names), names) + } + if got := names["registry.felis.svc:5000/felis/limbo:demo"].Annotations[annotationRefName]; got != "demo" { + t.Errorf("ref.name = %q, want the tag", got) + } + + // Every blob an entry needs is in the tar and hashes to its name; nothing from + // the other architecture or the attestation came along. + need := map[string]bool{} + for _, m := range index.Manifests { + body := files[blobPath(m.Digest)] + if digestOf(body) != m.Digest { + t.Fatalf("manifest %s missing or corrupt", m.Digest) + } + var mf manifest + json.Unmarshal(body, &mf) + need[m.Digest] = true + for _, d := range append([]descriptor{mf.Config}, mf.Layers...) { + need[d.Digest] = true + if digestOf(files[blobPath(d.Digest)]) != d.Digest { + t.Errorf("blob %s missing or corrupt", d.Digest) + } + } + } + for name := range files { + if strings.HasPrefix(name, "blobs/") && !need["sha256:"+strings.TrimPrefix(name, "blobs/sha256/")] { + t.Errorf("%s is in the bundle but no image uses it", name) + } + } + + // The same bytes again give the same bundle. + var again bytes.Buffer + if _, err := WriteBundle(context.Background(), s, []BundleImage{ + {Role: "felis", Name: "registry.felis.svc:5000/felis/felis:v1.2.3", Layout: felisLayout}, + {Role: "limbo", Name: "registry.felis.svc:5000/felis/limbo:demo", Layout: limboLayout}, + {Role: "registry", Name: PinnedName(pinned), Source: pinned}, + }, &again); err != nil { + t.Fatal(err) + } + if !bytes.Equal(out.Bytes(), again.Bytes()) { + t.Error("two runs over the same images wrote different bundles") + } + // Both runs above fall in the same second; a rebuild of the release a day later + // must write the same bytes too, so no header carries the time it was written. + tr := tar.NewReader(bytes.NewReader(out.Bytes())) + for { + h, err := tr.Next() + if err != nil { + break + } + if !h.ModTime.Equal(time.Unix(0, 0)) { + t.Errorf("%s is stamped %v", h.Name, h.ModTime) + } + } + + // Pushed from the bundle, the registry records the manifest containerd imported. + bundle := filepath.Join(t.TempDir(), "bundle.tar") + os.WriteFile(bundle, out.Bytes(), 0o644) + reg, host := startStack(t) + p := &Pusher{Scheme: "http", Username: registrygate.PrincipalPlatform, Password: "plat-secret", Attempts: 1} + for _, tc := range []struct{ name, repo, digest string }{ + {"registry.felis.svc:5000/felis/limbo:demo", "felis/limbo", limbo.Digest}, + {src.host() + "/tools/thing@" + idx, "felis/thing", srcArm}, + } { + got, err := p.PushLayout(context.Background(), bundle, tc.name, host+"/"+tc.repo+":demo") + if err != nil { + t.Fatal(err) + } + if got != tc.digest { + t.Errorf("pushed %s as %s, want %s", tc.name, got, tc.digest) + } + if !bytes.Equal(reg.manifests[tc.repo+":demo"], files[blobPath(tc.digest)]) { + t.Errorf("%s: the registry's manifest differs from the bundle's bytes", tc.repo) + } + if reg.types[tc.repo+":demo"] != mediaOCIManifest { + t.Errorf("%s: pushed as %q", tc.repo, reg.types[tc.repo+":demo"]) + } + } + puts := reg.puts + if _, err := p.PushLayout(context.Background(), bundle, "registry.felis.svc:5000/felis/limbo:demo", host+"/felis/limbo:demo"); err != nil { + t.Fatal(err) + } + if reg.puts != puts { + t.Errorf("a second push uploaded %d blobs the registry already held", reg.puts-puts) + } +} + +func TestBundleRefusesAnotherArchitecturesImage(t *testing.T) { + lb := newLayoutBuilder() + layout := lb.write(t, lb.image("amd64", "layer")) + _, err := WriteBundle(context.Background(), &Source{Platform: "linux/arm64"}, + []BundleImage{{Role: "limbo", Name: "r.example:5000/felis/limbo:demo", Layout: layout}}, io.Discard) + if err == nil || !strings.Contains(err.Error(), "is a linux/amd64 image, the bundle is for linux/arm64") { + t.Fatalf("WriteBundle = %v, want the architecture refused", err) + } +} + +func TestBundleRefusesANonImage(t *testing.T) { + // An OCI artifact manifest has an image manifest's shape but is no image: the + // installer's push would refuse it, so the release build must refuse it first. + lb := newLayoutBuilder() + img := lb.image("arm64", "layer") + var mf manifest + json.Unmarshal(lb.blobs[img.Digest], &mf) + mf.MediaType = "" + body, _ := json.Marshal(mf) + d := lb.add(body) + layout := lb.write(t, layoutDescriptor{MediaType: "application/vnd.oci.artifact.manifest.v1+json", Digest: d.Digest, Size: d.Size}) + _, err := WriteBundle(context.Background(), &Source{Platform: "linux/arm64"}, + []BundleImage{{Role: "limbo", Name: "r.example:5000/felis/limbo:demo", Layout: layout}}, io.Discard) + if err == nil || !strings.Contains(err.Error(), "want an image") { + t.Fatalf("WriteBundle = %v, want the artifact refused", err) + } +} + +func TestBundleRefusesATamperedBlob(t *testing.T) { + for swapped, want := range map[string]string{ + "the fake layer": "holds sha256:", + "the real layer, and more": "is longer than its 14 bytes", + "the real": "(8 bytes)", + } { + lb := newLayoutBuilder() + img := lb.image("arm64", "the real layer") + var mf manifest + json.Unmarshal(lb.blobs[img.Digest], &mf) + lb.blobs[mf.Layers[0].Digest] = []byte(swapped) + layout := lb.write(t, img) + _, err := WriteBundle(context.Background(), &Source{Platform: "linux/arm64"}, + []BundleImage{{Role: "limbo", Name: "r.example:5000/felis/limbo:demo", Layout: layout}}, io.Discard) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("layer swapped for %q: WriteBundle = %v, want %q", swapped, err, want) + } + } +} + +func TestBundleRefusesAnUnpinnedSource(t *testing.T) { + src := newFakeSource(t) + src.addIndex("2.8.3") + _, err := WriteBundle(context.Background(), &Source{Scheme: "http", Platform: "linux/arm64"}, + []BundleImage{{Role: "registry", Name: "x", Source: src.host() + "/tools/thing:2.8.3"}}, io.Discard) + if err == nil || !strings.Contains(err.Error(), "not pinned by digest") { + t.Fatalf("WriteBundle = %v, want a tag-only source refused", err) + } +} + +func TestPushLayoutNamesWhatTheBundleHolds(t *testing.T) { + lb := newLayoutBuilder() + img := lb.image("arm64", "layer") + img.Annotations = map[string]string{annotationImageName: "r.example:5000/felis/limbo:demo"} + layout := lb.write(t, img) + _, host := startStack(t) + p := &Pusher{Scheme: "http", Username: registrygate.PrincipalPlatform, Password: "plat-secret", Attempts: 1} + _, err := p.PushLayout(context.Background(), layout, "r.example:5000/felis/lobby:demo", host+"/felis/lobby:demo") + if err == nil || !strings.Contains(err.Error(), "holds no image named r.example:5000/felis/lobby:demo (it holds: r.example:5000/felis/limbo:demo)") { + t.Fatalf("PushLayout = %v, want the missing name reported with what is there", err) + } +} + +func TestPinnedName(t *testing.T) { + for in, want := range map[string]string{ + "docker.io/library/registry:2.8.3@sha256:ab": "docker.io/library/registry@sha256:ab", + "host:5000/a/b:tag@sha256:cd": "host:5000/a/b@sha256:cd", + "host:5000/a/b@sha256:ef": "host:5000/a/b@sha256:ef", + } { + if got := PinnedName(in); got != want { + t.Errorf("PinnedName(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/imagepush/layout.go b/internal/imagepush/layout.go new file mode 100644 index 0000000..6003674 --- /dev/null +++ b/internal/imagepush/layout.go @@ -0,0 +1,184 @@ +package imagepush + +import ( + "context" + "encoding/json" + "fmt" + "io" + "sort" + "strings" +) + +// An OCI image layout tar is what a release ships its images in: oci-layout, +// blobs/sha256/ and an index.json whose entries each name one image through +// the io.containerd.image.name annotation. `ctr images import` reads that +// annotation, so a host imports the images straight into k3s's containerd under +// the names the pods use, and PushLayout pushes the same bytes into the platform +// registry: the manifest goes up verbatim, so the registry and containerd agree on +// every digest. + +const ( + annotationImageName = "io.containerd.image.name" + annotationRefName = "org.opencontainers.image.ref.name" + layoutIndexFile = "index.json" + layoutMarkerFile = "oci-layout" + layoutMarkerContent = `{"imageLayoutVersion":"1.0.0"}` +) + +// layoutDescriptor is one entry of an index: index.json's or a nested index's. +type layoutDescriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` + Annotations map[string]string `json:"annotations,omitempty"` + Platform *platformSpec `json:"platform,omitempty"` +} + +type platformSpec struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + Variant string `json:"variant,omitempty"` +} + +type layoutIndex struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType,omitempty"` + Manifests []layoutDescriptor `json:"manifests"` +} + +// blobPath is where a layout keeps the blob with this digest. +func blobPath(digest string) string { + return "blobs/sha256/" + strings.TrimPrefix(digest, "sha256:") +} + +func isManifestType(mt string) bool { return mt == mediaOCIManifest || mt == mediaDockerManifest } +func isIndexType(mt string) bool { return mt == mediaOCIIndex || mt == mediaDockerList } + +// readLayoutIndex reads a layout tar's index.json. +func readLayoutIndex(tarPath string) (*layoutIndex, error) { + raw, err := readEntry(tarPath, layoutIndexFile, maxManifestBytes) + if err != nil { + return nil, err + } + var idx layoutIndex + if err := json.Unmarshal(raw, &idx); err != nil { + return nil, fmt.Errorf("imagepush: %s: %s: %w", tarPath, layoutIndexFile, err) + } + return &idx, nil +} + +// readLayoutBlob reads a small blob (a manifest, an index, a config) out of a layout +// tar and checks it against its descriptor. +func readLayoutBlob(tarPath string, d layoutDescriptor) ([]byte, error) { + if !sha256DigestRE.MatchString(d.Digest) { + return nil, fmt.Errorf("imagepush: %s: bad digest %q", tarPath, d.Digest) + } + if d.Size < 0 || d.Size > maxManifestBytes { + return nil, fmt.Errorf("imagepush: %s: blob %s has size %d", tarPath, d.Digest, d.Size) + } + body, err := readEntry(tarPath, blobPath(d.Digest), maxManifestBytes) + if err != nil { + return nil, err + } + if got := digestOf(body); got != d.Digest || int64(len(body)) != d.Size { + return nil, fmt.Errorf("imagepush: %s: blob %s (%d bytes) holds %s (%d bytes)", tarPath, d.Digest, d.Size, got, len(body)) + } + return body, nil +} + +// PushLayout uploads the image the OCI layout tar at tarPath names name (its +// io.containerd.image.name) as ref, and returns the manifest digest the registry +// recorded: the digest of the manifest in the tar, since it is pushed byte for byte. +func (p *Pusher) PushLayout(ctx context.Context, tarPath, name, ref string) (string, error) { + r, err := ParseRef(ref) + if err != nil { + return "", err + } + idx, err := readLayoutIndex(tarPath) + if err != nil { + return "", err + } + var ( + desc layoutDescriptor + found int + names []string + ) + for _, m := range idx.Manifests { + n := m.Annotations[annotationImageName] + names = append(names, n) + if n == name { + desc = m + found++ + } + } + switch { + case found == 0: + sort.Strings(names) + return "", fmt.Errorf("imagepush: %s holds no image named %s (it holds: %s)", tarPath, name, strings.Join(names, ", ")) + case found > 1: + return "", fmt.Errorf("imagepush: %s names %d images %s", tarPath, found, name) + } + // A bundle entry points straight at one platform's manifest: that is what makes + // the pushed digest the one containerd imported. + if !isManifestType(desc.MediaType) { + return "", fmt.Errorf("imagepush: %s: %s is a %q, want an image manifest", tarPath, name, desc.MediaType) + } + body, err := readLayoutBlob(tarPath, desc) + if err != nil { + return "", err + } + m, err := parseManifest(body, desc.MediaType) + if err != nil { + return "", fmt.Errorf("imagepush: %s: %s: %w", tarPath, name, err) + } + blobs := append([]descriptor{m.Config}, m.Layers...) + for i, b := range blobs { + err := p.retry(ctx, func() error { + return p.uploadBlob(ctx, r, b.Digest, b.Size, func() (io.ReadCloser, error) { + f, entry, err := openEntry(tarPath, blobPath(b.Digest)) + if err != nil { + return nil, err + } + return struct { + io.Reader + io.Closer + }{entry, f}, nil + }) + }) + if err != nil { + return "", fmt.Errorf("imagepush: %s: blob %d/%d (%s): %w", name, i+1, len(blobs), b.Digest, err) + } + } + var digest string + err = p.retry(ctx, func() error { + d, err := p.putManifest(ctx, r, desc.MediaType, body) + digest = d + return err + }) + if err != nil { + return "", fmt.Errorf("imagepush: %s: manifest: %w", r, err) + } + p.logf("pushed %s as %s@%s", name, r, digest) + return digest, nil +} + +// parseManifest reads an image manifest's config and layers and checks each is a +// well-formed descriptor. A mediaType field in the body must agree with mt. +func parseManifest(body []byte, mt string) (*manifest, error) { + var m manifest + if err := json.Unmarshal(body, &m); err != nil { + return nil, fmt.Errorf("manifest: %w", err) + } + if m.MediaType != "" && m.MediaType != mt { + return nil, fmt.Errorf("manifest says it is a %q, its descriptor a %q", m.MediaType, mt) + } + if len(m.Layers) == 0 { + return nil, fmt.Errorf("manifest has no layers") + } + for i, b := range append([]descriptor{m.Config}, m.Layers...) { + if !sha256DigestRE.MatchString(b.Digest) || b.Size < 0 { + return nil, fmt.Errorf("blob %d has digest %q size %d", i+1, b.Digest, b.Size) + } + } + return &m, nil +}