From 51179683780c2228702fff1f45db1eeae2e0b9ec Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 1 Jul 2026 12:55:55 +0800 Subject: [PATCH 1/4] feat: add manifest and ociutil packages (shared OCI helpers) Relocate epoch's leaf packages into cocoon-common so vk + operator can share them without depending on epoch: manifest (OCI media types + artifact classification) and ociutil (SHA-256, blob copy/verify, ref parsing). epoch keeps its own copies until it retires. --- manifest/media.go | 89 ++++++++++++++ manifest/types.go | 169 ++++++++++++++++++++++++++ manifest/types_test.go | 262 +++++++++++++++++++++++++++++++++++++++++ ociutil/ociutil.go | 62 ++++++++++ 4 files changed, 582 insertions(+) create mode 100644 manifest/media.go create mode 100644 manifest/types.go create mode 100644 manifest/types_test.go create mode 100644 ociutil/ociutil.go diff --git a/manifest/media.go b/manifest/media.go new file mode 100644 index 0000000..8017795 --- /dev/null +++ b/manifest/media.go @@ -0,0 +1,89 @@ +package manifest + +import "strings" + +const ( + // OCI / Docker manifest envelopes. + MediaTypeOCIManifest = "application/vnd.oci.image.manifest.v1+json" + MediaTypeOCIIndex = "application/vnd.oci.image.index.v1+json" + MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeDockerIndex = "application/vnd.docker.distribution.manifest.list.v2+json" + + // OCI / Docker config blob types. + MediaTypeOCIImageConfig = "application/vnd.oci.image.config.v1+json" + MediaTypeDockerConfig = "application/vnd.docker.container.image.v1+json" + + // Cocoonstack artifactType discriminators. + // WindowsImage is the legacy name; both are recognized. + ArtifactTypeOSImage = "application/vnd.cocoonstack.os-image.v1+json" + ArtifactTypeWindowsImage = "application/vnd.cocoonstack.windows-image.v1+json" + ArtifactTypeSnapshot = "application/vnd.cocoonstack.snapshot.v1+json" + + MediaTypeSnapshotConfig = "application/vnd.cocoonstack.snapshot.config.v1+json" + + // Disk layer media types. *Part variants are for split disks (GHCR per-layer limit). + MediaTypeDiskQcow2 = "application/vnd.cocoonstack.disk.qcow2" + MediaTypeDiskQcow2Part = "application/vnd.cocoonstack.disk.qcow2.part" + MediaTypeDiskRaw = "application/vnd.cocoonstack.disk.raw" + MediaTypeDiskRawPart = "application/vnd.cocoonstack.disk.raw.part" + + // Legacy windows-specific disk media types. + MediaTypeWindowsDiskQcow2 = "application/vnd.cocoonstack.windows.disk.qcow2" + MediaTypeWindowsDiskQcow2Part = "application/vnd.cocoonstack.windows.disk.qcow2.part" + MediaTypeWindowsDiskRaw = "application/vnd.cocoonstack.windows.disk.raw" + MediaTypeWindowsDiskRawPart = "application/vnd.cocoonstack.windows.disk.raw.part" + + // Snapshot-specific layer media types. + MediaTypeVMConfig = "application/vnd.cocoonstack.vm.config+json" + MediaTypeVMState = "application/vnd.cocoonstack.vm.state+json" + MediaTypeVMMemory = "application/vnd.cocoonstack.vm.memory" + MediaTypeVMCidata = "application/vnd.cocoonstack.vm.cidata" + + MediaTypeGeneric = "application/octet-stream" + MediaTypeTar = "application/x-tar" + + // OCI standard annotation keys. + AnnotationTitle = "org.opencontainers.image.title" + AnnotationCreated = "org.opencontainers.image.created" + AnnotationSource = "org.opencontainers.image.source" + AnnotationRevision = "org.opencontainers.image.revision" + AnnotationDescription = "org.opencontainers.image.description" + + // Cocoonstack annotation keys. + AnnotationSnapshotID = "cocoonstack.snapshot.id" + AnnotationSnapshotBaseImage = "cocoonstack.snapshot.baseimage" +) + +var snapshotFilenameMediaType = map[string]string{ + "config.json": MediaTypeVMConfig, + "state.json": MediaTypeVMState, + "memory-ranges": MediaTypeVMMemory, + "cidata.img": MediaTypeVMCidata, + "overlay.qcow2": MediaTypeDiskQcow2, +} + +// MediaTypeForCocoonFile returns the layer mediaType for a cocoon snapshot tar file. +func MediaTypeForCocoonFile(name string) string { + if mt, ok := snapshotFilenameMediaType[name]; ok { + return mt + } + switch { + case strings.HasSuffix(name, ".qcow2"): + return MediaTypeDiskQcow2 + case strings.HasSuffix(name, ".raw"): + return MediaTypeDiskRaw + } + return MediaTypeGeneric +} + +// IsDiskMediaType reports whether mt is a disk layer mediaType. +func IsDiskMediaType(mt string) bool { + switch mt { + case MediaTypeDiskQcow2, MediaTypeDiskQcow2Part, + MediaTypeDiskRaw, MediaTypeDiskRawPart, + MediaTypeWindowsDiskQcow2, MediaTypeWindowsDiskQcow2Part, + MediaTypeWindowsDiskRaw, MediaTypeWindowsDiskRawPart: + return true + } + return false +} diff --git a/manifest/types.go b/manifest/types.go new file mode 100644 index 0000000..b062d3e --- /dev/null +++ b/manifest/types.go @@ -0,0 +1,169 @@ +// Package manifest defines OCI manifest types and artifact classification. +package manifest + +import ( + "encoding/json" + "fmt" + "time" +) + +// Kind identifies the artifact classification of an OCI manifest. +type Kind int + +const ( + KindUnknown Kind = iota + KindContainerImage + KindCloudImage + KindSnapshot + KindImageIndex +) + +// String returns the human-readable name of the artifact kind. +func (k Kind) String() string { + switch k { + case KindContainerImage: + return "container-image" + case KindCloudImage: + return "cloud-image" + case KindSnapshot: + return "snapshot" + case KindImageIndex: + return "image-index" + default: + return "unknown" + } +} + +// Descriptor is an OCI content descriptor (mediaType + digest + size). +type Descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` + Annotations map[string]string `json:"annotations,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` +} + +// Title returns the org.opencontainers.image.title annotation, if any. +func (d Descriptor) Title() string { + if d.Annotations == nil { + return "" + } + return d.Annotations[AnnotationTitle] +} + +// OCIManifest represents both OCI image manifests and image indexes. +type OCIManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Config Descriptor `json:"config"` + Layers []Descriptor `json:"layers"` + Manifests []IndexManifest `json:"manifests,omitempty"` + Subject *Descriptor `json:"subject,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// IndexManifest describes one platform entry inside an OCI image index. +type IndexManifest struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` + Platform *Platform `json:"platform,omitempty"` +} + +// Platform describes OS and architecture for an index entry. +type Platform struct { + Architecture string `json:"architecture,omitempty"` + OS string `json:"os,omitempty"` + OSVersion string `json:"os.version,omitempty"` + Variant string `json:"variant,omitempty"` +} + +// Parse unmarshals raw JSON into an OCIManifest. +func Parse(raw []byte) (*OCIManifest, error) { + m := &OCIManifest{} + if err := json.Unmarshal(raw, m); err != nil { + return nil, fmt.Errorf("parse oci manifest: %w", err) + } + return m, nil +} + +// Classify returns the artifact kind from raw manifest JSON. +func Classify(raw []byte) (Kind, error) { + var probe struct { + MediaType string `json:"mediaType,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Config struct { + MediaType string `json:"mediaType"` + } `json:"config"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return KindUnknown, fmt.Errorf("classify manifest: %w", err) + } + return classifyFields(probe.ArtifactType, probe.Config.MediaType, probe.MediaType), nil +} + +// ClassifyParsed classifies an already-parsed manifest. +func ClassifyParsed(m *OCIManifest) Kind { + return classifyFields(m.ArtifactType, m.Config.MediaType, m.MediaType) +} + +// SnapshotFile holds per-file metadata stored in the snapshot config blob. +type SnapshotFile struct { + Mode int64 `json:"mode,omitempty"` + SparseMap string `json:"sparseMap,omitempty"` + SparseSize int64 `json:"sparseSize,omitempty"` +} + +// SnapshotConfig is the OCI config blob for snapshot manifests. +type SnapshotConfig struct { + SchemaVersion string `json:"schemaVersion"` + SnapshotID string `json:"snapshotId"` + Description string `json:"description,omitempty"` + Image string `json:"image,omitempty"` + ImageDigest string `json:"imageDigest,omitempty"` + ImageType string `json:"imageType,omitempty"` + ImageBlobIDs map[string]struct{} `json:"imageBlobIds,omitempty"` + Hypervisor string `json:"hypervisor,omitempty"` + CPU int `json:"cpu,omitempty"` + Memory int64 `json:"memory,omitempty"` + Storage int64 `json:"storage,omitempty"` + NICs int `json:"nics,omitempty"` + Network string `json:"network,omitempty"` + Windows bool `json:"windows,omitempty"` + Files map[string]SnapshotFile `json:"files,omitempty"` + CreatedAt time.Time `json:"createdAt,omitzero"` +} + +// Catalog is the global index of all repositories and their tags. +type Catalog struct { + Repositories map[string]*Repository `json:"repositories"` + UpdatedAt time.Time `json:"updatedAt,omitzero"` +} + +// Repository maps tag names to their manifest keys in the object store. +type Repository struct { + Tags map[string]string `json:"tags"` + UpdatedAt time.Time `json:"updatedAt,omitzero"` +} + +func classifyFields(artifactType, configMediaType, topMediaType string) Kind { + switch artifactType { + case ArtifactTypeOSImage, ArtifactTypeWindowsImage: + return KindCloudImage + case ArtifactTypeSnapshot: + return KindSnapshot + } + + switch configMediaType { + case MediaTypeOCIImageConfig, MediaTypeDockerConfig: + return KindContainerImage + } + + switch topMediaType { + case MediaTypeOCIIndex, MediaTypeDockerIndex: + return KindImageIndex + } + + return KindUnknown +} diff --git a/manifest/types_test.go b/manifest/types_test.go new file mode 100644 index 0000000..13c54b0 --- /dev/null +++ b/manifest/types_test.go @@ -0,0 +1,262 @@ +package manifest + +import "testing" + +func TestClassify(t *testing.T) { + tests := []struct { + name string + raw string + want Kind + }{ + { + name: "cloud image (unified os-image artifactType)", + raw: `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.cocoonstack.os-image.v1+json", + "config": {"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44","size":2}, + "layers": [ + {"mediaType":"application/vnd.cocoonstack.disk.qcow2.part","digest":"sha256:aa","size":1} + ] + }`, + want: KindCloudImage, + }, + { + name: "legacy windows-image artifactType (cocoonstack/windows builder)", + raw: `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.cocoonstack.windows-image.v1+json", + "config": {"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44","size":2}, + "layers": [ + {"mediaType":"application/vnd.cocoonstack.windows.disk.qcow2.part","digest":"sha256:aa","size":1} + ] + }`, + want: KindCloudImage, + }, + { + name: "snapshot oci artifact", + raw: `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.cocoonstack.snapshot.v1+json", + "config": {"mediaType":"application/vnd.cocoonstack.snapshot.config.v1+json","digest":"sha256:11","size":42}, + "layers": [ + {"mediaType":"application/vnd.cocoonstack.disk.qcow2","digest":"sha256:bb","size":1} + ] + }`, + want: KindSnapshot, + }, + { + name: "ubuntu docker buildx oci image (no artifactType, image config blob)", + raw: `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:cc","size":1500}, + "layers": [ + {"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:dd","size":987} + ] + }`, + want: KindContainerImage, + }, + { + name: "docker v2 manifest (no artifactType, docker config)", + raw: `{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": {"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:ee","size":700}, + "layers": [ + {"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:ff","size":42} + ] + }`, + want: KindContainerImage, + }, + { + name: "oci image index (multi-arch container, e.g. ghcr.io/cocoonstack/cocoon/ubuntu:24.04)", + raw: `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + {"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:aa","size":675,"platform":{"architecture":"amd64","os":"linux"}}, + {"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:bb","size":675,"platform":{"architecture":"arm64","os":"linux"}} + ] + }`, + want: KindImageIndex, + }, + { + name: "docker manifest list", + raw: `{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "manifests": [] + }`, + want: KindImageIndex, + }, + { + name: "manifest with no discriminator", + raw: `{"schemaVersion":2,"layers":[]}`, + want: KindUnknown, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Classify([]byte(tt.raw)) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if got != tt.want { + t.Errorf("Classify = %v, want %v", got, tt.want) + } + }) + } +} + +func TestClassifyMalformedJSON(t *testing.T) { + if _, err := Classify([]byte("not json")); err == nil { + t.Fatalf("expected error for non-JSON input") + } +} + +func TestParse(t *testing.T) { + raw := `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.cocoonstack.snapshot.v1+json", + "config": { + "mediaType": "application/vnd.cocoonstack.snapshot.config.v1+json", + "digest": "sha256:abc", + "size": 42 + }, + "layers": [ + { + "mediaType": "application/vnd.cocoonstack.vm.config+json", + "digest": "sha256:11", + "size": 100, + "annotations": {"org.opencontainers.image.title": "config.json"} + }, + { + "mediaType": "application/vnd.cocoonstack.disk.qcow2", + "digest": "sha256:22", + "size": 200, + "annotations": {"org.opencontainers.image.title": "overlay.qcow2"} + } + ], + "annotations": { + "cocoonstack.snapshot.id": "sid-1", + "cocoonstack.snapshot.baseimage": "ghcr.io/cocoonstack/cocoon/ubuntu:24.04" + } + }` + + m, err := Parse([]byte(raw)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if m.ArtifactType != ArtifactTypeSnapshot { + t.Errorf("ArtifactType = %q, want %q", m.ArtifactType, ArtifactTypeSnapshot) + } + if m.Config.MediaType != MediaTypeSnapshotConfig { + t.Errorf("Config.MediaType = %q, want %q", m.Config.MediaType, MediaTypeSnapshotConfig) + } + if len(m.Layers) != 2 { + t.Fatalf("len(Layers) = %d, want 2", len(m.Layers)) + } + if got := m.Layers[0].Title(); got != "config.json" { + t.Errorf("Layers[0].Title() = %q, want config.json", got) + } + if got := m.Layers[1].Title(); got != "overlay.qcow2" { + t.Errorf("Layers[1].Title() = %q, want overlay.qcow2", got) + } + if m.Annotations[AnnotationSnapshotBaseImage] != "ghcr.io/cocoonstack/cocoon/ubuntu:24.04" { + t.Errorf("baseimage annotation mismatch: %q", m.Annotations[AnnotationSnapshotBaseImage]) + } +} + +func TestParseIndex(t *testing.T) { + raw := `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:aaaa", + "size": 675, + "platform": {"architecture": "amd64", "os": "linux"} + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:bbbb", + "size": 675, + "platform": {"architecture": "arm64", "os": "linux", "variant": "v8"} + } + ] + }` + + m, err := Parse([]byte(raw)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := ClassifyParsed(m); got != KindImageIndex { + t.Errorf("ClassifyParsed = %v, want KindImageIndex", got) + } + if len(m.Manifests) != 2 { + t.Fatalf("len(Manifests) = %d, want 2", len(m.Manifests)) + } + if m.Manifests[0].Platform == nil || m.Manifests[0].Platform.Architecture != "amd64" { + t.Errorf("Manifests[0] arch = %+v, want amd64", m.Manifests[0].Platform) + } + if m.Manifests[1].Platform == nil || m.Manifests[1].Platform.Variant != "v8" { + t.Errorf("Manifests[1] variant = %+v, want v8", m.Manifests[1].Platform) + } +} + +func TestMediaTypeForCocoonFile(t *testing.T) { + tests := []struct { + filename string + want string + }{ + {"config.json", MediaTypeVMConfig}, + {"state.json", MediaTypeVMState}, + {"memory-ranges", MediaTypeVMMemory}, + {"cidata.img", MediaTypeVMCidata}, + {"overlay.qcow2", MediaTypeDiskQcow2}, + {"some-other-disk.qcow2", MediaTypeDiskQcow2}, + {"raw-disk.raw", MediaTypeDiskRaw}, + {"unknown", MediaTypeGeneric}, + } + for _, tt := range tests { + if got := MediaTypeForCocoonFile(tt.filename); got != tt.want { + t.Errorf("MediaTypeForCocoonFile(%q) = %q, want %q", tt.filename, got, tt.want) + } + } +} + +func TestIsDiskMediaType(t *testing.T) { + yes := []string{ + MediaTypeDiskQcow2, + MediaTypeDiskQcow2Part, + MediaTypeDiskRaw, + MediaTypeDiskRawPart, + MediaTypeWindowsDiskQcow2, + MediaTypeWindowsDiskQcow2Part, + MediaTypeWindowsDiskRaw, + MediaTypeWindowsDiskRawPart, + } + no := []string{ + MediaTypeVMConfig, + MediaTypeVMMemory, + MediaTypeGeneric, + "text/plain", + "", + } + for _, mt := range yes { + if !IsDiskMediaType(mt) { + t.Errorf("IsDiskMediaType(%q) = false, want true", mt) + } + } + for _, mt := range no { + if IsDiskMediaType(mt) { + t.Errorf("IsDiskMediaType(%q) = true, want false", mt) + } + } +} diff --git a/ociutil/ociutil.go b/ociutil/ociutil.go new file mode 100644 index 0000000..c0c81a9 --- /dev/null +++ b/ociutil/ociutil.go @@ -0,0 +1,62 @@ +// Package ociutil provides shared helpers for OCI blobs, digests, and refs. +package ociutil + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "strings" +) + +// SHA256Hex returns the hex-encoded SHA-256 digest of data. +func SHA256Hex(data []byte) string { + h := sha256.Sum256(data) + return hex.EncodeToString(h[:]) +} + +// CopyBlobExact copies exactly size bytes and verifies both length and sha256 digest. +func CopyBlobExact(dst io.Writer, body io.Reader, digest string, size int64) error { + h := sha256.New() + written, err := io.CopyN(io.MultiWriter(dst, h), body, size) + if err != nil { + return fmt.Errorf("copy blob %s: %w", digest, err) + } + if extra, _ := io.Copy(io.Discard, body); extra > 0 { + return fmt.Errorf("blob %s longer than manifest size %d (got %d extra)", digest, size, extra) + } + if written != size { + return fmt.Errorf("blob %s shorter than manifest size %d (got %d)", digest, size, written) + } + got := "sha256:" + hex.EncodeToString(h.Sum(nil)) + want := digest + if !strings.HasPrefix(want, "sha256:") { + want = "sha256:" + want + } + if got != want { + return fmt.Errorf("blob %s digest mismatch: got %s", digest, got) + } + return nil +} + +// HumanSize formats a byte count as a human-readable string (e.g. "1.2G"). +func HumanSize(b int64) string { + switch { + case b >= 1<<30: + return fmt.Sprintf("%.1fG", float64(b)/(1<<30)) + case b >= 1<<20: + return fmt.Sprintf("%.1fM", float64(b)/(1<<20)) + case b >= 1<<10: + return fmt.Sprintf("%.1fK", float64(b)/(1<<10)) + default: + return fmt.Sprintf("%dB", b) + } +} + +// ParseRef splits "name:tag" into name and tag; defaults tag to "latest". +func ParseRef(ref string) (string, string) { + if name, tag, ok := strings.Cut(ref, ":"); ok && name != "" { + return name, tag + } + return ref, "latest" +} From 266cfb8ae8e8b0b5647df6a5f27db5d695026b6a Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 1 Jul 2026 13:00:46 +0800 Subject: [PATCH 2/4] feat: add cloudimg and snapshot bridge packages Relocate epoch's snapshot OCI<->tar bridge (Pusher/Stream/Uploader/Downloader) and cloud-image stream into cocoon-common, imports repointed to the common manifest/ociutil packages, so vk + operator convert artifacts without depending on epoch. --- cloudimg/cloudimg.go | 69 +++++ cloudimg/cloudimg_test.go | 188 +++++++++++++ cloudimg/pull.go | 81 ++++++ snapshot/pull.go | 304 +++++++++++++++++++++ snapshot/push.go | 273 +++++++++++++++++++ snapshot/snapshot.go | 159 +++++++++++ snapshot/snapshot_test.go | 552 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 1626 insertions(+) create mode 100644 cloudimg/cloudimg.go create mode 100644 cloudimg/cloudimg_test.go create mode 100644 cloudimg/pull.go create mode 100644 snapshot/pull.go create mode 100644 snapshot/push.go create mode 100644 snapshot/snapshot.go create mode 100644 snapshot/snapshot_test.go diff --git a/cloudimg/cloudimg.go b/cloudimg/cloudimg.go new file mode 100644 index 0000000..e1fa73e --- /dev/null +++ b/cloudimg/cloudimg.go @@ -0,0 +1,69 @@ +// Package cloudimg streams OCI-packaged cloud images out of an OCI registry. +package cloudimg + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "slices" + + "github.com/cocoonstack/cocoon-common/manifest" + "github.com/cocoonstack/cocoon-common/ociutil" +) + +// BlobReader abstracts reading a blob by digest. +type BlobReader interface { + ReadBlob(ctx context.Context, digest string) (io.ReadCloser, error) +} + +// Stream concatenates disk layers sorted by title annotation. +func Stream(ctx context.Context, raw []byte, blobs BlobReader, w io.Writer) error { + m, err := manifest.Parse(raw) + if err != nil { + return fmt.Errorf("parse manifest: %w", err) + } + if kind := manifest.ClassifyParsed(m); kind != manifest.KindCloudImage { + return fmt.Errorf("manifest is %s, not a cloud image", kind) + } + + return StreamParsed(ctx, m, blobs, w) +} + +// StreamParsed streams disk layers from an already-parsed manifest. +func StreamParsed(ctx context.Context, m *manifest.OCIManifest, blobs BlobReader, w io.Writer) error { + disks := diskLayers(m.Layers) + if len(disks) == 0 { + return errors.New("cloud image manifest has no disk layers") + } + + for _, layer := range disks { + if err := copyBlob(ctx, blobs, layer, w); err != nil { + return err + } + } + return nil +} + +func diskLayers(layers []manifest.Descriptor) []manifest.Descriptor { + out := make([]manifest.Descriptor, 0, len(layers)) + for _, l := range layers { + if manifest.IsDiskMediaType(l.MediaType) { + out = append(out, l) + } + } + slices.SortStableFunc(out, func(a, b manifest.Descriptor) int { + return cmp.Compare(a.Title(), b.Title()) + }) + return out +} + +func copyBlob(ctx context.Context, blobs BlobReader, layer manifest.Descriptor, w io.Writer) error { + body, err := blobs.ReadBlob(ctx, layer.Digest) + if err != nil { + return fmt.Errorf("get blob %s: %w", layer.Digest, err) + } + defer func() { _ = body.Close() }() + return ociutil.CopyBlobExact(w, body, layer.Digest, layer.Size) +} diff --git a/cloudimg/cloudimg_test.go b/cloudimg/cloudimg_test.go new file mode 100644 index 0000000..7f31332 --- /dev/null +++ b/cloudimg/cloudimg_test.go @@ -0,0 +1,188 @@ +package cloudimg + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/cocoonstack/cocoon-common/manifest" +) + +// fakeBlobs is a tiny in-memory BlobReader for tests. +type fakeBlobs map[string][]byte + +func (f fakeBlobs) ReadBlob(_ context.Context, digest string) (io.ReadCloser, error) { + data, ok := f[digest] + if !ok { + return nil, errors.New("blob not found: " + digest) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +const ( + diskBlobA = "AAAA" + diskBlobB = "BBBB" + // Real sha256 digests of the byte contents above; required by CopyBlobExact's digest check. + digestA = "sha256:63c1dd951ffedf6f7fd968ad4efa39b8ed584f162f46e715114ee184f8de9201" + digestB = "sha256:4a8d8134f29b0b7b60c126f5532bc9f5d9bb73037373cf6fb872d81f1dcefdfd" +) + +var winManifest = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.cocoonstack.os-image.v1+json", + "config": {"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:00","size":2}, + "layers": [ + { + "mediaType": "application/vnd.cocoonstack.disk.qcow2.part", + "digest": "` + digestB + `", + "size": 4, + "annotations": {"org.opencontainers.image.title": "win.qcow2.01.qcow2.part"} + }, + { + "mediaType": "text/plain", + "digest": "sha256:cc", + "size": 32, + "annotations": {"org.opencontainers.image.title": "SHA256SUMS"} + }, + { + "mediaType": "application/vnd.cocoonstack.disk.qcow2.part", + "digest": "` + digestA + `", + "size": 4, + "annotations": {"org.opencontainers.image.title": "win.qcow2.00.qcow2.part"} + } + ] +}` + +func TestStreamConcatenatesDiskLayersInTitleOrder(t *testing.T) { + blobs := fakeBlobs{ + digestA: []byte(diskBlobA), + digestB: []byte(diskBlobB), + "sha256:cc": []byte("ignored-sha256sums"), + } + + var out bytes.Buffer + if err := Stream(t.Context(), []byte(winManifest), blobs, &out); err != nil { + t.Fatalf("Stream: %v", err) + } + if got, want := out.String(), "AAAABBBB"; got != want { + t.Errorf("Stream output = %q, want %q", got, want) + } +} + +func TestStreamRejectsContainerImage(t *testing.T) { + containerManifest := `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:00","size":1}, + "layers": [{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:11","size":1}] + }` + err := Stream(t.Context(), []byte(containerManifest), fakeBlobs{}, io.Discard) + if err == nil { + t.Fatal("expected error streaming container image") + } + if !strings.Contains(err.Error(), "not a cloud image") { + t.Errorf("error = %v, want %q substring", err, "not a cloud image") + } +} + +func TestStreamRejectsManifestWithNoDiskLayers(t *testing.T) { + noDisk := `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/vnd.cocoonstack.os-image.v1+json", + "config": {"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:00","size":2}, + "layers": [{"mediaType":"text/plain","digest":"sha256:11","size":1}] + }` + err := Stream(t.Context(), []byte(noDisk), fakeBlobs{}, io.Discard) + if err == nil { + t.Fatal("expected error for manifest with no disk layers") + } +} + +func TestDiskLayersFiltersAndSorts(t *testing.T) { + in := []manifest.Descriptor{ + {MediaType: "text/plain", Annotations: map[string]string{manifest.AnnotationTitle: "SHA256SUMS"}}, + {MediaType: manifest.MediaTypeDiskQcow2Part, Annotations: map[string]string{manifest.AnnotationTitle: "x.02.part"}}, + {MediaType: manifest.MediaTypeDiskQcow2Part, Annotations: map[string]string{manifest.AnnotationTitle: "x.00.part"}}, + {MediaType: manifest.MediaTypeDiskQcow2Part, Annotations: map[string]string{manifest.AnnotationTitle: "x.01.part"}}, + } + got := diskLayers(in) + want := []string{"x.00.part", "x.01.part", "x.02.part"} + if len(got) != len(want) { + t.Fatalf("len = %d, want %d", len(got), len(want)) + } + for i, w := range want { + if got[i].Title() != w { + t.Errorf("got[%d].Title() = %q, want %q", i, got[i].Title(), w) + } + } +} + +// fakeCocoon captures the cocoon image import stdin payload so puller tests +// can assert what cocoon would have received. +type fakeCocoon struct { + importPayload bytes.Buffer + importName string +} + +func (f *fakeCocoon) ImageImport(_ context.Context, name string) (io.WriteCloser, func() error, error) { + f.importName = name + return nopCloser{w: &f.importPayload}, func() error { return nil }, nil +} + +type nopCloser struct{ w io.Writer } + +func (n nopCloser) Write(p []byte) (int, error) { return n.w.Write(p) } +func (n nopCloser) Close() error { return nil } + +// fakeDownloader implements snapshot.Downloader from a static manifest + +// blob map. +type fakeDownloader struct { + manifest []byte + contentType string + blobs map[string][]byte +} + +func (f *fakeDownloader) GetManifest(_ context.Context, _, _ string) ([]byte, string, error) { + return f.manifest, f.contentType, nil +} + +func (f *fakeDownloader) GetBlob(_ context.Context, _, digest string) (io.ReadCloser, error) { + data, ok := f.blobs[digest] + if !ok { + return nil, errors.New("blob not found: " + digest) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func TestPullerPipesAssembledDiskToCocoonImport(t *testing.T) { + dl := &fakeDownloader{ + manifest: []byte(winManifest), + contentType: manifest.MediaTypeOCIManifest, + blobs: map[string][]byte{ + digestA: []byte(diskBlobA), + digestB: []byte(diskBlobB), + "sha256:cc": []byte("ignored"), + }, + } + cocoon := &fakeCocoon{} + puller := &Puller{Downloader: dl, Cocoon: cocoon} + + if err := puller.Pull(t.Context(), PullOptions{ + Name: "windows/win11", + Tag: "25h2", + LocalName: "win11", + }); err != nil { + t.Fatalf("Pull: %v", err) + } + if cocoon.importName != "win11" { + t.Errorf("import name = %q, want win11", cocoon.importName) + } + if got, want := cocoon.importPayload.String(), "AAAABBBB"; got != want { + t.Errorf("import payload = %q, want %q", got, want) + } +} diff --git a/cloudimg/pull.go b/cloudimg/pull.go new file mode 100644 index 0000000..63060ac --- /dev/null +++ b/cloudimg/pull.go @@ -0,0 +1,81 @@ +package cloudimg + +import ( + "context" + "errors" + "fmt" + "io" +) + +var _ BlobReader = blobReaderAdapter{} + +// Downloader abstracts OCI manifest and blob downloads. +type Downloader interface { + GetManifest(ctx context.Context, name, tag string) ([]byte, string, error) + GetBlob(ctx context.Context, name, digest string) (io.ReadCloser, error) +} + +// CocoonRunner abstracts the cocoon image import subprocess. +type CocoonRunner interface { + ImageImport(ctx context.Context, name string) (io.WriteCloser, func() error, error) +} + +// Puller downloads cloud-image artifacts and pipes them into cocoon image import. +type Puller struct { + Downloader Downloader + Cocoon CocoonRunner +} + +// PullOptions configures a cloud-image pull operation. +type PullOptions struct { + Name string // OCI repository name. Required. + Tag string // Defaults to "latest". + LocalName string // Override the cocoon-side image name. Empty = use Name. +} + +// Pull downloads a cloud-image artifact and pipes it into cocoon image import. +func (p *Puller) Pull(ctx context.Context, opts PullOptions) error { + if opts.Name == "" { + return errors.New("cloudimg pull: name is required") + } + if opts.Tag == "" { + opts.Tag = "latest" + } + localName := opts.LocalName + if localName == "" { + localName = opts.Name + } + + raw, _, err := p.Downloader.GetManifest(ctx, opts.Name, opts.Tag) + if err != nil { + return fmt.Errorf("get manifest %s:%s: %w", opts.Name, opts.Tag, err) + } + + stdin, wait, err := p.Cocoon.ImageImport(ctx, localName) + if err != nil { + return fmt.Errorf("start cocoon image import: %w", err) + } + + streamErr := Stream(ctx, raw, blobReaderAdapter{name: opts.Name, dl: p.Downloader}, stdin) + closeErr := stdin.Close() + waitErr := wait() + + switch { + case streamErr != nil: + return fmt.Errorf("stream cloudimg: %w", streamErr) + case closeErr != nil: + return fmt.Errorf("close cocoon stdin: %w", closeErr) + case waitErr != nil: + return fmt.Errorf("cocoon image import: %w", waitErr) + } + return nil +} + +type blobReaderAdapter struct { + name string + dl Downloader +} + +func (a blobReaderAdapter) ReadBlob(ctx context.Context, digest string) (io.ReadCloser, error) { + return a.dl.GetBlob(ctx, a.name, digest) +} diff --git a/snapshot/pull.go b/snapshot/pull.go new file mode 100644 index 0000000..f1ddabb --- /dev/null +++ b/snapshot/pull.go @@ -0,0 +1,304 @@ +package snapshot + +import ( + "archive/tar" + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/cocoon-common/manifest" + "github.com/cocoonstack/cocoon-common/ociutil" +) + +// Real configs hit ~1.4 MB on fragmented Windows VMs; 64 MiB leaves headroom while bounding pathological reads. +const maxSnapshotConfigSize = 64 << 20 + +// Puller downloads snapshot artifacts and pipes them into cocoon snapshot import. +type Puller struct { + Downloader Downloader + Cocoon CocoonRunner +} + +// PullOptions configures a snapshot pull operation. +type PullOptions struct { + Name string + Tag string + LocalName string // overrides cocoon-side snapshot name; empty = use Name + Description string + Progress func(string) +} + +// StreamOptions configures snapshot tar stream assembly. +type StreamOptions struct { + Name string + LocalName string // empty = use Name + Writer io.Writer + Progress func(string) +} + +// Pull downloads a snapshot artifact and feeds it to `cocoon snapshot import`. +func (p *Puller) Pull(ctx context.Context, opts PullOptions) error { + if opts.Name == "" { + return errors.New("snapshot pull: name is required") + } + if opts.Tag == "" { + opts.Tag = "latest" + } + localName := opts.LocalName + if localName == "" { + localName = opts.Name + } + + raw, _, err := p.Downloader.GetManifest(ctx, opts.Name, opts.Tag) + if err != nil { + return fmt.Errorf("get manifest %s:%s: %w", opts.Name, opts.Tag, err) + } + + stdin, wait, err := p.Cocoon.Import(ctx, ImportOptions{ + Name: localName, + Description: opts.Description, + }) + if err != nil { + return fmt.Errorf("start cocoon snapshot import: %w", err) + } + + streamErr := Stream(ctx, raw, p.Downloader, StreamOptions{ + Name: opts.Name, + LocalName: localName, + Writer: stdin, + Progress: opts.Progress, + }) + closeErr := stdin.Close() + waitErr := wait() + + switch { + case streamErr != nil: + return fmt.Errorf("stream snapshot: %w", streamErr) + case closeErr != nil: + return fmt.Errorf("close cocoon stdin: %w", closeErr) + case waitErr != nil: + return fmt.Errorf("cocoon snapshot import: %w", waitErr) + } + return nil +} + +// Stream reassembles a snapshot manifest into a cocoon-import tar stream. +// If raw is an OCI image-index (multi-platform), a child manifest is resolved +// via dl.GetManifest(..., childDigest) and streamed instead — preferring +// linux/amd64, falling back to the first non-attestation entry. +func Stream(ctx context.Context, raw []byte, dl Downloader, opts StreamOptions) error { + if opts.Name == "" { + return errors.New("snapshot stream: name is required") + } + if opts.Writer == nil { + return errors.New("snapshot stream: writer is required") + } + + m, err := manifest.Parse(raw) + if err != nil { + return fmt.Errorf("parse manifest: %w", err) + } + + if manifest.ClassifyParsed(m) == manifest.KindImageIndex { + child, err := pickIndexChild(ctx, m) + if err != nil { + return err + } + childRaw, _, err := dl.GetManifest(ctx, opts.Name, child.Digest) + if err != nil { + return fmt.Errorf("get child manifest %s: %w", child.Digest, err) + } + m, err = manifest.Parse(childRaw) + if err != nil { + return fmt.Errorf("parse child manifest: %w", err) + } + } + + if kind := manifest.ClassifyParsed(m); kind != manifest.KindSnapshot { + return fmt.Errorf("manifest is %s, not a snapshot", kind) + } + + return StreamParsed(ctx, m, dl, opts) +} + +// StreamParsed accepts an already-parsed manifest. +func StreamParsed(ctx context.Context, m *manifest.OCIManifest, dl Downloader, opts StreamOptions) error { + if opts.Name == "" { + return errors.New("snapshot stream: name is required") + } + if opts.Writer == nil { + return errors.New("snapshot stream: writer is required") + } + localName := opts.LocalName + if localName == "" { + localName = opts.Name + } + + cfg, err := FetchSnapshotConfig(ctx, dl, opts.Name, m.Config) + if err != nil { + return fmt.Errorf("fetch snapshot config: %w", err) + } + + return writeImportTar(ctx, dl, opts.Name, localName, cfg, m.Layers, opts.Writer, opts.Progress) +} + +// FetchSnapshotConfig downloads and parses the snapshot config blob. +func FetchSnapshotConfig(ctx context.Context, dl Downloader, name string, desc manifest.Descriptor) (*manifest.SnapshotConfig, error) { + if desc.MediaType != manifest.MediaTypeSnapshotConfig { + return nil, fmt.Errorf("unexpected config mediaType %q", desc.MediaType) + } + if desc.Size > maxSnapshotConfigSize { + return nil, fmt.Errorf("config blob too large: %d > %d", desc.Size, maxSnapshotConfigSize) + } + body, err := dl.GetBlob(ctx, name, desc.Digest) + if err != nil { + return nil, fmt.Errorf("get config blob %s: %w", desc.Digest, err) + } + defer func() { _ = body.Close() }() + data, err := io.ReadAll(io.LimitReader(body, maxSnapshotConfigSize+1)) + if err != nil { + return nil, fmt.Errorf("read config blob: %w", err) + } + if int64(len(data)) > maxSnapshotConfigSize { + return nil, fmt.Errorf("config blob exceeded cap %d while streaming", maxSnapshotConfigSize) + } + var cfg manifest.SnapshotConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parse snapshot config: %w", err) + } + return &cfg, nil +} + +// pickIndexChild selects the linux/amd64 child from an OCI image-index and +// falls back to the first non-attestation entry when no platform matches. +// A fallback selection is logged at Warn so operators can see which child +// was actually streamed instead of silently shipping a non-amd64 snapshot. +func pickIndexChild(ctx context.Context, m *manifest.OCIManifest) (manifest.IndexManifest, error) { + var fallback *manifest.IndexManifest + for i := range m.Manifests { + c := m.Manifests[i] + if c.Platform != nil && c.Platform.OS == "linux" && c.Platform.Architecture == "amd64" { + return c, nil + } + if fallback == nil && c.Platform != nil && c.Platform.Architecture != "unknown" { + fallback = &m.Manifests[i] + } + } + if fallback != nil { + log.WithFunc("snapshot.pickIndexChild").Warnf(ctx, + "image-index has no linux/amd64 child, falling back to %s/%s (%s)", + fallback.Platform.OS, fallback.Platform.Architecture, fallback.Digest) + return *fallback, nil + } + return manifest.IndexManifest{}, errors.New("image-index has no usable platform child") +} + +func writeImportTar(ctx context.Context, dl Downloader, name, localName string, cfg *manifest.SnapshotConfig, layers []manifest.Descriptor, w io.Writer, progress func(string)) error { + bw := bufio.NewWriterSize(w, 256<<10) + tw := tar.NewWriter(bw) + + now := nowFunc() + envelope := snapshotExportEnvelope{ + Version: 1, + Config: snapshotExportConfig{ + ID: cfg.SnapshotID, + Name: localName, + Description: cfg.Description, + Image: cfg.Image, + ImageDigest: cfg.ImageDigest, + ImageType: cfg.ImageType, + ImageBlobIDs: cfg.ImageBlobIDs, + Hypervisor: cfg.Hypervisor, + CPU: cfg.CPU, + Memory: cfg.Memory, + Storage: cfg.Storage, + NICs: cfg.NICs, + Network: cfg.Network, + Windows: cfg.Windows, + }, + } + envelopeJSON, err := json.MarshalIndent(envelope, "", " ") + if err != nil { + return fmt.Errorf("marshal snapshot envelope: %w", err) + } + envelopeJSON = append(envelopeJSON, '\n') + + if err := writeTarFile(tw, snapshotJSONName, envelopeJSON, 0o644, now); err != nil { + return fmt.Errorf("write snapshot envelope: %w", err) + } + + for _, layer := range layers { + title := layer.Title() + if title == "" { + return fmt.Errorf("layer %s missing %s annotation", layer.Digest, manifest.AnnotationTitle) + } + if progress != nil { + progress(fmt.Sprintf(" %s (%d bytes)", title, layer.Size)) + } + var fileMeta manifest.SnapshotFile + if cfg.Files != nil { + fileMeta = cfg.Files[title] + } + if err := streamLayerToTar(ctx, dl, name, layer, fileMeta, tw, now); err != nil { + return err + } + } + + if err := tw.Close(); err != nil { + return fmt.Errorf("close tar: %w", err) + } + return bw.Flush() +} + +func streamLayerToTar(ctx context.Context, dl Downloader, name string, layer manifest.Descriptor, fileMeta manifest.SnapshotFile, tw *tar.Writer, modTime time.Time) error { + mode := fileMeta.Mode + if mode == 0 { + mode = 0o640 + } + hdr := &tar.Header{ + Name: layer.Title(), + Size: layer.Size, + Mode: mode, + ModTime: modTime, + } + if fileMeta.SparseMap != "" { + if fileMeta.SparseSize <= 0 { + return fmt.Errorf("layer %s has sparse map without sparse size", layer.Digest) + } + hdr.PAXRecords = map[string]string{ + sparsePAXMap: fileMeta.SparseMap, + sparsePAXSize: strconv.FormatInt(fileMeta.SparseSize, 10), + } + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("write tar header: %w", err) + } + body, err := dl.GetBlob(ctx, name, layer.Digest) + if err != nil { + return fmt.Errorf("get blob %s: %w", layer.Digest, err) + } + defer func() { _ = body.Close() }() + return ociutil.CopyBlobExact(tw, body, layer.Digest, layer.Size) +} + +func writeTarFile(tw *tar.Writer, name string, data []byte, mode int64, modTime time.Time) error { + if err := tw.WriteHeader(&tar.Header{ + Name: name, + Size: int64(len(data)), + Mode: mode, + ModTime: modTime, + }); err != nil { + return fmt.Errorf("write tar header %s: %w", name, err) + } + if _, err := tw.Write(data); err != nil { + return fmt.Errorf("write tar body %s: %w", name, err) + } + return nil +} diff --git a/snapshot/push.go b/snapshot/push.go new file mode 100644 index 0000000..9473c66 --- /dev/null +++ b/snapshot/push.go @@ -0,0 +1,273 @@ +package snapshot + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strconv" + "time" + + "github.com/cocoonstack/cocoon-common/manifest" + "github.com/cocoonstack/cocoon-common/ociutil" +) + +// Pusher exports and uploads cocoon snapshots as OCI artifacts. +type Pusher struct { + Uploader Uploader + Cocoon CocoonRunner +} + +// PushOptions configures a snapshot push operation. +type PushOptions struct { + Name string + Tag string + BaseImage string // optional cocoonstack.snapshot.baseimage annotation + Source string + Revision string + Progress func(string) +} + +// PushResult contains the outcome of a successful push. +type PushResult struct { + Name string + Tag string + ManifestDigest string // sha256: + ManifestBytes []byte + TotalSize int64 + LayerCount int +} + +// Push exports a snapshot via cocoon and uploads it as an OCI artifact. +func (p *Pusher) Push(ctx context.Context, opts PushOptions) (*PushResult, error) { + if opts.Name == "" { + return nil, errors.New("snapshot push: name is required") + } + if opts.Tag == "" { + opts.Tag = "latest" + } + + stream, wait, err := p.Cocoon.Export(ctx, opts.Name) + if err != nil { + return nil, fmt.Errorf("cocoon export %s: %w", opts.Name, err) + } + + cfg, files, layers, readErr := p.readAndUploadEntries(ctx, opts.Name, stream, opts.Progress) + // Close before wait so mid-tar failures unblock the subprocess. + _ = stream.Close() + waitErr := wait() + if readErr != nil { + return nil, readErr + } + if waitErr != nil { + return nil, waitErr + } + if cfg == nil { + return nil, errMissingSnapshotJSON + } + + configDescriptor, err := p.uploadSnapshotConfig(ctx, opts.Name, cfg, files) + if err != nil { + return nil, fmt.Errorf("upload snapshot config: %w", err) + } + + manifestBytes, err := buildSnapshotManifest(configDescriptor, layers, opts) + if err != nil { + return nil, fmt.Errorf("build manifest: %w", err) + } + + if err := p.Uploader.PutManifest(ctx, opts.Name, opts.Tag, manifestBytes, manifest.MediaTypeOCIManifest); err != nil { + return nil, fmt.Errorf("put manifest %s:%s: %w", opts.Name, opts.Tag, err) + } + + manifestDigest := "sha256:" + ociutil.SHA256Hex(manifestBytes) + + var totalSize int64 + for _, l := range layers { + totalSize += l.Size + } + totalSize += configDescriptor.Size + + return &PushResult{ + Name: opts.Name, + Tag: opts.Tag, + ManifestDigest: manifestDigest, + ManifestBytes: manifestBytes, + TotalSize: totalSize, + LayerCount: len(layers), + }, nil +} + +func (p *Pusher) readAndUploadEntries(ctx context.Context, name string, r io.Reader, progress func(string)) (*snapshotExportConfig, map[string]manifest.SnapshotFile, []manifest.Descriptor, error) { + tr := tar.NewReader(r) + var ( + cfg *snapshotExportConfig + files = map[string]manifest.SnapshotFile{} + layers []manifest.Descriptor + ) + + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, nil, nil, fmt.Errorf("read tar entry: %w", err) + } + + if hdr.Name == snapshotJSONName { + var envelope snapshotExportEnvelope + if decErr := json.NewDecoder(tr).Decode(&envelope); decErr != nil { + return nil, nil, nil, fmt.Errorf("parse snapshot.json: %w", decErr) + } + cfg = &envelope.Config + continue + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + if hdr.Size < 0 { + return nil, nil, nil, fmt.Errorf("tar entry %s has negative size %d", hdr.Name, hdr.Size) + } + + desc, fileMeta, uploadErr := p.uploadTarEntry(ctx, name, hdr, tr) + if uploadErr != nil { + return nil, nil, nil, fmt.Errorf("upload %s: %w", hdr.Name, uploadErr) + } + files[hdr.Name] = fileMeta + layers = append(layers, desc) + if progress != nil { + progress(fmt.Sprintf(" %s -> %s (%d bytes)", hdr.Name, desc.Digest, desc.Size)) + } + } + + return cfg, files, layers, nil +} + +func (p *Pusher) uploadTarEntry(ctx context.Context, name string, hdr *tar.Header, body io.Reader) (manifest.Descriptor, manifest.SnapshotFile, error) { + tmp, err := os.CreateTemp("", "cocoon-snapshot-*") + if err != nil { + return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("create temp: %w", err) + } + defer func() { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + }() + + h := sha256.New() + written, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(body, hdr.Size)) + if err != nil { + return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("buffer entry: %w", err) + } + + digestHex := hex.EncodeToString(h.Sum(nil)) + digest := "sha256:" + digestHex + + exists, existsErr := p.Uploader.BlobExists(ctx, name, digest) + if existsErr != nil { + return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("check blob %s: %w", digest, existsErr) + } + if !exists { + if _, seekErr := tmp.Seek(0, io.SeekStart); seekErr != nil { + return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("seek temp: %w", seekErr) + } + if err := p.Uploader.PutBlob(ctx, name, digest, tmp, written); err != nil { + return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("put blob %s: %w", digest, err) + } + } + + fileMeta := manifest.SnapshotFile{Mode: hdr.Mode} + sparseMap, ok := hdr.PAXRecords[sparsePAXMap] + if ok { + fileMeta.SparseMap = sparseMap + rawSize, ok := hdr.PAXRecords[sparsePAXSize] + if !ok { + return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("sparse entry %s missing %s", hdr.Name, sparsePAXSize) + } + sparseSize, parseErr := strconv.ParseInt(rawSize, 10, 64) + if parseErr != nil { + return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("parse sparse size for %s: %w", hdr.Name, parseErr) + } + fileMeta.SparseSize = sparseSize + } + + return manifest.Descriptor{ + MediaType: manifest.MediaTypeForCocoonFile(hdr.Name), + Digest: digest, + Size: written, + Annotations: map[string]string{manifest.AnnotationTitle: hdr.Name}, + }, fileMeta, nil +} + +func (p *Pusher) uploadSnapshotConfig(ctx context.Context, name string, cfg *snapshotExportConfig, files map[string]manifest.SnapshotFile) (manifest.Descriptor, error) { + cfgBlob := manifest.SnapshotConfig{ + SchemaVersion: "v1", + SnapshotID: cfg.ID, + Description: cfg.Description, + Image: cfg.Image, + ImageDigest: cfg.ImageDigest, + ImageType: cfg.ImageType, + ImageBlobIDs: cfg.ImageBlobIDs, + Hypervisor: cfg.Hypervisor, + CPU: cfg.CPU, + Memory: cfg.Memory, + Storage: cfg.Storage, + NICs: cfg.NICs, + Network: cfg.Network, + Windows: cfg.Windows, + Files: files, + CreatedAt: nowFunc().UTC(), + } + data, err := json.Marshal(cfgBlob) + if err != nil { + return manifest.Descriptor{}, fmt.Errorf("marshal snapshot config: %w", err) + } + + digest := "sha256:" + ociutil.SHA256Hex(data) + exists, existsErr := p.Uploader.BlobExists(ctx, name, digest) + if existsErr != nil { + return manifest.Descriptor{}, fmt.Errorf("check config blob %s: %w", digest, existsErr) + } + if !exists { + if err := p.Uploader.PutBlob(ctx, name, digest, bytes.NewReader(data), int64(len(data))); err != nil { + return manifest.Descriptor{}, fmt.Errorf("put config blob: %w", err) + } + } + return manifest.Descriptor{ + MediaType: manifest.MediaTypeSnapshotConfig, + Digest: digest, + Size: int64(len(data)), + }, nil +} + +func buildSnapshotManifest(config manifest.Descriptor, layers []manifest.Descriptor, opts PushOptions) ([]byte, error) { + annotations := map[string]string{ + manifest.AnnotationCreated: nowFunc().UTC().Format(time.RFC3339), + } + if opts.BaseImage != "" { + annotations[manifest.AnnotationSnapshotBaseImage] = opts.BaseImage + } + if opts.Source != "" { + annotations[manifest.AnnotationSource] = opts.Source + } + if opts.Revision != "" { + annotations[manifest.AnnotationRevision] = opts.Revision + } + + m := manifest.OCIManifest{ + SchemaVersion: 2, + MediaType: manifest.MediaTypeOCIManifest, + ArtifactType: manifest.ArtifactTypeSnapshot, + Config: config, + Layers: layers, + Annotations: annotations, + } + return json.MarshalIndent(m, "", " ") +} diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go new file mode 100644 index 0000000..2924ba4 --- /dev/null +++ b/snapshot/snapshot.go @@ -0,0 +1,159 @@ +// Package snapshot pushes and pulls cocoon VM snapshots as OCI artifacts. +// Push streams from `cocoon snapshot export`; pull writes to `cocoon snapshot import`. +package snapshot + +import ( + "context" + "errors" + "fmt" + "io" + "os/exec" + "strings" + "time" +) + +const ( + // CocoonBinaryEnv is the env var that overrides the cocoon binary path. + CocoonBinaryEnv = "COCOON_BINARY" + snapshotJSONName = "snapshot.json" + + // cocoon uses custom PAX keys for sparse files; the stream preserves them. + sparsePAXMap = "COCOON.sparse.map" + sparsePAXSize = "COCOON.sparse.size" +) + +var ( + errMissingSnapshotJSON = errors.New("snapshot.json not found in export stream") + + nowFunc = time.Now // tests override + + _ CocoonRunner = (*ExecCocoon)(nil) +) + +// Uploader abstracts OCI blob and manifest uploads. +type Uploader interface { + BlobExists(ctx context.Context, name, digest string) (bool, error) + PutBlob(ctx context.Context, name, digest string, body io.Reader, size int64) error + PutManifest(ctx context.Context, name, tag string, data []byte, contentType string) error +} + +// Downloader abstracts OCI manifest and blob downloads. +type Downloader interface { + GetManifest(ctx context.Context, name, tag string) ([]byte, string, error) + GetBlob(ctx context.Context, name, digest string) (io.ReadCloser, error) +} + +// CocoonRunner abstracts the local `cocoon` CLI. Default is ExecCocoon; tests substitute a fake. +type CocoonRunner interface { + Export(ctx context.Context, name string) (io.ReadCloser, func() error, error) + Import(ctx context.Context, opts ImportOptions) (io.WriteCloser, func() error, error) +} + +// ImportOptions configures a cocoon snapshot import invocation. +type ImportOptions struct { + Name string + Description string +} + +// ExecCocoon runs the cocoon binary as a subprocess. +type ExecCocoon struct { + Binary string + Stderr io.Writer +} + +// Export streams a snapshot out of cocoon via `cocoon snapshot export`. +func (e *ExecCocoon) Export(ctx context.Context, name string) (io.ReadCloser, func() error, error) { + // cocoon CLI is the authoritative implementation for snapshot export; no Go library equivalent exists. + cmd := exec.CommandContext(ctx, e.Binary, "snapshot", "export", name, "-o", "-") //nolint:gosec // Binary was validated by ResolveCocoonBinary + cmd.Stderr = e.stderr() + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, fmt.Errorf("stdout pipe: %w", err) + } + if err := cmd.Start(); err != nil { + return nil, nil, fmt.Errorf("start cocoon snapshot export: %w", err) + } + return stdout, func() error { + if waitErr := cmd.Wait(); waitErr != nil { + return fmt.Errorf("cocoon snapshot export: %w", waitErr) + } + return nil + }, nil +} + +// Import starts a `cocoon snapshot import` subprocess accepting tar on stdin. +func (e *ExecCocoon) Import(ctx context.Context, opts ImportOptions) (io.WriteCloser, func() error, error) { + args := []string{"snapshot", "import", "--name", opts.Name} + if opts.Description != "" { + args = append(args, "--description", opts.Description) + } + return e.startWithStdinPipe(ctx, args, "cocoon snapshot import") +} + +// ImageImport starts a `cocoon image import` subprocess accepting data on stdin. +func (e *ExecCocoon) ImageImport(ctx context.Context, name string) (io.WriteCloser, func() error, error) { + return e.startWithStdinPipe(ctx, []string{"image", "import", name}, "cocoon image import") +} + +func (e *ExecCocoon) startWithStdinPipe(ctx context.Context, args []string, label string) (io.WriteCloser, func() error, error) { + // cocoon CLI is the authoritative implementation for snapshot/image import; no Go library equivalent exists. + cmd := exec.CommandContext(ctx, e.Binary, args...) //nolint:gosec // Binary was validated by ResolveCocoonBinary + cmd.Stdout = e.stderr() + cmd.Stderr = e.stderr() + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, nil, fmt.Errorf("stdin pipe: %w", err) + } + if err := cmd.Start(); err != nil { + _ = stdin.Close() + return nil, nil, fmt.Errorf("start %s: %w", label, err) + } + return stdin, func() error { + if waitErr := cmd.Wait(); waitErr != nil { + return fmt.Errorf("%s: %w", label, waitErr) + } + return nil + }, nil +} + +func (e *ExecCocoon) stderr() io.Writer { + if e.Stderr == nil { + return io.Discard + } + return e.Stderr +} + +// ResolveCocoonBinary finds the cocoon binary on PATH. +func ResolveCocoonBinary(envValue string) (string, error) { + bin := strings.TrimSpace(envValue) + if bin == "" { + bin = "cocoon" + } + resolved, err := exec.LookPath(bin) + if err != nil { + return "", fmt.Errorf("locate cocoon binary %q: %w", bin, err) + } + return resolved, nil +} + +type snapshotExportEnvelope struct { + Version int `json:"version"` + Config snapshotExportConfig `json:"config"` +} + +type snapshotExportConfig struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Image string `json:"image,omitempty"` + ImageDigest string `json:"image_digest,omitempty"` + ImageType string `json:"image_type,omitempty"` + ImageBlobIDs map[string]struct{} `json:"image_blob_ids,omitempty"` + Hypervisor string `json:"hypervisor,omitempty"` + CPU int `json:"cpu,omitempty"` + Memory int64 `json:"memory,omitempty"` + Storage int64 `json:"storage,omitempty"` + NICs int `json:"nics,omitempty"` + Network string `json:"network,omitempty"` + Windows bool `json:"windows,omitempty"` +} diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go new file mode 100644 index 0000000..04a38f9 --- /dev/null +++ b/snapshot/snapshot_test.go @@ -0,0 +1,552 @@ +package snapshot + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "sync" + "testing" + + "github.com/cocoonstack/cocoon-common/manifest" + "github.com/cocoonstack/cocoon-common/ociutil" +) + +// fakeUploader records every blob and manifest write so tests can assert +// the wire format produced by Pusher. +type fakeUploader struct { + mu sync.Mutex + blobs map[string][]byte // digest -> bytes + manifests map[string]fakeManifestUpload +} + +type fakeManifestUpload struct { + bytes []byte + contentType string +} + +func newFakeUploader() *fakeUploader { + return &fakeUploader{ + blobs: map[string][]byte{}, + manifests: map[string]fakeManifestUpload{}, + } +} + +func (f *fakeUploader) BlobExists(_ context.Context, _, digest string) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + _, ok := f.blobs[digest] + return ok, nil +} + +func (f *fakeUploader) PutBlob(_ context.Context, _, digest string, body io.Reader, _ int64) error { + data, err := io.ReadAll(body) + if err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + f.blobs[digest] = data + return nil +} + +func (f *fakeUploader) PutManifest(_ context.Context, name, tag string, data []byte, contentType string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.manifests[name+":"+tag] = fakeManifestUpload{bytes: data, contentType: contentType} + return nil +} + +func (f *fakeUploader) GetManifest(_ context.Context, name, tag string) ([]byte, string, error) { + f.mu.Lock() + defer f.mu.Unlock() + m, ok := f.manifests[name+":"+tag] + if !ok { + return nil, "", errors.New("not found") + } + return m.bytes, m.contentType, nil +} + +func (f *fakeUploader) GetBlob(_ context.Context, _, digest string) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + data, ok := f.blobs[digest] + if !ok { + return nil, errors.New("blob not found") + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +// fakeCocoon serves a deterministic snapshot tar from Export and captures +// the bytes Import receives so tests can assert pull-side reassembly. +type fakeCocoon struct { + exportTar []byte + importPayload bytes.Buffer + importOpts ImportOptions +} + +func (f *fakeCocoon) Export(_ context.Context, _ string) (io.ReadCloser, func() error, error) { + return io.NopCloser(bytes.NewReader(f.exportTar)), func() error { return nil }, nil +} + +func (f *fakeCocoon) Import(_ context.Context, opts ImportOptions) (io.WriteCloser, func() error, error) { + f.importOpts = opts + return &nopWriteCloser{w: &f.importPayload}, func() error { return nil }, nil +} + +type nopWriteCloser struct{ w io.Writer } + +func (n *nopWriteCloser) Write(p []byte) (int, error) { return n.w.Write(p) } +func (n *nopWriteCloser) Close() error { return nil } + +type exportTarEntry struct { + data []byte + mode int64 + pax map[string]string +} + +// buildExportTar produces a fake `cocoon snapshot export` tar containing a +// snapshot.json envelope plus the named files. +func buildExportTar(t *testing.T, cfg snapshotExportConfig, files map[string][]byte) []byte { + t.Helper() + entries := make(map[string]exportTarEntry, len(files)) + for name, data := range files { + entries[name] = exportTarEntry{data: data, mode: 0o640} + } + return buildExportTarEntries(t, cfg, entries) +} + +func buildExportTarEntries(t *testing.T, cfg snapshotExportConfig, files map[string]exportTarEntry) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + envelope := snapshotExportEnvelope{Version: 1, Config: cfg} + envBytes, err := json.Marshal(envelope) + if err != nil { + t.Fatalf("marshal envelope: %v", err) + } + if err := tw.WriteHeader(&tar.Header{Name: snapshotJSONName, Size: int64(len(envBytes)), Mode: 0o644}); err != nil { + t.Fatalf("write envelope header: %v", err) + } + if _, err := tw.Write(envBytes); err != nil { + t.Fatalf("write envelope: %v", err) + } + + // Stable order so the layer order in the produced manifest is testable. + for _, name := range []string{"config.json", "state.json", "memory-ranges", "overlay.qcow2", "cidata.img"} { + entry, ok := files[name] + if !ok { + continue + } + mode := entry.mode + if mode == 0 { + mode = 0o640 + } + hdr := &tar.Header{Name: name, Size: int64(len(entry.data)), Mode: mode, PAXRecords: entry.pax} + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write %s header: %v", name, err) + } + if _, err := tw.Write(entry.data); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + return buf.Bytes() +} + +func TestPushProducesOCISnapshotManifest(t *testing.T) { + files := map[string][]byte{ + "config.json": []byte(`{"cpu":4}`), + "state.json": []byte(`{"state":"running"}`), + "memory-ranges": []byte("memory bytes"), + "overlay.qcow2": []byte("qcow2 bytes"), + } + cfg := snapshotExportConfig{ + ID: "snap-id-1", + Name: "myvm", + Description: "snapshot desc", + Image: "ghcr.io/cocoonstack/cocoon/ubuntu:24.04", + ImageDigest: "sha256:abc123def456", + ImageType: "oci", + ImageBlobIDs: map[string]struct{}{"blob-a": {}}, + Hypervisor: "cloud-hypervisor", + CPU: 4, + Memory: 1 << 30, + Storage: 10 << 30, + NICs: 1, + Network: "dnsmasq-dhcp", + Windows: true, + } + + uploader := newFakeUploader() + cocoon := &fakeCocoon{exportTar: buildExportTar(t, cfg, files)} + pusher := &Pusher{Uploader: uploader, Cocoon: cocoon} + + result, err := pusher.Push(t.Context(), PushOptions{ + Name: "myvm", + Tag: "v1", + BaseImage: "ghcr.io/cocoonstack/cocoon/ubuntu:24.04", + }) + if err != nil { + t.Fatalf("Push: %v", err) + } + + // Manifest was uploaded with the right key + content type. + upload, ok := uploader.manifests["myvm:v1"] + if !ok { + t.Fatalf("manifest myvm:v1 not uploaded") + } + if upload.contentType != manifest.MediaTypeOCIManifest { + t.Errorf("manifest content-type = %q, want %q", upload.contentType, manifest.MediaTypeOCIManifest) + } + if !bytes.Equal(result.ManifestBytes, upload.bytes) { + t.Errorf("PushResult.ManifestBytes does not match what was uploaded") + } + + // Parse the manifest the pusher built and assert its OCI shape. + parsed, err := manifest.Parse(upload.bytes) + if err != nil { + t.Fatalf("parse manifest: %v", err) + } + if parsed.ArtifactType != manifest.ArtifactTypeSnapshot { + t.Errorf("artifactType = %q, want %q", parsed.ArtifactType, manifest.ArtifactTypeSnapshot) + } + if parsed.Config.MediaType != manifest.MediaTypeSnapshotConfig { + t.Errorf("config mediaType = %q, want %q", parsed.Config.MediaType, manifest.MediaTypeSnapshotConfig) + } + if !strings.HasPrefix(parsed.Config.Digest, "sha256:") { + t.Errorf("config digest %q lacks sha256: prefix", parsed.Config.Digest) + } + if parsed.Annotations[manifest.AnnotationSnapshotBaseImage] != "ghcr.io/cocoonstack/cocoon/ubuntu:24.04" { + t.Errorf("baseimage annotation missing: %v", parsed.Annotations) + } + + // Layers must include all four files in tar order with the right + // mediaType and title annotation. + wantLayers := []struct { + title string + mediaType string + }{ + {"config.json", manifest.MediaTypeVMConfig}, + {"state.json", manifest.MediaTypeVMState}, + {"memory-ranges", manifest.MediaTypeVMMemory}, + {"overlay.qcow2", manifest.MediaTypeDiskQcow2}, + } + if len(parsed.Layers) != len(wantLayers) { + t.Fatalf("layers len = %d, want %d", len(parsed.Layers), len(wantLayers)) + } + for i, want := range wantLayers { + got := parsed.Layers[i] + if got.MediaType != want.mediaType { + t.Errorf("layers[%d].mediaType = %q, want %q", i, got.MediaType, want.mediaType) + } + if got.Title() != want.title { + t.Errorf("layers[%d].title = %q, want %q", i, got.Title(), want.title) + } + if !strings.HasPrefix(got.Digest, "sha256:") { + t.Errorf("layers[%d].digest %q lacks sha256: prefix", i, got.Digest) + } + } + + // The config blob the pusher uploaded must round-trip into SnapshotConfig. + configBlob, ok := uploader.blobs[parsed.Config.Digest] + if !ok { + t.Fatalf("config blob %s not uploaded", parsed.Config.Digest) + } + var snapCfg manifest.SnapshotConfig + if err := json.Unmarshal(configBlob, &snapCfg); err != nil { + t.Fatalf("decode config blob: %v", err) + } + if snapCfg.SnapshotID != "snap-id-1" || snapCfg.CPU != 4 || snapCfg.Memory != 1<<30 { + t.Errorf("config blob mismatch: %+v", snapCfg) + } + if snapCfg.Description != "snapshot desc" || snapCfg.Hypervisor != "cloud-hypervisor" || snapCfg.Network != "dnsmasq-dhcp" || !snapCfg.Windows { + t.Errorf("config blob missing extended snapshot metadata: %+v", snapCfg) + } + if _, ok := snapCfg.ImageBlobIDs["blob-a"]; !ok { + t.Errorf("config blob missing image blob IDs: %+v", snapCfg.ImageBlobIDs) + } + if snapCfg.ImageDigest != "sha256:abc123def456" { + t.Errorf("config blob ImageDigest = %q, want %q", snapCfg.ImageDigest, "sha256:abc123def456") + } + if snapCfg.ImageType != "oci" { + t.Errorf("config blob ImageType = %q, want %q", snapCfg.ImageType, "oci") + } +} + +func TestPushOmitsBaseImageAnnotationWhenEmpty(t *testing.T) { + cocoon := &fakeCocoon{exportTar: buildExportTar(t, snapshotExportConfig{Name: "myvm"}, map[string][]byte{ + "config.json": []byte(`{}`), + })} + uploader := newFakeUploader() + pusher := &Pusher{Uploader: uploader, Cocoon: cocoon} + + if _, err := pusher.Push(t.Context(), PushOptions{Name: "myvm"}); err != nil { + t.Fatalf("Push: %v", err) + } + + parsed, err := manifest.Parse(uploader.manifests["myvm:latest"].bytes) + if err != nil { + t.Fatalf("parse manifest: %v", err) + } + if _, ok := parsed.Annotations[manifest.AnnotationSnapshotBaseImage]; ok { + t.Errorf("baseimage annotation should be absent when --base-image not provided") + } +} + +func TestPullReassemblesTarFromOCISnapshot(t *testing.T) { + files := map[string][]byte{ + "config.json": []byte(`{"cpu":2}`), + "state.json": []byte(`{"state":"saved"}`), + "memory-ranges": []byte("mem-bytes"), + "overlay.qcow2": []byte("qcow-bytes"), + } + cfg := snapshotExportConfig{ + ID: "snap-id-1", + Name: "myvm", + Description: "restored desc", + Image: "ghcr.io/cocoonstack/cocoon/ubuntu:24.04", + ImageDigest: "sha256:abc123def456", + ImageType: "oci", + ImageBlobIDs: map[string]struct{}{"blob-b": {}}, + Hypervisor: "cloud-hypervisor", + CPU: 2, + Memory: 1 << 30, + NICs: 1, + Network: "dnsmasq-dhcp", + } + + uploader := newFakeUploader() + cocoon := &fakeCocoon{exportTar: buildExportTar(t, cfg, files)} + pusher := &Pusher{Uploader: uploader, Cocoon: cocoon} + if _, err := pusher.Push(t.Context(), PushOptions{Name: "myvm", Tag: "v1"}); err != nil { + t.Fatalf("seed push: %v", err) + } + + pullCocoon := &fakeCocoon{} + puller := &Puller{Downloader: uploader, Cocoon: pullCocoon} + + if err := puller.Pull(t.Context(), PullOptions{Name: "myvm", Tag: "v1", LocalName: "myvm-restored"}); err != nil { + t.Fatalf("Pull: %v", err) + } + + if pullCocoon.importOpts.Name != "myvm-restored" { + t.Errorf("import name = %q, want myvm-restored", pullCocoon.importOpts.Name) + } + + tr := tar.NewReader(&pullCocoon.importPayload) + gotFiles := map[string][]byte{} + var gotEnvelope *snapshotExportEnvelope + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read tar: %v", err) + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("read tar body: %v", err) + } + if hdr.Name == snapshotJSONName { + var envelope snapshotExportEnvelope + if err := json.Unmarshal(body, &envelope); err != nil { + t.Fatalf("decode envelope: %v", err) + } + gotEnvelope = &envelope + continue + } + gotFiles[hdr.Name] = body + } + + if gotEnvelope == nil { + t.Fatal("snapshot.json not in import stream") + } + if gotEnvelope.Config.Name != "myvm-restored" { + t.Errorf("envelope name = %q, want myvm-restored", gotEnvelope.Config.Name) + } + if gotEnvelope.Config.ID != "snap-id-1" || gotEnvelope.Config.CPU != 2 { + t.Errorf("envelope config mismatch: %+v", gotEnvelope.Config) + } + if gotEnvelope.Config.Description != "restored desc" || gotEnvelope.Config.Hypervisor != "cloud-hypervisor" || gotEnvelope.Config.Network != "dnsmasq-dhcp" { + t.Errorf("envelope extended metadata mismatch: %+v", gotEnvelope.Config) + } + if _, ok := gotEnvelope.Config.ImageBlobIDs["blob-b"]; !ok { + t.Errorf("envelope image blob IDs missing: %+v", gotEnvelope.Config.ImageBlobIDs) + } + if gotEnvelope.Config.ImageDigest != "sha256:abc123def456" { + t.Errorf("envelope ImageDigest = %q, want %q", gotEnvelope.Config.ImageDigest, "sha256:abc123def456") + } + if gotEnvelope.Config.ImageType != "oci" { + t.Errorf("envelope ImageType = %q, want %q", gotEnvelope.Config.ImageType, "oci") + } + + for name, want := range files { + if got, ok := gotFiles[name]; !ok { + t.Errorf("import stream missing %s", name) + } else if !bytes.Equal(got, want) { + t.Errorf("import stream %s = %q, want %q", name, got, want) + } + } +} + +func TestPullPreservesSparseTarMetadata(t *testing.T) { + sparseMap := `[{"o":0,"l":4},{"o":12,"l":4}]` + entries := map[string]exportTarEntry{ + "config.json": {data: []byte(`{"cpu":2}`), mode: 0o640}, + "memory-ranges": { + data: []byte("ABCDWXYZ"), + mode: 0o640, + pax: map[string]string{ + sparsePAXMap: sparseMap, + sparsePAXSize: "16", + }, + }, + } + cfg := snapshotExportConfig{ + ID: "snap-id-sparse", + Name: "mysparse", + Hypervisor: "cloud-hypervisor", + } + + uploader := newFakeUploader() + cocoon := &fakeCocoon{exportTar: buildExportTarEntries(t, cfg, entries)} + pusher := &Pusher{Uploader: uploader, Cocoon: cocoon} + if _, err := pusher.Push(t.Context(), PushOptions{Name: "mysparse", Tag: "latest"}); err != nil { + t.Fatalf("seed push: %v", err) + } + + pullCocoon := &fakeCocoon{} + puller := &Puller{Downloader: uploader, Cocoon: pullCocoon} + if err := puller.Pull(t.Context(), PullOptions{Name: "mysparse", Tag: "latest", LocalName: "mysparse-restored"}); err != nil { + t.Fatalf("Pull: %v", err) + } + + tr := tar.NewReader(&pullCocoon.importPayload) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read tar: %v", err) + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("read tar body: %v", err) + } + if hdr.Name != "memory-ranges" { + continue + } + if got := hdr.PAXRecords[sparsePAXMap]; got != sparseMap { + t.Fatalf("sparse map = %q, want %q", got, sparseMap) + } + if got := hdr.PAXRecords[sparsePAXSize]; got != "16" { + t.Fatalf("sparse size = %q, want 16", got) + } + if !bytes.Equal(body, []byte("ABCDWXYZ")) { + t.Fatalf("sparse body = %q, want ABCDWXYZ", body) + } + return + } + + t.Fatal("memory-ranges entry not found") +} + +func TestPullRejectsNonSnapshotManifest(t *testing.T) { + uploader := newFakeUploader() + containerManifest := []byte(`{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:00","size":1}, + "layers": [] + }`) + uploader.manifests["foo:latest"] = fakeManifestUpload{ + bytes: containerManifest, + contentType: manifest.MediaTypeOCIManifest, + } + + puller := &Puller{Downloader: uploader, Cocoon: &fakeCocoon{}} + err := puller.Pull(t.Context(), PullOptions{Name: "foo", Tag: "latest"}) + if err == nil { + t.Fatal("expected error pulling container manifest as snapshot") + } + if !strings.Contains(err.Error(), "not a snapshot") { + t.Errorf("error = %v, want %q substring", err, "not a snapshot") + } +} + +func TestPushRequiresName(t *testing.T) { + pusher := &Pusher{Uploader: newFakeUploader(), Cocoon: &fakeCocoon{}} + _, err := pusher.Push(t.Context(), PushOptions{}) + if err == nil { + t.Fatal("expected error when name is empty") + } +} + +func TestFetchSnapshotConfigOverOneMiB(t *testing.T) { + uploader := newFakeUploader() + + mkMap := func(n int) string { + parts := make([]string, n) + for i := range parts { + parts[i] = `{"o":1234567890,"l":4096}` + } + return "[" + strings.Join(parts, ",") + "]" + } + cfg := manifest.SnapshotConfig{ + SchemaVersion: "v1", + SnapshotID: "01J0", + Hypervisor: "cloud-hypervisor", + Files: map[string]manifest.SnapshotFile{ + "memory-ranges": {Mode: 0o644, SparseMap: mkMap(25_000), SparseSize: 1 << 32}, + "overlay.qcow2": {Mode: 0o644, SparseMap: mkMap(25_000), SparseSize: 1 << 33}, + }, + } + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + if len(data) <= 1<<20 { + t.Fatalf("test setup: config %d bytes is not over the old 1 MiB cap", len(data)) + } + + digest := "sha256:" + ociutil.SHA256Hex(data) + uploader.blobs[digest] = data + desc := manifest.Descriptor{ + MediaType: manifest.MediaTypeSnapshotConfig, + Digest: digest, + Size: int64(len(data)), + } + + got, err := FetchSnapshotConfig(t.Context(), uploader, "vm-x", desc) + if err != nil { + t.Fatalf("FetchSnapshotConfig: %v", err) + } + if got.SnapshotID != cfg.SnapshotID { + t.Errorf("SnapshotID = %q, want %q", got.SnapshotID, cfg.SnapshotID) + } + if len(got.Files) != 2 { + t.Errorf("Files count = %d, want 2", len(got.Files)) + } +} + +func TestFetchSnapshotConfigRejectsOversizeDescriptor(t *testing.T) { + uploader := newFakeUploader() + desc := manifest.Descriptor{ + MediaType: manifest.MediaTypeSnapshotConfig, + Digest: "sha256:00", + Size: maxSnapshotConfigSize + 1, + } + _, err := FetchSnapshotConfig(t.Context(), uploader, "vm-x", desc) + if err == nil || !strings.Contains(err.Error(), "config blob too large") { + t.Fatalf("err = %v, want 'config blob too large' substring", err) + } +} From 6e675f810e8d3a1346ba04d3cda65175195c60c9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 1 Jul 2026 13:07:23 +0800 Subject: [PATCH 3/4] feat: add oci package (shared standard-OCI registry client) A go-containerregistry-backed Registry (Uploader/Downloader + HasManifest + DeleteManifest) that vk and the operator share to push/pull snapshots and cloud images and probe/roll back manifests against any OCI registry (e.g. Artifact Registry) with keychain auth. streamLayer pushes multi-GB blobs without buffering. --- go.mod | 23 ++++--- go.sum | 64 ++++++++++------- oci/oci.go | 177 ++++++++++++++++++++++++++++++++++++++++++++++++ oci/oci_test.go | 84 +++++++++++++++++++++++ oci/registry.go | 19 ++++++ 5 files changed, 335 insertions(+), 32 deletions(-) create mode 100644 oci/oci.go create mode 100644 oci/oci_test.go create mode 100644 oci/registry.go diff --git a/go.mod b/go.mod index 44a18ab..2a36ce6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cocoonstack/cocoon-common go 1.25.6 require ( + github.com/google/go-containerregistry v0.21.7 github.com/projecteru2/core v0.0.0-20241016125006-ff909eefe04c k8s.io/api v0.35.3 k8s.io/apimachinery v0.35.3 @@ -17,6 +18,8 @@ require ( github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/cli v29.5.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fatih/color v1.18.0 // indirect @@ -33,6 +36,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -42,24 +46,27 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/zerolog v1.29.1 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/mod v0.31.0 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.46.0 // indirect google.golang.org/grpc v1.72.2 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/go.sum b/go.sum index 5ca8d2b..94e9fa3 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/cli v29.5.3+incompatible h1:nbEFfz774vBwQ5KRYv7c/AghjReqnGISvrRhzjV0evs= +github.com/docker/cli v29.5.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= @@ -137,6 +141,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.7 h1:/vPFuVXDjtFREsVArW+0h1CIl5urnOhzei4X2DMW9IU= +github.com/google/go-containerregistry v0.21.7/go.mod h1:kjSbt7/zMsKLWfnHrIvKvhXHUw91jbe9DNjPPJ32gXE= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= @@ -179,6 +185,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -241,6 +249,10 @@ github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zw github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -275,6 +287,8 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -328,22 +342,22 @@ github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -373,8 +387,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -392,11 +406,11 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -404,8 +418,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -429,19 +443,19 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -458,8 +472,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk= +golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -529,6 +543,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ= diff --git a/oci/oci.go b/oci/oci.go new file mode 100644 index 0000000..110a63e --- /dev/null +++ b/oci/oci.go @@ -0,0 +1,177 @@ +package oci + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var ( + _ Registry = (*OCIRegistry)(nil) + + // errBlobUncompressed guards the DiffID/Uncompressed accessors: cocoon blobs + // are opaque content-addressed bytes and WriteLayer only reads Compressed(). + errBlobUncompressed = errors.New("cocoon blob layers expose only compressed bytes") +) + +// OCIRegistry is a Registry backed by a standard OCI Distribution registry +// (e.g. Artifact Registry), using OCI upload sessions and keychain auth. +type OCIRegistry struct { + base string // registry host + repo prefix, e.g. "asia-docker.pkg.dev/proj/repo" + opts []remote.Option +} + +// NewOCIRegistry roots a client at base, authenticating via keychain (e.g. +// authn.DefaultKeychain, or a MultiKeychain with google.Keychain for GCP AR). +func NewOCIRegistry(base string, keychain authn.Keychain) *OCIRegistry { + return &OCIRegistry{base: base, opts: []remote.Option{remote.WithAuthFromKeychain(keychain)}} +} + +// GetManifest fetches the raw manifest bytes and media type at repo:tag. +func (r *OCIRegistry) GetManifest(ctx context.Context, repo, tag string) ([]byte, string, error) { + ref, err := name.ParseReference(r.base + "/" + repo + ":" + tag) + if err != nil { + return nil, "", fmt.Errorf("parse ref %s:%s: %w", repo, tag, err) + } + desc, err := remote.Get(ref, r.callOpts(ctx)...) + if err != nil { + return nil, "", fmt.Errorf("get manifest %s:%s: %w", repo, tag, err) + } + return desc.Manifest, string(desc.MediaType), nil +} + +// GetBlob streams the blob at the given digest. +func (r *OCIRegistry) GetBlob(ctx context.Context, repo, digest string) (io.ReadCloser, error) { + ref, err := name.NewDigest(r.base + "/" + repo + "@" + digest) + if err != nil { + return nil, fmt.Errorf("parse digest %s@%s: %w", repo, digest, err) + } + layer, err := remote.Layer(ref, r.callOpts(ctx)...) + if err != nil { + return nil, fmt.Errorf("get blob %s@%s: %w", repo, digest, err) + } + return layer.Compressed() +} + +// BlobExists reports whether the blob is already present, so pushes can skip it. +func (r *OCIRegistry) BlobExists(ctx context.Context, repo, digest string) (bool, error) { + ref, err := name.NewDigest(r.base + "/" + repo + "@" + digest) + if err != nil { + return false, fmt.Errorf("parse digest %s@%s: %w", repo, digest, err) + } + // remote.Layer is lazy; Size() issues the HEAD that reveals whether it exists. + layer, err := remote.Layer(ref, r.callOpts(ctx)...) + if err == nil { + _, err = layer.Size() + } + if err == nil { + return true, nil + } + return false, ignoreNotFound(err, "head blob "+repo+"@"+digest) +} + +// HasManifest reports whether a manifest exists at repo:tag. +func (r *OCIRegistry) HasManifest(ctx context.Context, repo, tag string) (bool, error) { + ref, err := name.ParseReference(r.base + "/" + repo + ":" + tag) + if err != nil { + return false, fmt.Errorf("parse ref %s:%s: %w", repo, tag, err) + } + if _, err := remote.Head(ref, r.callOpts(ctx)...); err != nil { + return false, ignoreNotFound(err, "head manifest "+repo+":"+tag) + } + return true, nil +} + +// PutBlob uploads a blob of the given digest/size via a standard upload session. +func (r *OCIRegistry) PutBlob(ctx context.Context, repo, digest string, body io.Reader, size int64) error { + repoRef, err := name.NewRepository(r.base + "/" + repo) + if err != nil { + return fmt.Errorf("parse repo %s: %w", repo, err) + } + hash, err := v1.NewHash(digest) + if err != nil { + return fmt.Errorf("parse digest %s: %w", digest, err) + } + if err := remote.WriteLayer(repoRef, &streamLayer{hash: hash, size: size, body: body}, r.callOpts(ctx)...); err != nil { + return fmt.Errorf("put blob %s@%s: %w", repo, digest, err) + } + return nil +} + +// PutManifest uploads a manifest at repo:tag with the given content type. +func (r *OCIRegistry) PutManifest(ctx context.Context, repo, tag string, data []byte, contentType string) error { + ref, err := name.ParseReference(r.base + "/" + repo + ":" + tag) + if err != nil { + return fmt.Errorf("parse ref %s:%s: %w", repo, tag, err) + } + if err := remote.Put(ref, rawManifest{data: data, mediaType: types.MediaType(contentType)}, r.callOpts(ctx)...); err != nil { + return fmt.Errorf("put manifest %s:%s: %w", repo, tag, err) + } + return nil +} + +// DeleteManifest removes the manifest at repo:reference (tag or digest). +func (r *OCIRegistry) DeleteManifest(ctx context.Context, repo, reference string) error { + // A digest (sha256:...) joins the repo with '@'; a tag with ':'. + sep := ":" + if strings.ContainsRune(reference, ':') { + sep = "@" + } + ref, err := name.ParseReference(r.base + "/" + repo + sep + reference) + if err != nil { + return fmt.Errorf("parse ref %s%s%s: %w", repo, sep, reference, err) + } + if err := remote.Delete(ref, r.callOpts(ctx)...); err != nil { + return fmt.Errorf("delete manifest %s: %w", reference, err) + } + return nil +} + +func (r *OCIRegistry) callOpts(ctx context.Context) []remote.Option { + return append(r.opts, remote.WithContext(ctx)) +} + +// ignoreNotFound maps a registry 404 to a nil error (absent, not failed) and +// wraps anything else. +func ignoreNotFound(err error, action string) error { + var terr *transport.Error + if errors.As(err, &terr) && terr.StatusCode == http.StatusNotFound { + return nil + } + return fmt.Errorf("%s: %w", action, err) +} + +// streamLayer is a v1.Layer over a body with a known digest and size, so PutBlob +// streams a raw blob without buffering it (WriteLayer reads only Compressed()). +// body is single-use: a retried upload fails the digest check, not corrupts. +type streamLayer struct { + hash v1.Hash + size int64 + body io.Reader +} + +func (l *streamLayer) Digest() (v1.Hash, error) { return l.hash, nil } +func (l *streamLayer) Size() (int64, error) { return l.size, nil } +func (l *streamLayer) Compressed() (io.ReadCloser, error) { return io.NopCloser(l.body), nil } +func (l *streamLayer) MediaType() (types.MediaType, error) { return types.OCILayer, nil } +func (l *streamLayer) DiffID() (v1.Hash, error) { return v1.Hash{}, errBlobUncompressed } +func (l *streamLayer) Uncompressed() (io.ReadCloser, error) { return nil, errBlobUncompressed } + +// rawManifest is a remote.Taggable over pre-serialized manifest bytes. +type rawManifest struct { + data []byte + mediaType types.MediaType +} + +func (m rawManifest) RawManifest() ([]byte, error) { return m.data, nil } +func (m rawManifest) MediaType() (types.MediaType, error) { return m.mediaType, nil } diff --git a/oci/oci_test.go b/oci/oci_test.go new file mode 100644 index 0000000..d488c0f --- /dev/null +++ b/oci/oci_test.go @@ -0,0 +1,84 @@ +package oci + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/registry" +) + +// TestOCIRegistryRoundTrip exercises the full Registry surface against an +// in-memory OCI registry: a blob and a custom-artifactType manifest survive a +// put -> exists -> get -> delete round trip. +func TestOCIRegistryRoundTrip(t *testing.T) { + srv := httptest.NewServer(registry.New()) + t.Cleanup(srv.Close) + + r := NewOCIRegistry(strings.TrimPrefix(srv.URL, "http://")+"/cocoon", authn.DefaultKeychain) + ctx := context.Background() + + blob := []byte("hello cocoon blob") + sum := sha256.Sum256(blob) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + if ok, err := r.BlobExists(ctx, "myvm", digest); err != nil || ok { + t.Fatalf("BlobExists before put = (%v, %v), want (false, nil)", ok, err) + } + if err := r.PutBlob(ctx, "myvm", digest, bytes.NewReader(blob), int64(len(blob))); err != nil { + t.Fatalf("PutBlob: %v", err) + } + if ok, err := r.BlobExists(ctx, "myvm", digest); err != nil || !ok { + t.Fatalf("BlobExists after put = (%v, %v), want (true, nil)", ok, err) + } + + rc, err := r.GetBlob(ctx, "myvm", digest) + if err != nil { + t.Fatalf("GetBlob: %v", err) + } + got, _ := io.ReadAll(rc) + _ = rc.Close() + if !bytes.Equal(got, blob) { + t.Fatalf("GetBlob = %q, want %q", got, blob) + } + + const mt = "application/vnd.oci.image.manifest.v1+json" + mf := []byte(`{"schemaVersion":2,"mediaType":"` + mt + + `","artifactType":"application/vnd.cocoonstack.snapshot.v1+json","config":{"mediaType":` + + `"application/vnd.cocoonstack.snapshot.config.v1+json","digest":"` + digest + + `","size":` + strconv.Itoa(len(blob)) + `},"layers":[]}`) + + if ok, err := r.HasManifest(ctx, "myvm", "hibernate"); err != nil || ok { + t.Fatalf("HasManifest before put = (%v, %v), want (false, nil)", ok, err) + } + if err := r.PutManifest(ctx, "myvm", "hibernate", mf, mt); err != nil { + t.Fatalf("PutManifest: %v", err) + } + if ok, err := r.HasManifest(ctx, "myvm", "hibernate"); err != nil || !ok { + t.Fatalf("HasManifest after put = (%v, %v), want (true, nil)", ok, err) + } + raw, gotMT, err := r.GetManifest(ctx, "myvm", "hibernate") + if err != nil { + t.Fatalf("GetManifest: %v", err) + } + if !bytes.Equal(raw, mf) { + t.Fatalf("GetManifest bytes mismatch") + } + if gotMT != mt { + t.Fatalf("GetManifest mediaType = %q, want %q", gotMT, mt) + } + + if err := r.DeleteManifest(ctx, "myvm", "hibernate"); err != nil { + t.Fatalf("DeleteManifest: %v", err) + } + if _, _, err := r.GetManifest(ctx, "myvm", "hibernate"); err == nil { + t.Fatal("GetManifest after delete: want error, got nil") + } +} diff --git a/oci/registry.go b/oci/registry.go new file mode 100644 index 0000000..a643883 --- /dev/null +++ b/oci/registry.go @@ -0,0 +1,19 @@ +// Package oci provides a standard OCI Distribution registry client that +// satisfies the snapshot Uploader/Downloader contracts, so cocoon snapshots +// and cloud images can live in any OCI registry (e.g. Artifact Registry). +package oci + +import ( + "context" + + "github.com/cocoonstack/cocoon-common/snapshot" +) + +// Registry is the OCI backend shared by vk (push/pull) and the operator +// (existence probe + rollback). +type Registry interface { + snapshot.Uploader + snapshot.Downloader + HasManifest(ctx context.Context, repo, tag string) (bool, error) + DeleteManifest(ctx context.Context, repo, reference string) error +} From fd96a6d3865b689690c5fe933fd85d10cdf02e31 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 1 Jul 2026 14:20:02 +0800 Subject: [PATCH 4/4] refactor: address cocoon /code self-audit Rename to the cocoon idiom: Uploader.BlobExists -> HasBlob (pairs with HasManifest) and CocoonRunner.ImageImport -> ImportImage (verb-noun), across interfaces, impls, callers, and fakes. Plus test fixes: t.Context() over context.Background(), hoist the cloudimg-test const/var above the type, and compare io.EOF with errors.Is. --- cloudimg/cloudimg_test.go | 24 ++++++++++++------------ cloudimg/pull.go | 4 ++-- oci/oci.go | 4 ++-- oci/oci_test.go | 11 +++++------ snapshot/push.go | 4 ++-- snapshot/snapshot.go | 6 +++--- snapshot/snapshot_test.go | 6 +++--- 7 files changed, 29 insertions(+), 30 deletions(-) diff --git a/cloudimg/cloudimg_test.go b/cloudimg/cloudimg_test.go index 7f31332..439c827 100644 --- a/cloudimg/cloudimg_test.go +++ b/cloudimg/cloudimg_test.go @@ -11,17 +11,6 @@ import ( "github.com/cocoonstack/cocoon-common/manifest" ) -// fakeBlobs is a tiny in-memory BlobReader for tests. -type fakeBlobs map[string][]byte - -func (f fakeBlobs) ReadBlob(_ context.Context, digest string) (io.ReadCloser, error) { - data, ok := f[digest] - if !ok { - return nil, errors.New("blob not found: " + digest) - } - return io.NopCloser(bytes.NewReader(data)), nil -} - const ( diskBlobA = "AAAA" diskBlobB = "BBBB" @@ -57,6 +46,17 @@ var winManifest = `{ ] }` +// fakeBlobs is a tiny in-memory BlobReader for tests. +type fakeBlobs map[string][]byte + +func (f fakeBlobs) ReadBlob(_ context.Context, digest string) (io.ReadCloser, error) { + data, ok := f[digest] + if !ok { + return nil, errors.New("blob not found: " + digest) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + func TestStreamConcatenatesDiskLayersInTitleOrder(t *testing.T) { blobs := fakeBlobs{ digestA: []byte(diskBlobA), @@ -129,7 +129,7 @@ type fakeCocoon struct { importName string } -func (f *fakeCocoon) ImageImport(_ context.Context, name string) (io.WriteCloser, func() error, error) { +func (f *fakeCocoon) ImportImage(_ context.Context, name string) (io.WriteCloser, func() error, error) { f.importName = name return nopCloser{w: &f.importPayload}, func() error { return nil }, nil } diff --git a/cloudimg/pull.go b/cloudimg/pull.go index 63060ac..1e87127 100644 --- a/cloudimg/pull.go +++ b/cloudimg/pull.go @@ -17,7 +17,7 @@ type Downloader interface { // CocoonRunner abstracts the cocoon image import subprocess. type CocoonRunner interface { - ImageImport(ctx context.Context, name string) (io.WriteCloser, func() error, error) + ImportImage(ctx context.Context, name string) (io.WriteCloser, func() error, error) } // Puller downloads cloud-image artifacts and pipes them into cocoon image import. @@ -51,7 +51,7 @@ func (p *Puller) Pull(ctx context.Context, opts PullOptions) error { return fmt.Errorf("get manifest %s:%s: %w", opts.Name, opts.Tag, err) } - stdin, wait, err := p.Cocoon.ImageImport(ctx, localName) + stdin, wait, err := p.Cocoon.ImportImage(ctx, localName) if err != nil { return fmt.Errorf("start cocoon image import: %w", err) } diff --git a/oci/oci.go b/oci/oci.go index 110a63e..c91bfc7 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -63,8 +63,8 @@ func (r *OCIRegistry) GetBlob(ctx context.Context, repo, digest string) (io.Read return layer.Compressed() } -// BlobExists reports whether the blob is already present, so pushes can skip it. -func (r *OCIRegistry) BlobExists(ctx context.Context, repo, digest string) (bool, error) { +// HasBlob reports whether the blob is already present, so pushes can skip it. +func (r *OCIRegistry) HasBlob(ctx context.Context, repo, digest string) (bool, error) { ref, err := name.NewDigest(r.base + "/" + repo + "@" + digest) if err != nil { return false, fmt.Errorf("parse digest %s@%s: %w", repo, digest, err) diff --git a/oci/oci_test.go b/oci/oci_test.go index d488c0f..1536c9f 100644 --- a/oci/oci_test.go +++ b/oci/oci_test.go @@ -2,7 +2,6 @@ package oci import ( "bytes" - "context" "crypto/sha256" "encoding/hex" "io" @@ -23,20 +22,20 @@ func TestOCIRegistryRoundTrip(t *testing.T) { t.Cleanup(srv.Close) r := NewOCIRegistry(strings.TrimPrefix(srv.URL, "http://")+"/cocoon", authn.DefaultKeychain) - ctx := context.Background() + ctx := t.Context() blob := []byte("hello cocoon blob") sum := sha256.Sum256(blob) digest := "sha256:" + hex.EncodeToString(sum[:]) - if ok, err := r.BlobExists(ctx, "myvm", digest); err != nil || ok { - t.Fatalf("BlobExists before put = (%v, %v), want (false, nil)", ok, err) + if ok, err := r.HasBlob(ctx, "myvm", digest); err != nil || ok { + t.Fatalf("HasBlob before put = (%v, %v), want (false, nil)", ok, err) } if err := r.PutBlob(ctx, "myvm", digest, bytes.NewReader(blob), int64(len(blob))); err != nil { t.Fatalf("PutBlob: %v", err) } - if ok, err := r.BlobExists(ctx, "myvm", digest); err != nil || !ok { - t.Fatalf("BlobExists after put = (%v, %v), want (true, nil)", ok, err) + if ok, err := r.HasBlob(ctx, "myvm", digest); err != nil || !ok { + t.Fatalf("HasBlob after put = (%v, %v), want (true, nil)", ok, err) } rc, err := r.GetBlob(ctx, "myvm", digest) diff --git a/snapshot/push.go b/snapshot/push.go index 9473c66..e8396b4 100644 --- a/snapshot/push.go +++ b/snapshot/push.go @@ -170,7 +170,7 @@ func (p *Pusher) uploadTarEntry(ctx context.Context, name string, hdr *tar.Heade digestHex := hex.EncodeToString(h.Sum(nil)) digest := "sha256:" + digestHex - exists, existsErr := p.Uploader.BlobExists(ctx, name, digest) + exists, existsErr := p.Uploader.HasBlob(ctx, name, digest) if existsErr != nil { return manifest.Descriptor{}, manifest.SnapshotFile{}, fmt.Errorf("check blob %s: %w", digest, existsErr) } @@ -231,7 +231,7 @@ func (p *Pusher) uploadSnapshotConfig(ctx context.Context, name string, cfg *sna } digest := "sha256:" + ociutil.SHA256Hex(data) - exists, existsErr := p.Uploader.BlobExists(ctx, name, digest) + exists, existsErr := p.Uploader.HasBlob(ctx, name, digest) if existsErr != nil { return manifest.Descriptor{}, fmt.Errorf("check config blob %s: %w", digest, existsErr) } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 2924ba4..036a8f6 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -32,7 +32,7 @@ var ( // Uploader abstracts OCI blob and manifest uploads. type Uploader interface { - BlobExists(ctx context.Context, name, digest string) (bool, error) + HasBlob(ctx context.Context, name, digest string) (bool, error) PutBlob(ctx context.Context, name, digest string, body io.Reader, size int64) error PutManifest(ctx context.Context, name, tag string, data []byte, contentType string) error } @@ -90,8 +90,8 @@ func (e *ExecCocoon) Import(ctx context.Context, opts ImportOptions) (io.WriteCl return e.startWithStdinPipe(ctx, args, "cocoon snapshot import") } -// ImageImport starts a `cocoon image import` subprocess accepting data on stdin. -func (e *ExecCocoon) ImageImport(ctx context.Context, name string) (io.WriteCloser, func() error, error) { +// ImportImage starts a `cocoon image import` subprocess accepting data on stdin. +func (e *ExecCocoon) ImportImage(ctx context.Context, name string) (io.WriteCloser, func() error, error) { return e.startWithStdinPipe(ctx, []string{"image", "import", name}, "cocoon image import") } diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index 04a38f9..d9a9b99 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -35,7 +35,7 @@ func newFakeUploader() *fakeUploader { } } -func (f *fakeUploader) BlobExists(_ context.Context, _, digest string) (bool, error) { +func (f *fakeUploader) HasBlob(_ context.Context, _, digest string) (bool, error) { f.mu.Lock() defer f.mu.Unlock() _, ok := f.blobs[digest] @@ -345,7 +345,7 @@ func TestPullReassemblesTarFromOCISnapshot(t *testing.T) { var gotEnvelope *snapshotExportEnvelope for { hdr, err := tr.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } if err != nil { @@ -432,7 +432,7 @@ func TestPullPreservesSparseTarMetadata(t *testing.T) { tr := tar.NewReader(&pullCocoon.importPayload) for { hdr, err := tr.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } if err != nil {