From d45441d8e9ae4e51f3f63697e5d37b7ef160528b Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 18:35:28 +0300 Subject: [PATCH 01/14] remove old functions Signed-off-by: Pavel Okhlopkov --- internal/mirror/cmd/pull/pull.go | 114 +++++++++++++------------------ internal/mirror/cmd/push/push.go | 15 +--- 2 files changed, 50 insertions(+), 79 deletions(-) diff --git a/internal/mirror/cmd/pull/pull.go b/internal/mirror/cmd/pull/pull.go index 57b6ec97..a920e06e 100644 --- a/internal/mirror/cmd/pull/pull.go +++ b/internal/mirror/cmd/pull/pull.go @@ -246,83 +246,67 @@ func (p *Puller) Execute(ctx context.Context) error { return err } - if os.Getenv("NEW_PULL") == "true" { - logger := dkplog.NewNop() - - if log.DebugLogLevel() >= 3 { - logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug)) - } - - // Create registry client for module operations - clientOpts := ®client.Options{ - Insecure: p.params.Insecure, - TLSSkipVerify: p.params.SkipTLSVerification, - Logger: logger, - } - - if p.params.RegistryAuth != nil { - clientOpts.Auth = p.params.RegistryAuth - } - - var c registry.Client - c = regclient.NewClientWithOptions(p.params.DeckhouseRegistryRepo, clientOpts) + logger := dkplog.NewNop() - if os.Getenv("STUB_REGISTRY_CLIENT") == "true" { - c = stub.NewRegistryClientStub() - } + if log.DebugLogLevel() >= 3 { + logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug)) + } - // Scope to the registry path and modules suffix - if p.params.RegistryPath != "" { - c = c.WithSegment(p.params.RegistryPath) - } + // Create registry client for module operations + clientOpts := ®client.Options{ + Insecure: p.params.Insecure, + TLSSkipVerify: p.params.SkipTLSVerification, + Logger: logger, + } - // Create module filter from CLI flags - filter, err := p.createModuleFilter() - if err != nil { - return err - } + if p.params.RegistryAuth != nil { + clientOpts.Auth = p.params.RegistryAuth + } - svc := mirror.NewPullService( - registryservice.NewService(c, logger), - pullflags.TempDir, - pullflags.DeckhouseTag, - &mirror.PullServiceOptions{ - SkipPlatform: pullflags.NoPlatform, - SkipSecurity: pullflags.NoSecurityDB, - SkipModules: pullflags.NoModules, - OnlyExtraImages: pullflags.OnlyExtraImages, - IgnoreSuspend: pullflags.IgnoreSuspend, - ModuleFilter: filter, - BundleDir: pullflags.ImagesBundlePath, - BundleChunkSize: pullflags.ImagesBundleChunkSizeGB * 1000 * 1000 * 1000, - }, - logger.Named("pull"), - p.logger, - ) - - err = svc.Pull(ctx) - if err != nil { - // Handle context cancellation gracefully - if errors.Is(err, context.Canceled) { - p.logger.WarnLn("Operation cancelled by user") - return nil - } - return fmt.Errorf("pull from registry: %w", err) - } + var c registry.Client + c = regclient.NewClientWithOptions(p.params.DeckhouseRegistryRepo, clientOpts) - return nil + if os.Getenv("STUB_REGISTRY_CLIENT") == "true" { + c = stub.NewRegistryClientStub() } - if err := p.pullPlatform(); err != nil { - return err + // Scope to the registry path and modules suffix + if p.params.RegistryPath != "" { + c = c.WithSegment(p.params.RegistryPath) } - if err := p.pullSecurityDatabases(); err != nil { + // Create module filter from CLI flags + filter, err := p.createModuleFilter() + if err != nil { return err } - if err := p.pullModules(); err != nil { - return err + svc := mirror.NewPullService( + registryservice.NewService(c, logger), + pullflags.TempDir, + pullflags.DeckhouseTag, + &mirror.PullServiceOptions{ + SkipPlatform: pullflags.NoPlatform, + SkipSecurity: pullflags.NoSecurityDB, + SkipModules: pullflags.NoModules, + OnlyExtraImages: pullflags.OnlyExtraImages, + IgnoreSuspend: pullflags.IgnoreSuspend, + ModuleFilter: filter, + BundleDir: pullflags.ImagesBundlePath, + BundleChunkSize: pullflags.ImagesBundleChunkSizeGB * 1000 * 1000 * 1000, + }, + logger.Named("pull"), + p.logger, + ) + + err = svc.Pull(ctx) + if err != nil { + // Handle context cancellation gracefully + if errors.Is(err, context.Canceled) { + p.logger.WarnLn("Operation cancelled by user") + return nil + } + return fmt.Errorf("pull from registry: %w", err) } if err := p.computeGOSTDigests(); err != nil { diff --git a/internal/mirror/cmd/push/push.go b/internal/mirror/cmd/push/push.go index 2bf9e68e..314bb9d0 100644 --- a/internal/mirror/cmd/push/push.go +++ b/internal/mirror/cmd/push/push.go @@ -292,20 +292,7 @@ func (p *Pusher) Execute() error { return err } - // Use new push service when NEW_PULL env is set - if os.Getenv("NEW_PULL") == "true" { - return p.executeNewPush() - } - - if err := p.pushStaticPackages(); err != nil { - return err - } - - if err := p.pushModules(); err != nil { - return err - } - - return nil + return p.executeNewPush() } // executeNewPush runs the push using the push service. From a61a7073d5e4fecbd3ae048c1dfdd8793b9743ec Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 18:44:07 +0300 Subject: [PATCH 02/14] remove old Signed-off-by: Pavel Okhlopkov --- internal/mirror/cmd/pull/pull.go | 191 -------------------------- internal/mirror/cmd/pull/pull_test.go | 117 ---------------- internal/mirror/cmd/push/push.go | 178 ------------------------ 3 files changed, 486 deletions(-) diff --git a/internal/mirror/cmd/pull/pull.go b/internal/mirror/cmd/pull/pull.go index a920e06e..d6625d35 100644 --- a/internal/mirror/cmd/pull/pull.go +++ b/internal/mirror/cmd/pull/pull.go @@ -29,7 +29,6 @@ import ( "syscall" "time" - "github.com/Masterminds/semver/v3" "github.com/google/go-containerregistry/pkg/authn" "github.com/hashicorp/go-multierror" "github.com/samber/lo" @@ -44,8 +43,6 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/mirror" pullflags "github.com/deckhouse/deckhouse-cli/internal/mirror/cmd/pull/flags" "github.com/deckhouse/deckhouse-cli/internal/mirror/gostsums" - "github.com/deckhouse/deckhouse-cli/internal/mirror/operations" - "github.com/deckhouse/deckhouse-cli/internal/mirror/releases" "github.com/deckhouse/deckhouse-cli/internal/version" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/modules" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" @@ -135,24 +132,6 @@ func setupLogger() *log.SLogger { return log.NewSLogger(logLevel) } -func findTagsToMirror(pullParams *params.PullParams, logger *log.SLogger, client registry.Client) ([]string, []string, error) { - strickTags := []string{} - if pullParams.DeckhouseTag != "" { - strickTags = append(strickTags, pullParams.DeckhouseTag) - } - - versionsToMirror, channelsToMirror, err := versionsToMirrorFunc(pullParams, client, strickTags) - if err != nil { - return nil, nil, fmt.Errorf("Find versions to mirror: %w", err) - } - - logger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) - - return lo.Map(versionsToMirror, func(v semver.Version, _ int) string { - return "v" + v.String() - }), channelsToMirror, nil -} - func buildPullParams(logger params.Logger) *params.PullParams { mirrorCtx := ¶ms.PullParams{ BaseParams: params.BaseParams{ @@ -212,9 +191,6 @@ func lastPullWasTooLongAgoToRetry(pullParams *params.PullParams) bool { return time.Since(s.ModTime()) > 24*time.Hour } -// versionsToMirrorFunc allows mocking releases.VersionsToMirror in tests -var versionsToMirrorFunc = releases.VersionsToMirror - // Puller encapsulates the logic for pulling Deckhouse components type Puller struct { cmd *cobra.Command @@ -326,59 +302,6 @@ func (p *Puller) cleanupWorkingDirectory() error { return nil } -// pullPlatform pulls the Deckhouse platform components -func (p *Puller) pullPlatform() error { - if p.params.SkipPlatform { - return nil - } - - logger := dkplog.NewNop() - - if log.DebugLogLevel() >= 3 { - logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug)) - } - - // Create registry client for module operations - clientOpts := ®client.Options{ - Insecure: p.params.Insecure, - TLSSkipVerify: p.params.SkipTLSVerification, - Logger: logger, - } - - if p.params.RegistryAuth != nil { - clientOpts.Auth = p.params.RegistryAuth - } - - var c registry.Client - c = regclient.NewClientWithOptions(p.params.DeckhouseRegistryRepo, clientOpts) - - if os.Getenv("STUB_REGISTRY_CLIENT") == "true" { - c = stub.NewRegistryClientStub() - } - - // Scope to the registry path and modules suffix - if p.params.RegistryPath != "" { - c = c.WithSegment(p.params.RegistryPath) - } - - return p.logger.Process("Pull Deckhouse Kubernetes Platform", func() error { - if err := p.validatePlatformAccess(); err != nil { - return err - } - - tagsToMirror, channelsToMirror, err := findTagsToMirror(p.params, p.logger, c) - if err != nil { - return fmt.Errorf("Find tags to mirror: %w", err) - } - - if err = operations.PullDeckhousePlatform(p.params, channelsToMirror, tagsToMirror, c); err != nil { - return err - } - - return nil - }) -} - // validatePlatformAccess validates access to the platform registry // check access with strict tag, stable or lts // if any resource found - access is ok @@ -410,120 +333,6 @@ func (p *Puller) validatePlatformAccess() error { return nil } -// pullSecurityDatabases pulls the security databases -func (p *Puller) pullSecurityDatabases() error { - if p.params.SkipSecurityDatabases { - return nil - } - - logger := dkplog.NewNop() - - if log.DebugLogLevel() >= 3 { - logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug)) - } - - // Create registry client for module operations - clientOpts := ®client.Options{ - Insecure: p.params.Insecure, - TLSSkipVerify: p.params.SkipTLSVerification, - Logger: logger, - } - - if p.params.RegistryAuth != nil { - clientOpts.Auth = p.params.RegistryAuth - } - - var c registry.Client - c = regclient.NewClientWithOptions(p.params.DeckhouseRegistryRepo, clientOpts) - - if os.Getenv("STUB_REGISTRY_CLIENT") == "true" { - c = stub.NewRegistryClientStub() - } - - // Scope to the registry path and modules suffix - if p.params.RegistryPath != "" { - c = c.WithSegment(p.params.RegistryPath) - } - - return p.logger.Process("Pull Security Databases", func() error { - ctx, cancel := context.WithTimeout(p.cmd.Context(), 15*time.Second) - defer cancel() - - imageRef := p.params.DeckhouseRegistryRepo + "/security/trivy-db:2" - err := p.accessValidator.ValidateReadAccessForImage(ctx, imageRef, p.validationOpts...) - switch { - case errors.Is(err, validation.ErrImageUnavailable): - p.logger.Warnf("Skipping pull of security databases: %v", err) - return nil - case err != nil: - return fmt.Errorf("Source registry is not accessible: %w", err) - } - - if err := operations.PullSecurityDatabases(p.params, c); err != nil { - return err - } - return nil - }) -} - -// pullModules pulls the Deckhouse modules -func (p *Puller) pullModules() error { - if p.params.SkipModules && !p.params.OnlyExtraImages { - return nil - } - - processName := "Pull Modules" - if p.params.OnlyExtraImages { - processName = "Pull Extra Images" - } - - logger := dkplog.NewNop() - - if log.DebugLogLevel() >= 3 { - logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug)) - } - - // Create registry client for module operations - clientOpts := ®client.Options{ - Insecure: p.params.Insecure, - TLSSkipVerify: p.params.SkipTLSVerification, - Logger: logger, - } - - if p.params.RegistryAuth != nil { - clientOpts.Auth = p.params.RegistryAuth - } - - var c registry.Client - c = regclient.NewClientWithOptions(p.params.DeckhouseRegistryRepo, clientOpts) - - if os.Getenv("STUB_REGISTRY_CLIENT") == "true" { - c = stub.NewRegistryClientStub() - } - - // Scope to the registry path and modules suffix - if p.params.RegistryPath != "" { - c = c.WithSegment(p.params.RegistryPath) - } - - if p.params.ModulesPathSuffix != "" { - c = c.WithSegment(p.params.ModulesPathSuffix) - } - - return p.logger.Process(processName, func() error { - if err := p.validateModulesAccess(); err != nil { - return err - } - - filter, err := p.createModuleFilter() - if err != nil { - return err - } - - return operations.PullModules(p.params, filter, c) - }) -} - // validateModulesAccess validates access to the modules registry func (p *Puller) validateModulesAccess() error { modulesRepo := path.Join(p.params.DeckhouseRegistryRepo, p.params.ModulesPathSuffix) diff --git a/internal/mirror/cmd/pull/pull_test.go b/internal/mirror/cmd/pull/pull_test.go index 3bdadae9..0c20a94f 100644 --- a/internal/mirror/cmd/pull/pull_test.go +++ b/internal/mirror/cmd/pull/pull_test.go @@ -38,8 +38,6 @@ import ( "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/validation" - mock "github.com/deckhouse/deckhouse-cli/pkg/mock" - "github.com/deckhouse/deckhouse/pkg/registry" ) func TestNewCommand(t *testing.T) { @@ -104,75 +102,6 @@ func TestSetupLogger(t *testing.T) { } } -func TestFindTagsToMirror(t *testing.T) { - logger := log.NewSLogger(slog.LevelInfo) - - tests := []struct { - name string - deckhouseTag string - sinceVersion *semver.Version - expectError bool - expectedTags []string - }{ - { - name: "specific tag provided", - deckhouseTag: "v1.57.3", - expectedTags: []string{"v1.57.3"}, - }, - { - name: "no tag, should call releases.VersionsToMirror", - deckhouseTag: "", - expectError: true, // Will fail because releases.VersionsToMirror needs real params - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var originalVersionsToMirrorFunc func(*params.PullParams, registry.Client, []string) ([]semver.Version, []string, error) - if tt.deckhouseTag == "" { - // Mock for the no tag case to avoid panic - originalVersionsToMirrorFunc = versionsToMirrorFunc - versionsToMirrorFunc = func(pullParams *params.PullParams, client registry.Client, tagsToMirror []string) ([]semver.Version, []string, error) { - return nil, nil, fmt.Errorf("mock error for no tag case") - } - defer func() { versionsToMirrorFunc = originalVersionsToMirrorFunc }() - } else { - // Mock for specific tag case - originalVersionsToMirrorFunc = versionsToMirrorFunc - versionsToMirrorFunc = func(pullParams *params.PullParams, client registry.Client, tagsToMirror []string) ([]semver.Version, []string, error) { - if len(tagsToMirror) > 0 { - v, err := semver.NewVersion(strings.TrimPrefix(tagsToMirror[0], "v")) - if err != nil { - return nil, nil, err - } - return []semver.Version{*v}, []string{"alpha"}, nil - } - return nil, nil, fmt.Errorf("no tags") - } - defer func() { versionsToMirrorFunc = originalVersionsToMirrorFunc }() - } - - pullParams := ¶ms.PullParams{ - BaseParams: params.BaseParams{ - Logger: logger, - }, - DeckhouseTag: tt.deckhouseTag, - SinceVersion: tt.sinceVersion, - } - - client := mock.NewRegistryClientMock(t) - tags, _, err := findTagsToMirror(pullParams, logger, client) - - if tt.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tt.expectedTags, tags) - } - }) - } -} - func TestBuildPullParams(t *testing.T) { // Setup test environment variables originalTempDir := pullflags.TempDir @@ -981,39 +910,6 @@ func TestErrorMessages(t *testing.T) { assert.Equal(t, "pull failed, see the log for details", err.Error()) } -func TestFindTagsToMirrorWithVersionsSuccess(t *testing.T) { - // Save original function - originalVersionsToMirrorFunc := versionsToMirrorFunc - defer func() { versionsToMirrorFunc = originalVersionsToMirrorFunc }() - - // Mock the function to return successful versions - versionsToMirrorFunc = func(pullParams *params.PullParams, client registry.Client, tagsToMirror []string) ([]semver.Version, []string, error) { - return []semver.Version{ - *semver.MustParse("1.50.0"), - *semver.MustParse("1.51.0"), - *semver.MustParse("1.52.0"), - }, nil, nil - } - - logger := log.NewSLogger(slog.LevelInfo) - - // Test the case where we need to call versions lookup - originalDeckhouseTag := pullflags.DeckhouseTag - defer func() { pullflags.DeckhouseTag = originalDeckhouseTag }() - - pullflags.DeckhouseTag = "" // Force versions lookup - - pullParams := ¶ms.PullParams{ - DeckhouseTag: "", - SinceVersion: nil, - } - - client := mock.NewRegistryClientMock(t) - tags, _, err := findTagsToMirror(pullParams, logger, client) - assert.NoError(t, err) - assert.Equal(t, []string{"v1.50.0", "v1.51.0", "v1.52.0"}, tags) -} - func TestNewPuller(t *testing.T) { // Save original global variables originalTempDir := pullflags.TempDir @@ -1437,19 +1333,6 @@ func BenchmarkBuildPullParams(b *testing.B) { } } -func BenchmarkFindTagsToMirror(b *testing.B) { - logger := log.NewSLogger(slog.LevelInfo) - pullParams := ¶ms.PullParams{ - DeckhouseTag: "v1.57.3", - } - - client := mock.NewRegistryClientMock(b) - - for i := 0; i < b.N; i++ { - _, _, _ = findTagsToMirror(pullParams, logger, client) - } -} - func BenchmarkGetSourceRegistryAuthProvider(b *testing.B) { for i := 0; i < b.N; i++ { _ = getSourceRegistryAuthProvider() diff --git a/internal/mirror/cmd/push/push.go b/internal/mirror/cmd/push/push.go index 314bb9d0..a1223b71 100644 --- a/internal/mirror/cmd/push/push.go +++ b/internal/mirror/cmd/push/push.go @@ -20,18 +20,15 @@ import ( "context" "errors" "fmt" - "io" "log/slog" "os" "os/signal" "path" "path/filepath" - "strings" "syscall" "time" "github.com/google/go-containerregistry/pkg/authn" - "github.com/samber/lo" "github.com/spf13/cobra" dkplog "github.com/deckhouse/deckhouse/pkg/log" @@ -39,8 +36,6 @@ import ( regclient "github.com/deckhouse/deckhouse/pkg/registry/client" "github.com/deckhouse/deckhouse-cli/internal/mirror" - "github.com/deckhouse/deckhouse-cli/internal/mirror/chunked" - "github.com/deckhouse/deckhouse-cli/internal/mirror/operations" "github.com/deckhouse/deckhouse-cli/internal/version" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" @@ -110,93 +105,6 @@ func NewCommand() *cobra.Command { return pushCmd } -func pushModules(pushParams *params.PushParams, logger params.Logger, client registry.Client) error { - bundleContents, err := os.ReadDir(pushParams.BundleDir) - if err != nil { - return fmt.Errorf("List bundle directory: %w", err) - } - - modulePackages := lo.Compact(lo.Map(bundleContents, func(item os.DirEntry, _ int) string { - fileExt := filepath.Ext(item.Name()) - pkgName, _, ok := strings.Cut(strings.TrimPrefix(item.Name(), "module-"), ".") - switch { - case !ok: - fallthrough - case fileExt != ".tar" && fileExt != ".chunk": - fallthrough - case !strings.HasPrefix(item.Name(), "module-"): - return "" - } - return pkgName - })) - - successfullyPushedModules := make([]string, 0) - for _, modulePackageName := range modulePackages { - if lo.Contains(successfullyPushedModules, modulePackageName) { - continue - } - - if err = logger.Process("Push module: "+modulePackageName, func() error { - pkg, err := openPackage(pushParams, "module-"+modulePackageName) - if err != nil { - return fmt.Errorf("Open package %q: %w", modulePackageName, err) - } - - if err = operations.PushModule(pushParams, modulePackageName, pkg, client); err != nil { - return fmt.Errorf("Failed to push module %q: %w", modulePackageName, err) - } - - successfullyPushedModules = append(successfullyPushedModules, modulePackageName) - - return nil - }); err != nil { - logger.WarnLn(err) - } - } - - if len(successfullyPushedModules) > 0 { - logger.Infof("Modules pushed: %v", strings.Join(successfullyPushedModules, ", ")) - } - - return nil -} - -func pushStaticPackages(pushParams *params.PushParams, logger params.Logger, client registry.Client) error { - packages := []string{"platform", "security"} - for _, pkgName := range packages { - pkg, err := openPackage(pushParams, pkgName) - switch { - case errors.Is(err, os.ErrNotExist): - logger.InfoLn(pkgName, "package is not present, skipping") - continue - case err != nil: - return fmt.Errorf("open package %q: %w", pkgName, err) - } - - switch pkgName { - case "platform": - if err = logger.Process("Push Deckhouse platform", func() error { - return operations.PushDeckhousePlatform(pushParams, pkg, client) - }); err != nil { - return fmt.Errorf("Push Deckhouse Platform: %w", err) - } - case "security": - if err = logger.Process("Push security databases", func() error { - return operations.PushSecurityDatabases(pushParams, pkg, client) - }); err != nil { - return fmt.Errorf("Push Security Databases: %w", err) - } - default: - return errors.New("Unknown package " + pkgName) - } - - if err = pkg.Close(); err != nil { - logger.Warnf("Could not close bundle package %s: %v", pkgName, err) - } - } - return nil -} - func setupLogger() *log.SLogger { logLevel := slog.LevelInfo if log.DebugLogLevel() >= 3 { @@ -242,28 +150,6 @@ func validateRegistryAccess(ctx context.Context, pushParams *params.PushParams) return nil } -func openPackage(pushParams *params.PushParams, pkgName string) (io.ReadCloser, error) { - p := filepath.Join(pushParams.BundleDir, pkgName+".tar") - pkg, err := os.Open(p) - switch { - case errors.Is(err, os.ErrNotExist): - return openChunkedPackage(pushParams, pkgName) - case err != nil: - return nil, fmt.Errorf("Read bundle package %s: %w", pkgName, err) - } - - return pkg, nil -} - -func openChunkedPackage(pushParams *params.PushParams, pkgName string) (io.ReadCloser, error) { - pkg, err := chunked.Open(pushParams.BundleDir, pkgName+".tar") - if err != nil { - return nil, fmt.Errorf("Open bundle package %q: %w", pkgName, err) - } - - return pkg, nil -} - // Pusher handles the push operation for Deckhouse distribution type Pusher struct { logger params.Logger @@ -363,67 +249,3 @@ func (p *Pusher) validateRegistryAccess() error { } return nil } - -// pushStaticPackages pushes platform and security packages -func (p *Pusher) pushStaticPackages() error { - logger := dkplog.NewNop() - - if log.DebugLogLevel() >= 3 { - logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug)) - } - - // Create registry client for module operations - clientOpts := ®client.Options{ - Insecure: p.pushParams.Insecure, - TLSSkipVerify: p.pushParams.SkipTLSVerification, - Logger: logger, - } - - if p.pushParams.RegistryAuth != nil { - clientOpts.Auth = p.pushParams.RegistryAuth - } - - var client registry.Client - client = regclient.NewClientWithOptions(p.pushParams.RegistryHost, clientOpts) - - // Scope to the registry path and modules suffix - if p.pushParams.RegistryPath != "" { - client = client.WithSegment(p.pushParams.RegistryPath) - } - - return pushStaticPackages(p.pushParams, p.logger, client) -} - -// pushModules pushes module packages -func (p *Pusher) pushModules() error { - logger := dkplog.NewNop() - - if log.DebugLogLevel() >= 3 { - logger = dkplog.NewLogger(dkplog.WithLevel(slog.LevelDebug)) - } - - // Create registry client for module operations - clientOpts := ®client.Options{ - Insecure: p.pushParams.Insecure, - TLSSkipVerify: p.pushParams.SkipTLSVerification, - Logger: logger, // Will use default logger - } - - if p.pushParams.RegistryAuth != nil { - clientOpts.Auth = p.pushParams.RegistryAuth - } - - var client registry.Client - client = regclient.NewClientWithOptions(p.pushParams.RegistryHost, clientOpts) - - // Scope to the registry path and modules suffix - if p.pushParams.RegistryPath != "" { - client = client.WithSegment(p.pushParams.RegistryPath) - } - - if p.pushParams.ModulesPathSuffix != "" { - client = client.WithSegment(p.pushParams.ModulesPathSuffix) - } - - return pushModules(p.pushParams, p.logger, client) -} From 3fcb74aeef372e1076f36661999395f5d8597b4b Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 19:09:59 +0300 Subject: [PATCH 03/14] remove lo Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/platform.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index d4c68b83..f6428237 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -31,7 +31,6 @@ import ( "time" "github.com/Masterminds/semver/v3" - "github.com/samber/lo" dkplog "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/deckhouse/pkg/registry/client" @@ -51,6 +50,10 @@ type Options struct { // SinceVersion specifies the minimum version to start mirroring from (optional) SinceVersion *semver.Version // TargetTag specifies a specific tag to mirror instead of determining versions automatically + // it can be: + // semver f.e. vX.Y.Z + // channel f.e. alpha/beta/stable + // any other tag TargetTag string // BundleDir is the directory to store the bundle BundleDir string @@ -193,13 +196,12 @@ func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, error) { svc.userLogger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) - // Convert versions to tag format (add "v" prefix) - return lo.Map( - versionsToMirror, - func(v semver.Version, _ int) string { - return "v" + v.String() - }, - ), nil + vers := make([]string, len(versionsToMirror)) + for _, v := range versionsToMirror { + vers = append(vers, "v"+v.String()) + } + + return vers, nil } // versionsToMirrorFunc determines which Deckhouse release versions should be mirrored From 2eefb4fe469a483dd62488a8eb72fcbf201510e6 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 19:23:35 +0300 Subject: [PATCH 04/14] fix versions to mirror Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/layout.go | 9 +- internal/mirror/platform/platform.go | 122 ++++++++++++++++++--------- 2 files changed, 85 insertions(+), 46 deletions(-) diff --git a/internal/mirror/platform/layout.go b/internal/mirror/platform/layout.go index 36f7618c..9b2685e2 100644 --- a/internal/mirror/platform/layout.go +++ b/internal/mirror/platform/layout.go @@ -61,13 +61,8 @@ func (l *ImageDownloadList) FillDeckhouseImages(deckhouseVersions []string) { } } -func (l *ImageDownloadList) FillForTag(tag string) { - // If we are to pull only the specific requested version, we should not pull any release channels at all. - if tag != "" { - return - } - - for _, channel := range internal.GetAllDefaultReleaseChannels() { +func (l *ImageDownloadList) FillForChannels(channels []string) { + for _, channel := range channels { l.Deckhouse[l.rootURL+":"+channel] = nil l.DeckhouseInstall[path.Join(l.rootURL, internal.InstallSegment)+":"+channel] = nil l.DeckhouseInstallStandalone[path.Join(l.rootURL, internal.InstallStandaloneSegment)+":"+channel] = nil diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index f6428237..b55d0243 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -125,13 +125,13 @@ func (svc *Service) PullPlatform(ctx context.Context) error { return fmt.Errorf("validate platform access: %w", err) } - tagsToMirror, err := svc.findTagsToMirror(ctx) + tagsToMirror, channelsToMirror, err := svc.findTagsToMirror(ctx) if err != nil { return fmt.Errorf("find tags to mirror: %w", err) } svc.downloadList.FillDeckhouseImages(tagsToMirror) - svc.downloadList.FillForTag(svc.options.TargetTag) + svc.downloadList.FillForChannels(channelsToMirror) err = svc.pullDeckhousePlatform(ctx, tagsToMirror) if err != nil { @@ -180,71 +180,117 @@ func (svc *Service) validatePlatformAccess(ctx context.Context) error { // findTagsToMirror determines which Deckhouse release tags should be mirrored // If a specific target tag is set, it returns only that tag // Otherwise, it finds all relevant versions that should be mirrored based on channels and version ranges -func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, error) { - // If a specific tag is requested, skip the complex version determination logic +func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, []string, error) { + strickTags := []string{} if svc.options.TargetTag != "" { - svc.userLogger.Infof("Skipped releases lookup as tag %q is specifically requested with --deckhouse-tag", svc.options.TargetTag) - - return []string{svc.options.TargetTag}, nil + strickTags = append(strickTags, svc.options.TargetTag) } - // Determine which versions should be mirrored based on release channels and version constraints - versionsToMirror, err := svc.versionsToMirrorFunc(ctx) + versionsToMirror, channelsToMirror, err := svc.versionsToMirrorFunc(ctx, strickTags) if err != nil { - return nil, fmt.Errorf("find versions to mirror: %w", err) + return nil, nil, fmt.Errorf("Find versions to mirror: %w", err) } - svc.userLogger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) + svc.logger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) vers := make([]string, len(versionsToMirror)) for _, v := range versionsToMirror { vers = append(vers, "v"+v.String()) } - return vers, nil + return vers, channelsToMirror, nil +} + +type releaseChannelVersionResult struct { + ver *semver.Version + err error } // versionsToMirrorFunc determines which Deckhouse release versions should be mirrored // It collects current versions from all release channels and filters available releases // to include only versions that should be mirrored based on the mirroring strategy -func (svc *Service) versionsToMirrorFunc(ctx context.Context) ([]semver.Version, error) { +func (svc *Service) versionsToMirrorFunc(ctx context.Context, tagsToMirror []string) ([]semver.Version, []string, error) { logger := svc.userLogger + if len(tagsToMirror) > 0 { + logger.Infof("Skipped releases lookup as tag %q is specifically requested with --deckhouse-tag", svc.options.TargetTag) + } + + vers := make([]*semver.Version, 0, 1) + + for _, tag := range tagsToMirror { + v, err := semver.NewVersion(tag) + if err != nil { + continue + } + + vers = append(vers, v) + } + releaseChannelsToCopy := internal.GetAllDefaultReleaseChannels() releaseChannelsToCopy = append(releaseChannelsToCopy, internal.LTSChannel) - releaseChannelsVersions := make(map[string]*semver.Version, len(releaseChannelsToCopy)) + releaseChannelsVersionsResult := make(map[string]releaseChannelVersionResult, len(releaseChannelsToCopy)) for _, channel := range releaseChannelsToCopy { - version, err := svc.getReleaseChannelVersionFromRegistry(ctx, channel) + v, err := svc.getReleaseChannelVersionFromRegistry(ctx, channel) if err != nil { if channel == internal.LTSChannel { - if !errors.Is(err, client.ErrImageNotFound) { - svc.userLogger.Warnf("Skipping LTS channel: %v", err) - } else { - svc.userLogger.Warnf("Skipping LTS channel, because it's not required") + if err != nil { + logger.Warnf("Skipping LTS channel: %v", err) + continue } - - continue } - return nil, fmt.Errorf("get %s release version from registry: %w", channel, err) + releaseChannelsVersionsResult[channel] = releaseChannelVersionResult{ver: v, err: err} } + } + + releaseChannelsVersions := make(map[string]*semver.Version, len(releaseChannelsToCopy)) + + _, ltsChannelFound := releaseChannelsVersionsResult[internal.LTSChannel] + for channel, res := range releaseChannelsVersionsResult { + if !ltsChannelFound && res.err != nil { + return nil, nil, fmt.Errorf("get %s release version from registry: %w", channel, res.err) + } + + if res.err == nil { + releaseChannelsVersions[channel] = res.ver + } + } - if version == nil { - // Channel was skipped (e.g., suspended and ignoreSuspendedChannels is true) + mappedChannels := make(map[string]struct{}, len(releaseChannelsVersions)) + for channel, v := range releaseChannelsVersions { + if len(tagsToMirror) == 0 { + vers = append(vers, v) + mappedChannels[channel] = struct{}{} continue } - releaseChannelsVersions[channel] = version + for _, tag := range tagsToMirror { + if tag == "v"+v.String() || tag == channel { + vers = append(vers, v) + mappedChannels[channel] = struct{}{} + } + } } - rockSolidVersion := releaseChannelsVersions[internal.RockSolidChannel] + channels := make([]string, 0, len(mappedChannels)) + for channel := range mappedChannels { + channels = append(channels, channel) + } - mirrorFromVersion := *rockSolidVersion + if len(tagsToMirror) > 0 { + return deduplicateVersions(vers), channels, nil + } - if svc.options.SinceVersion != nil { - if svc.options.SinceVersion.LessThan(rockSolidVersion) { - mirrorFromVersion = *svc.options.SinceVersion + var mirrorFromVersion *semver.Version + rockSolidVersion, found := releaseChannelsVersions[internal.RockSolidChannel] + if found { + mirrorFromVersion = rockSolidVersion + if svc.options.SinceVersion != nil { + if svc.options.SinceVersion.LessThan(rockSolidVersion) { + mirrorFromVersion = svc.options.SinceVersion + } } } @@ -252,20 +298,18 @@ func (svc *Service) versionsToMirrorFunc(ctx context.Context) ([]semver.Version, tags, err := svc.deckhouseService.ReleaseChannels().ListTags(ctx) if err != nil { - return nil, fmt.Errorf("get tags from Deckhouse registry: %w", err) + return nil, nil, fmt.Errorf("get tags from Deckhouse registry: %w", err) } - alphaChannelVersion := releaseChannelsVersions[internal.AlphaChannel] - - versionsAboveMinimal := filterVersionsBetween(&mirrorFromVersion, alphaChannelVersion, tags) - versionsAboveMinimal = filterOnlyLatestPatches(versionsAboveMinimal) + alphaChannelVersion, found := releaseChannelsVersions[internal.AlphaChannel] + if found { + versionsAboveMinimal := filterVersionsBetween(mirrorFromVersion, alphaChannelVersion, tags) + versionsAboveMinimal = filterOnlyLatestPatches(versionsAboveMinimal) - vers := make([]*semver.Version, 0, len(releaseChannelsVersions)) - for _, v := range releaseChannelsVersions { - vers = append(vers, v) + return deduplicateVersions(append(vers, versionsAboveMinimal...)), channels, nil } - return deduplicateVersions(append(vers, versionsAboveMinimal...)), nil + return deduplicateVersions(vers), channels, nil } // getReleaseChannelVersionFromRegistry retrieves the current version for a specific release channel From 3a08854ab7163a843c1d5e2fdbb6448b5c17adad Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 19:29:04 +0300 Subject: [PATCH 05/14] bump logger Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/platform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index b55d0243..5f4a14bf 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -191,7 +191,7 @@ func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, []string, e return nil, nil, fmt.Errorf("Find versions to mirror: %w", err) } - svc.logger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) + svc.userLogger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) vers := make([]string, len(versionsToMirror)) for _, v := range versionsToMirror { From 168ce71ccf5c256c3e52f915f199dbf950b3ffcf Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 20:44:53 +0300 Subject: [PATCH 06/14] bump Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/platform.go | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index 5f4a14bf..8c587af1 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -186,14 +186,14 @@ func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, []string, e strickTags = append(strickTags, svc.options.TargetTag) } - versionsToMirror, channelsToMirror, err := svc.versionsToMirrorFunc(ctx, strickTags) + versionsToMirror, channelsToMirror, err := svc.versionsToMirror(ctx, strickTags) if err != nil { return nil, nil, fmt.Errorf("Find versions to mirror: %w", err) } svc.userLogger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) - vers := make([]string, len(versionsToMirror)) + vers := make([]string, 0, len(versionsToMirror)) for _, v := range versionsToMirror { vers = append(vers, "v"+v.String()) } @@ -206,10 +206,10 @@ type releaseChannelVersionResult struct { err error } -// versionsToMirrorFunc determines which Deckhouse release versions should be mirrored +// versionsToMirror determines which Deckhouse release versions should be mirrored // It collects current versions from all release channels and filters available releases // to include only versions that should be mirrored based on the mirroring strategy -func (svc *Service) versionsToMirrorFunc(ctx context.Context, tagsToMirror []string) ([]semver.Version, []string, error) { +func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) ([]semver.Version, []string, error) { logger := svc.userLogger if len(tagsToMirror) > 0 { @@ -233,16 +233,16 @@ func (svc *Service) versionsToMirrorFunc(ctx context.Context, tagsToMirror []str releaseChannelsVersionsResult := make(map[string]releaseChannelVersionResult, len(releaseChannelsToCopy)) for _, channel := range releaseChannelsToCopy { v, err := svc.getReleaseChannelVersionFromRegistry(ctx, channel) - if err != nil { - if channel == internal.LTSChannel { - if err != nil { - logger.Warnf("Skipping LTS channel: %v", err) - continue - } - } - releaseChannelsVersionsResult[channel] = releaseChannelVersionResult{ver: v, err: err} + // If LTS channel is missing or errored, just warn and continue (it is optional) + if err != nil && channel == internal.LTSChannel { + logger.Warnf("Skipping LTS channel: %v", err) + continue + } + + // Always record the result except for LTS channel, we will decide later if we can continue without it + releaseChannelsVersionsResult[channel] = releaseChannelVersionResult{ver: v, err: err} } releaseChannelsVersions := make(map[string]*semver.Version, len(releaseChannelsToCopy)) @@ -755,15 +755,19 @@ func filterOnlyLatestPatches(versions []*semver.Version) []*semver.Version { // deduplicateVersions removes duplicate versions from the list. // This is necessary because channel versions and filtered versions might overlap. func deduplicateVersions(versions []*semver.Version) []semver.Version { - m := map[semver.Version]struct{}{} + m := map[string]struct{}{} for _, v := range versions { - m[*v] = struct{}{} + if v == nil { + continue + } + m[v.String()] = struct{}{} } vers := make([]semver.Version, 0, len(m)) - for k := range maps.Keys(m) { - vers = append(vers, k) + for s := range m { + // semver.MustParse returns a canonical semver.Version value suitable for comparison + vers = append(vers, *semver.MustParse("v" + s)) } return vers From 56da8d19cc201d1a3ae74c2eb3cc33539bbaf5cd Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 21:30:26 +0300 Subject: [PATCH 07/14] add platform test and feature to fetch custom tags Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/platform.go | 51 +++- internal/mirror/platform/platform_test.go | 315 ++++++++++++++++++++++ pkg/stub/registry_client.go | 25 +- 3 files changed, 372 insertions(+), 19 deletions(-) create mode 100644 internal/mirror/platform/platform_test.go diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index 8c587af1..8934b617 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -186,19 +186,21 @@ func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, []string, e strickTags = append(strickTags, svc.options.TargetTag) } - versionsToMirror, channelsToMirror, err := svc.versionsToMirror(ctx, strickTags) + result, err := svc.versionsToMirror(ctx, strickTags) if err != nil { return nil, nil, fmt.Errorf("Find versions to mirror: %w", err) } - svc.userLogger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) + svc.userLogger.Infof("Deckhouse releases to pull: %+v", result.Versions) - vers := make([]string, 0, len(versionsToMirror)) - for _, v := range versionsToMirror { + vers := make([]string, 0, len(result.Versions)+len(result.CustomTags)) + for _, v := range result.Versions { vers = append(vers, "v"+v.String()) } + // Add custom tags as-is (without "v" prefix) + vers = append(vers, result.CustomTags...) - return vers, channelsToMirror, nil + return vers, result.Channels, nil } type releaseChannelVersionResult struct { @@ -206,10 +208,20 @@ type releaseChannelVersionResult struct { err error } +// VersionsToMirrorResult contains the result of versionsToMirror operation +type VersionsToMirrorResult struct { + // Versions contains semver versions to mirror + Versions []semver.Version + // Channels contains release channels to mirror + Channels []string + // CustomTags contains custom tags (non-semver, non-channel tags) to mirror + CustomTags []string +} + // versionsToMirror determines which Deckhouse release versions should be mirrored // It collects current versions from all release channels and filters available releases // to include only versions that should be mirrored based on the mirroring strategy -func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) ([]semver.Version, []string, error) { +func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) (*VersionsToMirrorResult, error) { logger := svc.userLogger if len(tagsToMirror) > 0 { @@ -217,10 +229,15 @@ func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) } vers := make([]*semver.Version, 0, 1) + customTags := make([]string, 0) for _, tag := range tagsToMirror { v, err := semver.NewVersion(tag) if err != nil { + // Not a valid semver and not a channel name - treat as custom tag + if !internal.ChannelIsValid(tag) { + customTags = append(customTags, tag) + } continue } @@ -250,7 +267,7 @@ func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) _, ltsChannelFound := releaseChannelsVersionsResult[internal.LTSChannel] for channel, res := range releaseChannelsVersionsResult { if !ltsChannelFound && res.err != nil { - return nil, nil, fmt.Errorf("get %s release version from registry: %w", channel, res.err) + return nil, fmt.Errorf("get %s release version from registry: %w", channel, res.err) } if res.err == nil { @@ -280,7 +297,11 @@ func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) } if len(tagsToMirror) > 0 { - return deduplicateVersions(vers), channels, nil + return &VersionsToMirrorResult{ + Versions: deduplicateVersions(vers), + Channels: channels, + CustomTags: customTags, + }, nil } var mirrorFromVersion *semver.Version @@ -298,7 +319,7 @@ func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) tags, err := svc.deckhouseService.ReleaseChannels().ListTags(ctx) if err != nil { - return nil, nil, fmt.Errorf("get tags from Deckhouse registry: %w", err) + return nil, fmt.Errorf("get tags from Deckhouse registry: %w", err) } alphaChannelVersion, found := releaseChannelsVersions[internal.AlphaChannel] @@ -306,10 +327,18 @@ func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) versionsAboveMinimal := filterVersionsBetween(mirrorFromVersion, alphaChannelVersion, tags) versionsAboveMinimal = filterOnlyLatestPatches(versionsAboveMinimal) - return deduplicateVersions(append(vers, versionsAboveMinimal...)), channels, nil + return &VersionsToMirrorResult{ + Versions: deduplicateVersions(append(vers, versionsAboveMinimal...)), + Channels: channels, + CustomTags: customTags, + }, nil } - return deduplicateVersions(vers), channels, nil + return &VersionsToMirrorResult{ + Versions: deduplicateVersions(vers), + Channels: channels, + CustomTags: customTags, + }, nil } // getReleaseChannelVersionFromRegistry retrieves the current version for a specific release channel diff --git a/internal/mirror/platform/platform_test.go b/internal/mirror/platform/platform_test.go new file mode 100644 index 00000000..fd68acc2 --- /dev/null +++ b/internal/mirror/platform/platform_test.go @@ -0,0 +1,315 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package platform + +import ( + "context" + "log/slog" + "testing" + + "github.com/Masterminds/semver/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + dkplog "github.com/deckhouse/deckhouse/pkg/log" + + "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" + registryservice "github.com/deckhouse/deckhouse-cli/pkg/registry/service" + "github.com/deckhouse/deckhouse-cli/pkg/stub" +) + +func TestService_versionsToMirror(t *testing.T) { + tests := []struct { + name string + strictTags []string + options *Options + wantVersions []string // expected versions in format "v1.72.10", etc. + wantChannels []string // expected channels + wantCustomTags []string // expected custom tags + wantErr bool + wantErrContains string + }{ + { + name: "with strict tags - single semver version", + strictTags: []string{"v1.72.10"}, + options: &Options{}, + wantVersions: []string{ + "v1.72.10", + }, + // v1.72.10 matches only alpha channel version + wantChannels: []string{"alpha"}, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "with strict tags - multiple semver versions", + strictTags: []string{"v1.72.10", "v1.71.0", "v1.70.0"}, + options: &Options{}, + wantVersions: []string{ + "v1.72.10", + "v1.71.0", + "v1.70.0", + }, + // v1.72.10 matches alpha, v1.71.0 matches beta, v1.70.0 matches early-access + wantChannels: []string{ + "alpha", "beta", "early-access", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "with strict tags - channel name", + strictTags: []string{"stable"}, + options: &Options{}, + wantVersions: []string{ + "v1.69.0", // stable channel version + }, + wantChannels: []string{ + "stable", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "with strict tags - custom tag", + strictTags: []string{"pr12345"}, + options: &Options{}, + wantVersions: []string{ + // No semver versions for custom tag + }, + wantChannels: []string{ + // No channels matched + }, + wantCustomTags: []string{ + "pr12345", // custom tag is returned as-is + }, + wantErr: false, + }, + { + name: "no strict tags - full discovery", + strictTags: []string{}, + options: &Options{}, + // When no strict tags, it returns all channel versions + wantVersions: []string{ + "v1.72.10", // alpha + "v1.71.0", // beta + "v1.70.0", // early-access + "v1.69.0", // stable + "v1.68.0", // rock-solid + }, + wantChannels: []string{ + "alpha", "beta", "early-access", "stable", "rock-solid", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "no strict tags with SinceVersion", + strictTags: []string{}, + options: &Options{ + SinceVersion: semver.MustParse("v1.69.0"), + }, + // When no strict tags, it returns all channel versions + wantVersions: []string{ + "v1.72.10", // alpha + "v1.71.0", // beta + "v1.70.0", // early-access + "v1.69.0", // stable + "v1.68.0", // rock-solid + }, + wantChannels: []string{ + "alpha", "beta", "early-access", "stable", "rock-solid", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: tt.options, + logger: logger, + userLogger: userLogger, + } + + // Call the function under test + result, err := svc.versionsToMirror(context.Background(), tt.strictTags) + + // Check error + if tt.wantErr { + require.Error(t, err) + if tt.wantErrContains != "" { + assert.Contains(t, err.Error(), tt.wantErrContains) + } + return + } + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Check versions (order doesn't matter for this test) + assert.ElementsMatch(t, tt.wantVersions, gotVersions, "versions mismatch") + + // Check channels (order doesn't matter) + assert.ElementsMatch(t, tt.wantChannels, result.Channels, "channels mismatch") + + // Check custom tags (order doesn't matter) + assert.ElementsMatch(t, tt.wantCustomTags, result.CustomTags, "custom tags mismatch") + }) + } +} + +func TestService_versionsToMirror_WithTargetTag(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance with TargetTag + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: &Options{ + TargetTag: "v1.72.10", + }, + logger: logger, + userLogger: userLogger, + } + + // Call the function under test + result, err := svc.versionsToMirror(context.Background(), []string{"v1.72.10"}) + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Should only return the specific tag + assert.ElementsMatch(t, []string{"v1.72.10"}, gotVersions) + // Channels should match the version (alpha channel has v1.72.10) + assert.ElementsMatch(t, []string{"alpha"}, result.Channels) + // No custom tags + assert.Empty(t, result.CustomTags) +} + +func TestService_versionsToMirror_CustomTagWithSemver(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: &Options{}, + logger: logger, + userLogger: userLogger, + } + + // Call with mix of semver version and custom tag + strictTags := []string{"v1.72.10", "pr12345"} + result, err := svc.versionsToMirror(context.Background(), strictTags) + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Should have both semver version and custom tag + assert.Contains(t, gotVersions, "v1.72.10", "should include semver version") + assert.Contains(t, result.CustomTags, "pr12345", "should include custom tag") + assert.Contains(t, result.Channels, "alpha", "should include alpha channel") +} + +func TestService_versionsToMirror_Deduplication(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: &Options{}, + logger: logger, + userLogger: userLogger, + } + + // Call with duplicate versions in strictTags + strictTags := []string{"v1.72.10", "v1.72.10", "alpha"} // alpha also points to v1.72.10 + result, err := svc.versionsToMirror(context.Background(), strictTags) + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Should deduplicate + assert.Equal(t, 1, len(gotVersions), "expected deduplicated versions") + assert.Contains(t, gotVersions, "v1.72.10") + + // Channels should include alpha + assert.Contains(t, result.Channels, "alpha") + + // No custom tags + assert.Empty(t, result.CustomTags) +} diff --git a/pkg/stub/registry_client.go b/pkg/stub/registry_client.go index e0449a3c..92493f7b 100644 --- a/pkg/stub/registry_client.go +++ b/pkg/stub/registry_client.go @@ -476,20 +476,20 @@ func (s *RegistryClientStub) initializeRegistries() { // Registry 1: dynamic source s.addRegistry(source, map[string][]string{ - "": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, - "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions - "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, - "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, + "": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, // including custom tag + "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions + "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, + "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, }) // Registry 2: gcr.io s.addRegistry("gcr.io/google-containers", map[string][]string{ - "": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, + "": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, "pause": {"3.9", "latest"}, "kube-apiserver": {"v1.28.0", "v1.29.0", "latest"}, "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, - "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, - "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, + "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, + "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, }) // Registry 3: quay.io @@ -843,7 +843,16 @@ func (s *RegistryClientStub) GetImage(_ context.Context, tag string, _ ...regist } } - // Fall back to all registries + // Fall back to searching in the specific registry first, then all registries + if regData, exists := s.registries[registry]; exists { + for _, repoData := range regData.repositories { + if imageData, exists := repoData.images[tag]; exists { + return imageData.image.(pkg.ClientImage), nil + } + } + } + + // Last resort: search all registries for _, regData := range s.registries { for _, repoData := range regData.repositories { if imageData, exists := repoData.images[tag]; exists { From 572a14c5ee97fd4c70729aaa8deb6d79eaae9645 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 21:39:11 +0300 Subject: [PATCH 08/14] bump Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/platform.go | 226 ++++++++++++++++++--------- 1 file changed, 152 insertions(+), 74 deletions(-) diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index 8934b617..c8b0418f 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -218,127 +218,205 @@ type VersionsToMirrorResult struct { CustomTags []string } +// parsedTags represents the parsed input tags categorized by type +type parsedTags struct { + semverVersions []*semver.Version + customTags []string +} + +// channelVersions represents the fetched versions from release channels +type channelVersions map[string]*semver.Version + // versionsToMirror determines which Deckhouse release versions should be mirrored // It collects current versions from all release channels and filters available releases // to include only versions that should be mirrored based on the mirroring strategy func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) (*VersionsToMirrorResult, error) { - logger := svc.userLogger + if len(tagsToMirror) > 0 { + svc.userLogger.Infof("Skipped releases lookup as tag %q is specifically requested with --deckhouse-tag", svc.options.TargetTag) + } + // Parse input tags into categories + parsed := svc.parseInputTags(tagsToMirror) + + // Fetch current versions from all release channels + channelVersions, err := svc.fetchReleaseChannelVersions(ctx) + if err != nil { + return nil, err + } + + // Match channels and versions based on requested tags + versions, matchedChannels := svc.matchChannelsToTags(tagsToMirror, channelVersions, parsed.semverVersions) + + // If specific tags requested, return immediately if len(tagsToMirror) > 0 { - logger.Infof("Skipped releases lookup as tag %q is specifically requested with --deckhouse-tag", svc.options.TargetTag) + return &VersionsToMirrorResult{ + Versions: deduplicateVersions(versions), + Channels: matchedChannels, + CustomTags: parsed.customTags, + }, nil } - vers := make([]*semver.Version, 0, 1) - customTags := make([]string, 0) + // For full discovery mode, expand version range + expandedVersions, err := svc.expandVersionRange(ctx, channelVersions, versions) + if err != nil { + return nil, err + } + + return &VersionsToMirrorResult{ + Versions: deduplicateVersions(expandedVersions), + Channels: matchedChannels, + CustomTags: parsed.customTags, + }, nil +} + +// parseInputTags categorizes input tags into semver versions and custom tags +func (svc *Service) parseInputTags(tags []string) parsedTags { + result := parsedTags{ + semverVersions: make([]*semver.Version, 0, len(tags)), + customTags: make([]string, 0), + } - for _, tag := range tagsToMirror { - v, err := semver.NewVersion(tag) + for _, tag := range tags { + version, err := semver.NewVersion(tag) if err != nil { - // Not a valid semver and not a channel name - treat as custom tag + // Not a valid semver - check if it's a custom tag (not a channel name) if !internal.ChannelIsValid(tag) { - customTags = append(customTags, tag) + result.customTags = append(result.customTags, tag) } continue } - - vers = append(vers, v) + result.semverVersions = append(result.semverVersions, version) } - releaseChannelsToCopy := internal.GetAllDefaultReleaseChannels() - releaseChannelsToCopy = append(releaseChannelsToCopy, internal.LTSChannel) + return result +} - releaseChannelsVersionsResult := make(map[string]releaseChannelVersionResult, len(releaseChannelsToCopy)) - for _, channel := range releaseChannelsToCopy { - v, err := svc.getReleaseChannelVersionFromRegistry(ctx, channel) +// fetchReleaseChannelVersions retrieves current versions from all release channels +func (svc *Service) fetchReleaseChannelVersions(ctx context.Context) (channelVersions, error) { + channelsToFetch := append(internal.GetAllDefaultReleaseChannels(), internal.LTSChannel) + + // Fetch versions from all channels + channelResults := make(map[string]releaseChannelVersionResult, len(channelsToFetch)) + for _, channel := range channelsToFetch { + version, err := svc.getReleaseChannelVersionFromRegistry(ctx, channel) - // If LTS channel is missing or errored, just warn and continue (it is optional) + // LTS channel is optional - warn and continue if missing if err != nil && channel == internal.LTSChannel { - logger.Warnf("Skipping LTS channel: %v", err) + svc.userLogger.Warnf("Skipping LTS channel: %v", err) continue - } - // Always record the result except for LTS channel, we will decide later if we can continue without it - releaseChannelsVersionsResult[channel] = releaseChannelVersionResult{ver: v, err: err} + channelResults[channel] = releaseChannelVersionResult{ver: version, err: err} } - releaseChannelsVersions := make(map[string]*semver.Version, len(releaseChannelsToCopy)) + // Validate and extract successful channel versions + return svc.validateChannelResults(channelResults) +} + +// validateChannelResults validates channel fetch results and extracts successful versions +func (svc *Service) validateChannelResults(results map[string]releaseChannelVersionResult) (channelVersions, error) { + versions := make(channelVersions, len(results)) + _, ltsExists := results[internal.LTSChannel] - _, ltsChannelFound := releaseChannelsVersionsResult[internal.LTSChannel] - for channel, res := range releaseChannelsVersionsResult { - if !ltsChannelFound && res.err != nil { - return nil, fmt.Errorf("get %s release version from registry: %w", channel, res.err) + for channel, result := range results { + // If LTS doesn't exist, all other channels must succeed + if !ltsExists && result.err != nil { + return nil, fmt.Errorf("get %s release version from registry: %w", channel, result.err) } - if res.err == nil { - releaseChannelsVersions[channel] = res.ver + if result.err == nil { + versions[channel] = result.ver } } - mappedChannels := make(map[string]struct{}, len(releaseChannelsVersions)) - for channel, v := range releaseChannelsVersions { - if len(tagsToMirror) == 0 { - vers = append(vers, v) - mappedChannels[channel] = struct{}{} - continue - } + return versions, nil +} - for _, tag := range tagsToMirror { - if tag == "v"+v.String() || tag == channel { - vers = append(vers, v) - mappedChannels[channel] = struct{}{} +// matchChannelsToTags matches requested tags to channel versions and returns matching versions and channels +func (svc *Service) matchChannelsToTags(requestedTags []string, channelVersions channelVersions, semverVersions []*semver.Version) ([]*semver.Version, []string) { + versions := make([]*semver.Version, 0, len(semverVersions)) + versions = append(versions, semverVersions...) + + matchedChannels := make(map[string]struct{}) + + // If no specific tags requested, mirror all channels + if len(requestedTags) == 0 { + for channel, version := range channelVersions { + versions = append(versions, version) + matchedChannels[channel] = struct{}{} + } + return versions, mapKeysToSlice(matchedChannels) + } + + // Match specific tags to channels + for channel, version := range channelVersions { + for _, tag := range requestedTags { + if svc.tagMatchesChannel(tag, channel, version) { + versions = append(versions, version) + matchedChannels[channel] = struct{}{} + break } } } - channels := make([]string, 0, len(mappedChannels)) - for channel := range mappedChannels { - channels = append(channels, channel) - } + return versions, mapKeysToSlice(matchedChannels) +} - if len(tagsToMirror) > 0 { - return &VersionsToMirrorResult{ - Versions: deduplicateVersions(vers), - Channels: channels, - CustomTags: customTags, - }, nil - } +// tagMatchesChannel checks if a tag matches a channel (by name or version) +func (svc *Service) tagMatchesChannel(tag, channelName string, channelVersion *semver.Version) bool { + return tag == channelName || tag == "v"+channelVersion.String() +} - var mirrorFromVersion *semver.Version - rockSolidVersion, found := releaseChannelsVersions[internal.RockSolidChannel] - if found { - mirrorFromVersion = rockSolidVersion - if svc.options.SinceVersion != nil { - if svc.options.SinceVersion.LessThan(rockSolidVersion) { - mirrorFromVersion = svc.options.SinceVersion - } - } +// expandVersionRange expands the version range for full discovery mode +func (svc *Service) expandVersionRange(ctx context.Context, channelVersions channelVersions, baseVersions []*semver.Version) ([]*semver.Version, error) { + minVersion := svc.determineMinimumVersion(channelVersions) + maxVersion := channelVersions[internal.AlphaChannel] + + if maxVersion == nil { + // No alpha channel - return base versions only + return baseVersions, nil } - logger.Debugf("listing deckhouse releases") + svc.userLogger.Debugf("listing deckhouse releases") - tags, err := svc.deckhouseService.ReleaseChannels().ListTags(ctx) + // Fetch all available tags + allTags, err := svc.deckhouseService.ReleaseChannels().ListTags(ctx) if err != nil { return nil, fmt.Errorf("get tags from Deckhouse registry: %w", err) } - alphaChannelVersion, found := releaseChannelsVersions[internal.AlphaChannel] - if found { - versionsAboveMinimal := filterVersionsBetween(mirrorFromVersion, alphaChannelVersion, tags) - versionsAboveMinimal = filterOnlyLatestPatches(versionsAboveMinimal) + // Filter and get latest patches + filteredVersions := filterVersionsBetween(minVersion, maxVersion, allTags) + latestPatches := filterOnlyLatestPatches(filteredVersions) - return &VersionsToMirrorResult{ - Versions: deduplicateVersions(append(vers, versionsAboveMinimal...)), - Channels: channels, - CustomTags: customTags, - }, nil + return append(baseVersions, latestPatches...), nil +} + +// determineMinimumVersion determines the minimum version for mirroring based on configuration +func (svc *Service) determineMinimumVersion(channelVersions channelVersions) *semver.Version { + rockSolidVersion := channelVersions[internal.RockSolidChannel] + if rockSolidVersion == nil { + return nil } - return &VersionsToMirrorResult{ - Versions: deduplicateVersions(vers), - Channels: channels, - CustomTags: customTags, - }, nil + // Use rock-solid as baseline + minVersion := rockSolidVersion + + // Override with SinceVersion if it's older + if svc.options.SinceVersion != nil && svc.options.SinceVersion.LessThan(rockSolidVersion) { + minVersion = svc.options.SinceVersion + } + + return minVersion +} + +// mapKeysToSlice converts a map's keys to a slice +func mapKeysToSlice(m map[string]struct{}) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + return keys } // getReleaseChannelVersionFromRegistry retrieves the current version for a specific release channel From f0f7b9da68fc7a9d11301f10ce979d616518f4ca Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 16 Feb 2026 21:54:29 +0300 Subject: [PATCH 09/14] lint Signed-off-by: Pavel Okhlopkov --- internal/mirror/modules/modules.go | 2 +- internal/mirror/platform/platform.go | 4 ++-- internal/mirror/security/layout.go | 2 +- internal/utilk8s/clientset.go | 5 ++--- pkg/libmirror/layouts/tag_resolver.go | 5 +++-- pkg/stub/registry_client.go | 16 +++++++++++++--- 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/internal/mirror/modules/modules.go b/internal/mirror/modules/modules.go index c3580a66..e64b66fc 100644 --- a/internal/mirror/modules/modules.go +++ b/internal/mirror/modules/modules.go @@ -653,7 +653,7 @@ func (svc *Service) extractInternalDigestImages(ctx context.Context, moduleName // pullVexImages finds and pulls VEX attestation images for module images func (svc *Service) pullVexImages(ctx context.Context, moduleName string, downloadList *ImageDownloadList) { - allImages := make([]string, 0) + allImages := make([]string, 0, len(downloadList.Module)+len(downloadList.ModuleExtra)) for img := range downloadList.Module { allImages = append(allImages, img) diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index c8b0418f..f4bec4a9 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -294,7 +294,7 @@ func (svc *Service) parseInputTags(tags []string) parsedTags { // fetchReleaseChannelVersions retrieves current versions from all release channels func (svc *Service) fetchReleaseChannelVersions(ctx context.Context) (channelVersions, error) { channelsToFetch := append(internal.GetAllDefaultReleaseChannels(), internal.LTSChannel) - + // Fetch versions from all channels channelResults := make(map[string]releaseChannelVersionResult, len(channelsToFetch)) for _, channel := range channelsToFetch { @@ -336,7 +336,7 @@ func (svc *Service) validateChannelResults(results map[string]releaseChannelVers func (svc *Service) matchChannelsToTags(requestedTags []string, channelVersions channelVersions, semverVersions []*semver.Version) ([]*semver.Version, []string) { versions := make([]*semver.Version, 0, len(semverVersions)) versions = append(versions, semverVersions...) - + matchedChannels := make(map[string]struct{}) // If no specific tags requested, mirror all channels diff --git a/internal/mirror/security/layout.go b/internal/mirror/security/layout.go index 011302a4..73e064d8 100644 --- a/internal/mirror/security/layout.go +++ b/internal/mirror/security/layout.go @@ -102,7 +102,7 @@ func (l *ImageLayouts) setLayoutByMirrorType(rootFolder string, mirrorType inter // AsList returns a list of layout.Path's in it. Undefined path's are not included in the list. func (l *ImageLayouts) AsList() []layout.Path { - paths := make([]layout.Path, 0) + paths := make([]layout.Path, 0, len(l.Security)) for _, layout := range l.Security { paths = append(paths, layout.Path()) } diff --git a/internal/utilk8s/clientset.go b/internal/utilk8s/clientset.go index c91f1f6e..c8068df5 100644 --- a/internal/utilk8s/clientset.go +++ b/internal/utilk8s/clientset.go @@ -21,11 +21,10 @@ func SetupK8sClientSet(kubeconfigPath, contextName string) (*rest.Config, *kuber } } - chain := []string{} - loadingRules := &clientcmd.ClientConfigLoadingRules{} - // use splitlist func to use separator from OS specific kubeconfigFiles := filepath.SplitList(kubeconfigPath) + chain := make([]string, 0, len(kubeconfigFiles)) + loadingRules := &clientcmd.ClientConfigLoadingRules{} chain = append(chain, deduplicate(kubeconfigFiles)...) if len(chain) > 1 { diff --git a/pkg/libmirror/layouts/tag_resolver.go b/pkg/libmirror/layouts/tag_resolver.go index 36cfcc08..315585c0 100644 --- a/pkg/libmirror/layouts/tag_resolver.go +++ b/pkg/libmirror/layouts/tag_resolver.go @@ -60,12 +60,13 @@ func NopTagToDigestMappingFunc(_ string) *v1.Hash { // Example usage: // err := resolver.ResolveTagsDigestsForImageLayouts(mirrorCtx, layouts) func (r *TagsResolver) ResolveTagsDigestsForImageLayouts(mirrorCtx *params.BaseParams, layouts *ImageLayouts) error { - imageSets := []map[string]struct{}{ + imageSets := make([]map[string]struct{}, 0, 4+len(layouts.Modules)*2) + imageSets = append(imageSets, layouts.DeckhouseImages, layouts.ReleaseChannelImages, layouts.InstallImages, layouts.InstallStandaloneImages, - } + ) for _, moduleImageLayout := range layouts.Modules { imageSets = append(imageSets, moduleImageLayout.ModuleImages) diff --git a/pkg/stub/registry_client.go b/pkg/stub/registry_client.go index 92493f7b..c6014ed3 100644 --- a/pkg/stub/registry_client.go +++ b/pkg/stub/registry_client.go @@ -477,7 +477,7 @@ func (s *RegistryClientStub) initializeRegistries() { // Registry 1: dynamic source s.addRegistry(source, map[string][]string{ "": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, // including custom tag - "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions + "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, }) @@ -871,7 +871,13 @@ func (s *RegistryClientStub) PushImage(_ context.Context, _ string, _ v1.Image, // ListTags retrieves all available tags func (s *RegistryClientStub) ListTags(_ context.Context, _ ...registry.ListTagsOption) ([]string, error) { - var allTags []string + total := 0 + for _, regData := range s.registries { + for _, repoData := range regData.repositories { + total += len(repoData.tags) + } + } + allTags := make([]string, 0, total) for _, regData := range s.registries { for _, repoData := range regData.repositories { allTags = append(allTags, repoData.tags...) @@ -882,7 +888,11 @@ func (s *RegistryClientStub) ListTags(_ context.Context, _ ...registry.ListTagsO // ListRepositories retrieves all sub-repositories func (s *RegistryClientStub) ListRepositories(_ context.Context, _ ...registry.ListRepositoriesOption) ([]string, error) { - var allRepos []string + total := 0 + for _, regData := range s.registries { + total += len(regData.repositories) + } + allRepos := make([]string, 0, total) for _, regData := range s.registries { for repo := range regData.repositories { allRepos = append(allRepos, repo) From 91ae7071355aff2ef6a513997c1bb36fe8b7aa19 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Tue, 17 Feb 2026 10:08:28 +0300 Subject: [PATCH 10/14] add tags Signed-off-by: Pavel Okhlopkov --- internal/mirror/modules/modules.go | 2 +- internal/mirror/platform/layout.go | 9 +- internal/mirror/platform/platform.go | 273 +++++++++++++++---- internal/mirror/platform/platform_test.go | 315 ++++++++++++++++++++++ internal/mirror/security/layout.go | 2 +- internal/utilk8s/clientset.go | 5 +- pkg/libmirror/layouts/tag_resolver.go | 5 +- pkg/stub/registry_client.go | 39 ++- 8 files changed, 568 insertions(+), 82 deletions(-) create mode 100644 internal/mirror/platform/platform_test.go diff --git a/internal/mirror/modules/modules.go b/internal/mirror/modules/modules.go index c3580a66..e64b66fc 100644 --- a/internal/mirror/modules/modules.go +++ b/internal/mirror/modules/modules.go @@ -653,7 +653,7 @@ func (svc *Service) extractInternalDigestImages(ctx context.Context, moduleName // pullVexImages finds and pulls VEX attestation images for module images func (svc *Service) pullVexImages(ctx context.Context, moduleName string, downloadList *ImageDownloadList) { - allImages := make([]string, 0) + allImages := make([]string, 0, len(downloadList.Module)+len(downloadList.ModuleExtra)) for img := range downloadList.Module { allImages = append(allImages, img) diff --git a/internal/mirror/platform/layout.go b/internal/mirror/platform/layout.go index 36f7618c..9b2685e2 100644 --- a/internal/mirror/platform/layout.go +++ b/internal/mirror/platform/layout.go @@ -61,13 +61,8 @@ func (l *ImageDownloadList) FillDeckhouseImages(deckhouseVersions []string) { } } -func (l *ImageDownloadList) FillForTag(tag string) { - // If we are to pull only the specific requested version, we should not pull any release channels at all. - if tag != "" { - return - } - - for _, channel := range internal.GetAllDefaultReleaseChannels() { +func (l *ImageDownloadList) FillForChannels(channels []string) { + for _, channel := range channels { l.Deckhouse[l.rootURL+":"+channel] = nil l.DeckhouseInstall[path.Join(l.rootURL, internal.InstallSegment)+":"+channel] = nil l.DeckhouseInstallStandalone[path.Join(l.rootURL, internal.InstallStandaloneSegment)+":"+channel] = nil diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index d4c68b83..f4bec4a9 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -31,7 +31,6 @@ import ( "time" "github.com/Masterminds/semver/v3" - "github.com/samber/lo" dkplog "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/deckhouse/pkg/registry/client" @@ -51,6 +50,10 @@ type Options struct { // SinceVersion specifies the minimum version to start mirroring from (optional) SinceVersion *semver.Version // TargetTag specifies a specific tag to mirror instead of determining versions automatically + // it can be: + // semver f.e. vX.Y.Z + // channel f.e. alpha/beta/stable + // any other tag TargetTag string // BundleDir is the directory to store the bundle BundleDir string @@ -122,13 +125,13 @@ func (svc *Service) PullPlatform(ctx context.Context) error { return fmt.Errorf("validate platform access: %w", err) } - tagsToMirror, err := svc.findTagsToMirror(ctx) + tagsToMirror, channelsToMirror, err := svc.findTagsToMirror(ctx) if err != nil { return fmt.Errorf("find tags to mirror: %w", err) } svc.downloadList.FillDeckhouseImages(tagsToMirror) - svc.downloadList.FillForTag(svc.options.TargetTag) + svc.downloadList.FillForChannels(channelsToMirror) err = svc.pullDeckhousePlatform(ctx, tagsToMirror) if err != nil { @@ -177,93 +180,243 @@ func (svc *Service) validatePlatformAccess(ctx context.Context) error { // findTagsToMirror determines which Deckhouse release tags should be mirrored // If a specific target tag is set, it returns only that tag // Otherwise, it finds all relevant versions that should be mirrored based on channels and version ranges -func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, error) { - // If a specific tag is requested, skip the complex version determination logic +func (svc *Service) findTagsToMirror(ctx context.Context) ([]string, []string, error) { + strickTags := []string{} if svc.options.TargetTag != "" { - svc.userLogger.Infof("Skipped releases lookup as tag %q is specifically requested with --deckhouse-tag", svc.options.TargetTag) - - return []string{svc.options.TargetTag}, nil + strickTags = append(strickTags, svc.options.TargetTag) } - // Determine which versions should be mirrored based on release channels and version constraints - versionsToMirror, err := svc.versionsToMirrorFunc(ctx) + result, err := svc.versionsToMirror(ctx, strickTags) if err != nil { - return nil, fmt.Errorf("find versions to mirror: %w", err) + return nil, nil, fmt.Errorf("Find versions to mirror: %w", err) + } + + svc.userLogger.Infof("Deckhouse releases to pull: %+v", result.Versions) + + vers := make([]string, 0, len(result.Versions)+len(result.CustomTags)) + for _, v := range result.Versions { + vers = append(vers, "v"+v.String()) } + // Add custom tags as-is (without "v" prefix) + vers = append(vers, result.CustomTags...) - svc.userLogger.Infof("Deckhouse releases to pull: %+v", versionsToMirror) + return vers, result.Channels, nil +} + +type releaseChannelVersionResult struct { + ver *semver.Version + err error +} + +// VersionsToMirrorResult contains the result of versionsToMirror operation +type VersionsToMirrorResult struct { + // Versions contains semver versions to mirror + Versions []semver.Version + // Channels contains release channels to mirror + Channels []string + // CustomTags contains custom tags (non-semver, non-channel tags) to mirror + CustomTags []string +} - // Convert versions to tag format (add "v" prefix) - return lo.Map( - versionsToMirror, - func(v semver.Version, _ int) string { - return "v" + v.String() - }, - ), nil +// parsedTags represents the parsed input tags categorized by type +type parsedTags struct { + semverVersions []*semver.Version + customTags []string } -// versionsToMirrorFunc determines which Deckhouse release versions should be mirrored +// channelVersions represents the fetched versions from release channels +type channelVersions map[string]*semver.Version + +// versionsToMirror determines which Deckhouse release versions should be mirrored // It collects current versions from all release channels and filters available releases // to include only versions that should be mirrored based on the mirroring strategy -func (svc *Service) versionsToMirrorFunc(ctx context.Context) ([]semver.Version, error) { - logger := svc.userLogger +func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) (*VersionsToMirrorResult, error) { + if len(tagsToMirror) > 0 { + svc.userLogger.Infof("Skipped releases lookup as tag %q is specifically requested with --deckhouse-tag", svc.options.TargetTag) + } - releaseChannelsToCopy := internal.GetAllDefaultReleaseChannels() - releaseChannelsToCopy = append(releaseChannelsToCopy, internal.LTSChannel) + // Parse input tags into categories + parsed := svc.parseInputTags(tagsToMirror) - releaseChannelsVersions := make(map[string]*semver.Version, len(releaseChannelsToCopy)) - for _, channel := range releaseChannelsToCopy { - version, err := svc.getReleaseChannelVersionFromRegistry(ctx, channel) - if err != nil { - if channel == internal.LTSChannel { - if !errors.Is(err, client.ErrImageNotFound) { - svc.userLogger.Warnf("Skipping LTS channel: %v", err) - } else { - svc.userLogger.Warnf("Skipping LTS channel, because it's not required") - } + // Fetch current versions from all release channels + channelVersions, err := svc.fetchReleaseChannelVersions(ctx) + if err != nil { + return nil, err + } - continue - } + // Match channels and versions based on requested tags + versions, matchedChannels := svc.matchChannelsToTags(tagsToMirror, channelVersions, parsed.semverVersions) + + // If specific tags requested, return immediately + if len(tagsToMirror) > 0 { + return &VersionsToMirrorResult{ + Versions: deduplicateVersions(versions), + Channels: matchedChannels, + CustomTags: parsed.customTags, + }, nil + } + + // For full discovery mode, expand version range + expandedVersions, err := svc.expandVersionRange(ctx, channelVersions, versions) + if err != nil { + return nil, err + } + + return &VersionsToMirrorResult{ + Versions: deduplicateVersions(expandedVersions), + Channels: matchedChannels, + CustomTags: parsed.customTags, + }, nil +} + +// parseInputTags categorizes input tags into semver versions and custom tags +func (svc *Service) parseInputTags(tags []string) parsedTags { + result := parsedTags{ + semverVersions: make([]*semver.Version, 0, len(tags)), + customTags: make([]string, 0), + } - return nil, fmt.Errorf("get %s release version from registry: %w", channel, err) + for _, tag := range tags { + version, err := semver.NewVersion(tag) + if err != nil { + // Not a valid semver - check if it's a custom tag (not a channel name) + if !internal.ChannelIsValid(tag) { + result.customTags = append(result.customTags, tag) + } + continue } + result.semverVersions = append(result.semverVersions, version) + } - if version == nil { - // Channel was skipped (e.g., suspended and ignoreSuspendedChannels is true) + return result +} + +// fetchReleaseChannelVersions retrieves current versions from all release channels +func (svc *Service) fetchReleaseChannelVersions(ctx context.Context) (channelVersions, error) { + channelsToFetch := append(internal.GetAllDefaultReleaseChannels(), internal.LTSChannel) + + // Fetch versions from all channels + channelResults := make(map[string]releaseChannelVersionResult, len(channelsToFetch)) + for _, channel := range channelsToFetch { + version, err := svc.getReleaseChannelVersionFromRegistry(ctx, channel) + + // LTS channel is optional - warn and continue if missing + if err != nil && channel == internal.LTSChannel { + svc.userLogger.Warnf("Skipping LTS channel: %v", err) continue } - releaseChannelsVersions[channel] = version + channelResults[channel] = releaseChannelVersionResult{ver: version, err: err} } - rockSolidVersion := releaseChannelsVersions[internal.RockSolidChannel] + // Validate and extract successful channel versions + return svc.validateChannelResults(channelResults) +} - mirrorFromVersion := *rockSolidVersion +// validateChannelResults validates channel fetch results and extracts successful versions +func (svc *Service) validateChannelResults(results map[string]releaseChannelVersionResult) (channelVersions, error) { + versions := make(channelVersions, len(results)) + _, ltsExists := results[internal.LTSChannel] - if svc.options.SinceVersion != nil { - if svc.options.SinceVersion.LessThan(rockSolidVersion) { - mirrorFromVersion = *svc.options.SinceVersion + for channel, result := range results { + // If LTS doesn't exist, all other channels must succeed + if !ltsExists && result.err != nil { + return nil, fmt.Errorf("get %s release version from registry: %w", channel, result.err) + } + + if result.err == nil { + versions[channel] = result.ver } } - logger.Debugf("listing deckhouse releases") + return versions, nil +} + +// matchChannelsToTags matches requested tags to channel versions and returns matching versions and channels +func (svc *Service) matchChannelsToTags(requestedTags []string, channelVersions channelVersions, semverVersions []*semver.Version) ([]*semver.Version, []string) { + versions := make([]*semver.Version, 0, len(semverVersions)) + versions = append(versions, semverVersions...) + + matchedChannels := make(map[string]struct{}) + + // If no specific tags requested, mirror all channels + if len(requestedTags) == 0 { + for channel, version := range channelVersions { + versions = append(versions, version) + matchedChannels[channel] = struct{}{} + } + return versions, mapKeysToSlice(matchedChannels) + } + + // Match specific tags to channels + for channel, version := range channelVersions { + for _, tag := range requestedTags { + if svc.tagMatchesChannel(tag, channel, version) { + versions = append(versions, version) + matchedChannels[channel] = struct{}{} + break + } + } + } - tags, err := svc.deckhouseService.ReleaseChannels().ListTags(ctx) + return versions, mapKeysToSlice(matchedChannels) +} + +// tagMatchesChannel checks if a tag matches a channel (by name or version) +func (svc *Service) tagMatchesChannel(tag, channelName string, channelVersion *semver.Version) bool { + return tag == channelName || tag == "v"+channelVersion.String() +} + +// expandVersionRange expands the version range for full discovery mode +func (svc *Service) expandVersionRange(ctx context.Context, channelVersions channelVersions, baseVersions []*semver.Version) ([]*semver.Version, error) { + minVersion := svc.determineMinimumVersion(channelVersions) + maxVersion := channelVersions[internal.AlphaChannel] + + if maxVersion == nil { + // No alpha channel - return base versions only + return baseVersions, nil + } + + svc.userLogger.Debugf("listing deckhouse releases") + + // Fetch all available tags + allTags, err := svc.deckhouseService.ReleaseChannels().ListTags(ctx) if err != nil { return nil, fmt.Errorf("get tags from Deckhouse registry: %w", err) } - alphaChannelVersion := releaseChannelsVersions[internal.AlphaChannel] + // Filter and get latest patches + filteredVersions := filterVersionsBetween(minVersion, maxVersion, allTags) + latestPatches := filterOnlyLatestPatches(filteredVersions) + + return append(baseVersions, latestPatches...), nil +} - versionsAboveMinimal := filterVersionsBetween(&mirrorFromVersion, alphaChannelVersion, tags) - versionsAboveMinimal = filterOnlyLatestPatches(versionsAboveMinimal) +// determineMinimumVersion determines the minimum version for mirroring based on configuration +func (svc *Service) determineMinimumVersion(channelVersions channelVersions) *semver.Version { + rockSolidVersion := channelVersions[internal.RockSolidChannel] + if rockSolidVersion == nil { + return nil + } - vers := make([]*semver.Version, 0, len(releaseChannelsVersions)) - for _, v := range releaseChannelsVersions { - vers = append(vers, v) + // Use rock-solid as baseline + minVersion := rockSolidVersion + + // Override with SinceVersion if it's older + if svc.options.SinceVersion != nil && svc.options.SinceVersion.LessThan(rockSolidVersion) { + minVersion = svc.options.SinceVersion } - return deduplicateVersions(append(vers, versionsAboveMinimal...)), nil + return minVersion +} + +// mapKeysToSlice converts a map's keys to a slice +func mapKeysToSlice(m map[string]struct{}) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + return keys } // getReleaseChannelVersionFromRegistry retrieves the current version for a specific release channel @@ -709,15 +862,19 @@ func filterOnlyLatestPatches(versions []*semver.Version) []*semver.Version { // deduplicateVersions removes duplicate versions from the list. // This is necessary because channel versions and filtered versions might overlap. func deduplicateVersions(versions []*semver.Version) []semver.Version { - m := map[semver.Version]struct{}{} + m := map[string]struct{}{} for _, v := range versions { - m[*v] = struct{}{} + if v == nil { + continue + } + m[v.String()] = struct{}{} } vers := make([]semver.Version, 0, len(m)) - for k := range maps.Keys(m) { - vers = append(vers, k) + for s := range m { + // semver.MustParse returns a canonical semver.Version value suitable for comparison + vers = append(vers, *semver.MustParse("v" + s)) } return vers diff --git a/internal/mirror/platform/platform_test.go b/internal/mirror/platform/platform_test.go new file mode 100644 index 00000000..fd68acc2 --- /dev/null +++ b/internal/mirror/platform/platform_test.go @@ -0,0 +1,315 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package platform + +import ( + "context" + "log/slog" + "testing" + + "github.com/Masterminds/semver/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + dkplog "github.com/deckhouse/deckhouse/pkg/log" + + "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" + registryservice "github.com/deckhouse/deckhouse-cli/pkg/registry/service" + "github.com/deckhouse/deckhouse-cli/pkg/stub" +) + +func TestService_versionsToMirror(t *testing.T) { + tests := []struct { + name string + strictTags []string + options *Options + wantVersions []string // expected versions in format "v1.72.10", etc. + wantChannels []string // expected channels + wantCustomTags []string // expected custom tags + wantErr bool + wantErrContains string + }{ + { + name: "with strict tags - single semver version", + strictTags: []string{"v1.72.10"}, + options: &Options{}, + wantVersions: []string{ + "v1.72.10", + }, + // v1.72.10 matches only alpha channel version + wantChannels: []string{"alpha"}, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "with strict tags - multiple semver versions", + strictTags: []string{"v1.72.10", "v1.71.0", "v1.70.0"}, + options: &Options{}, + wantVersions: []string{ + "v1.72.10", + "v1.71.0", + "v1.70.0", + }, + // v1.72.10 matches alpha, v1.71.0 matches beta, v1.70.0 matches early-access + wantChannels: []string{ + "alpha", "beta", "early-access", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "with strict tags - channel name", + strictTags: []string{"stable"}, + options: &Options{}, + wantVersions: []string{ + "v1.69.0", // stable channel version + }, + wantChannels: []string{ + "stable", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "with strict tags - custom tag", + strictTags: []string{"pr12345"}, + options: &Options{}, + wantVersions: []string{ + // No semver versions for custom tag + }, + wantChannels: []string{ + // No channels matched + }, + wantCustomTags: []string{ + "pr12345", // custom tag is returned as-is + }, + wantErr: false, + }, + { + name: "no strict tags - full discovery", + strictTags: []string{}, + options: &Options{}, + // When no strict tags, it returns all channel versions + wantVersions: []string{ + "v1.72.10", // alpha + "v1.71.0", // beta + "v1.70.0", // early-access + "v1.69.0", // stable + "v1.68.0", // rock-solid + }, + wantChannels: []string{ + "alpha", "beta", "early-access", "stable", "rock-solid", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + { + name: "no strict tags with SinceVersion", + strictTags: []string{}, + options: &Options{ + SinceVersion: semver.MustParse("v1.69.0"), + }, + // When no strict tags, it returns all channel versions + wantVersions: []string{ + "v1.72.10", // alpha + "v1.71.0", // beta + "v1.70.0", // early-access + "v1.69.0", // stable + "v1.68.0", // rock-solid + }, + wantChannels: []string{ + "alpha", "beta", "early-access", "stable", "rock-solid", + }, + wantCustomTags: []string{}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: tt.options, + logger: logger, + userLogger: userLogger, + } + + // Call the function under test + result, err := svc.versionsToMirror(context.Background(), tt.strictTags) + + // Check error + if tt.wantErr { + require.Error(t, err) + if tt.wantErrContains != "" { + assert.Contains(t, err.Error(), tt.wantErrContains) + } + return + } + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Check versions (order doesn't matter for this test) + assert.ElementsMatch(t, tt.wantVersions, gotVersions, "versions mismatch") + + // Check channels (order doesn't matter) + assert.ElementsMatch(t, tt.wantChannels, result.Channels, "channels mismatch") + + // Check custom tags (order doesn't matter) + assert.ElementsMatch(t, tt.wantCustomTags, result.CustomTags, "custom tags mismatch") + }) + } +} + +func TestService_versionsToMirror_WithTargetTag(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance with TargetTag + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: &Options{ + TargetTag: "v1.72.10", + }, + logger: logger, + userLogger: userLogger, + } + + // Call the function under test + result, err := svc.versionsToMirror(context.Background(), []string{"v1.72.10"}) + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Should only return the specific tag + assert.ElementsMatch(t, []string{"v1.72.10"}, gotVersions) + // Channels should match the version (alpha channel has v1.72.10) + assert.ElementsMatch(t, []string{"alpha"}, result.Channels) + // No custom tags + assert.Empty(t, result.CustomTags) +} + +func TestService_versionsToMirror_CustomTagWithSemver(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: &Options{}, + logger: logger, + userLogger: userLogger, + } + + // Call with mix of semver version and custom tag + strictTags := []string{"v1.72.10", "pr12345"} + result, err := svc.versionsToMirror(context.Background(), strictTags) + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Should have both semver version and custom tag + assert.Contains(t, gotVersions, "v1.72.10", "should include semver version") + assert.Contains(t, result.CustomTags, "pr12345", "should include custom tag") + assert.Contains(t, result.Channels, "alpha", "should include alpha channel") +} + +func TestService_versionsToMirror_Deduplication(t *testing.T) { + // Create stub registry client + stubClient := stub.NewRegistryClientStub() + + // Create logger + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelInfo)) + userLogger := log.NewSLogger(slog.LevelInfo) + + // Create DeckhouseService with stub client + deckhouseService := registryservice.NewDeckhouseService(stubClient, logger) + + // Create Service instance + svc := &Service{ + deckhouseService: deckhouseService, + downloadList: NewImageDownloadList(stubClient.GetRegistry()), + options: &Options{}, + logger: logger, + userLogger: userLogger, + } + + // Call with duplicate versions in strictTags + strictTags := []string{"v1.72.10", "v1.72.10", "alpha"} // alpha also points to v1.72.10 + result, err := svc.versionsToMirror(context.Background(), strictTags) + require.NoError(t, err) + require.NotNil(t, result) + + // Convert versions to strings for easier comparison + gotVersions := make([]string, len(result.Versions)) + for i, v := range result.Versions { + gotVersions[i] = "v" + v.String() + } + + // Should deduplicate + assert.Equal(t, 1, len(gotVersions), "expected deduplicated versions") + assert.Contains(t, gotVersions, "v1.72.10") + + // Channels should include alpha + assert.Contains(t, result.Channels, "alpha") + + // No custom tags + assert.Empty(t, result.CustomTags) +} diff --git a/internal/mirror/security/layout.go b/internal/mirror/security/layout.go index 011302a4..73e064d8 100644 --- a/internal/mirror/security/layout.go +++ b/internal/mirror/security/layout.go @@ -102,7 +102,7 @@ func (l *ImageLayouts) setLayoutByMirrorType(rootFolder string, mirrorType inter // AsList returns a list of layout.Path's in it. Undefined path's are not included in the list. func (l *ImageLayouts) AsList() []layout.Path { - paths := make([]layout.Path, 0) + paths := make([]layout.Path, 0, len(l.Security)) for _, layout := range l.Security { paths = append(paths, layout.Path()) } diff --git a/internal/utilk8s/clientset.go b/internal/utilk8s/clientset.go index c91f1f6e..c8068df5 100644 --- a/internal/utilk8s/clientset.go +++ b/internal/utilk8s/clientset.go @@ -21,11 +21,10 @@ func SetupK8sClientSet(kubeconfigPath, contextName string) (*rest.Config, *kuber } } - chain := []string{} - loadingRules := &clientcmd.ClientConfigLoadingRules{} - // use splitlist func to use separator from OS specific kubeconfigFiles := filepath.SplitList(kubeconfigPath) + chain := make([]string, 0, len(kubeconfigFiles)) + loadingRules := &clientcmd.ClientConfigLoadingRules{} chain = append(chain, deduplicate(kubeconfigFiles)...) if len(chain) > 1 { diff --git a/pkg/libmirror/layouts/tag_resolver.go b/pkg/libmirror/layouts/tag_resolver.go index 36cfcc08..315585c0 100644 --- a/pkg/libmirror/layouts/tag_resolver.go +++ b/pkg/libmirror/layouts/tag_resolver.go @@ -60,12 +60,13 @@ func NopTagToDigestMappingFunc(_ string) *v1.Hash { // Example usage: // err := resolver.ResolveTagsDigestsForImageLayouts(mirrorCtx, layouts) func (r *TagsResolver) ResolveTagsDigestsForImageLayouts(mirrorCtx *params.BaseParams, layouts *ImageLayouts) error { - imageSets := []map[string]struct{}{ + imageSets := make([]map[string]struct{}, 0, 4+len(layouts.Modules)*2) + imageSets = append(imageSets, layouts.DeckhouseImages, layouts.ReleaseChannelImages, layouts.InstallImages, layouts.InstallStandaloneImages, - } + ) for _, moduleImageLayout := range layouts.Modules { imageSets = append(imageSets, moduleImageLayout.ModuleImages) diff --git a/pkg/stub/registry_client.go b/pkg/stub/registry_client.go index e0449a3c..c6014ed3 100644 --- a/pkg/stub/registry_client.go +++ b/pkg/stub/registry_client.go @@ -476,20 +476,20 @@ func (s *RegistryClientStub) initializeRegistries() { // Registry 1: dynamic source s.addRegistry(source, map[string][]string{ - "": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, - "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions - "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, - "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, + "": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, // including custom tag + "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions + "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, + "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, }) // Registry 2: gcr.io s.addRegistry("gcr.io/google-containers", map[string][]string{ - "": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, + "": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, "pause": {"3.9", "latest"}, "kube-apiserver": {"v1.28.0", "v1.29.0", "latest"}, "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, - "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, - "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "alpha", "beta", "early-access", "stable", "rock-solid"}, + "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, + "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, }) // Registry 3: quay.io @@ -843,7 +843,16 @@ func (s *RegistryClientStub) GetImage(_ context.Context, tag string, _ ...regist } } - // Fall back to all registries + // Fall back to searching in the specific registry first, then all registries + if regData, exists := s.registries[registry]; exists { + for _, repoData := range regData.repositories { + if imageData, exists := repoData.images[tag]; exists { + return imageData.image.(pkg.ClientImage), nil + } + } + } + + // Last resort: search all registries for _, regData := range s.registries { for _, repoData := range regData.repositories { if imageData, exists := repoData.images[tag]; exists { @@ -862,7 +871,13 @@ func (s *RegistryClientStub) PushImage(_ context.Context, _ string, _ v1.Image, // ListTags retrieves all available tags func (s *RegistryClientStub) ListTags(_ context.Context, _ ...registry.ListTagsOption) ([]string, error) { - var allTags []string + total := 0 + for _, regData := range s.registries { + for _, repoData := range regData.repositories { + total += len(repoData.tags) + } + } + allTags := make([]string, 0, total) for _, regData := range s.registries { for _, repoData := range regData.repositories { allTags = append(allTags, repoData.tags...) @@ -873,7 +888,11 @@ func (s *RegistryClientStub) ListTags(_ context.Context, _ ...registry.ListTagsO // ListRepositories retrieves all sub-repositories func (s *RegistryClientStub) ListRepositories(_ context.Context, _ ...registry.ListRepositoriesOption) ([]string, error) { - var allRepos []string + total := 0 + for _, regData := range s.registries { + total += len(regData.repositories) + } + allRepos := make([]string, 0, total) for _, regData := range s.registries { for repo := range regData.repositories { allRepos = append(allRepos, repo) From 82f385a4ea317b9aeac2a898ad3ab5f7f8de2856 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Tue, 17 Feb 2026 11:37:11 +0300 Subject: [PATCH 11/14] bump Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/platform_test.go | 22 +++++++++--------- pkg/registry/service/deckhouse_service.go | 10 +++++---- pkg/stub/registry_client.go | 27 +++++++++++++++-------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/internal/mirror/platform/platform_test.go b/internal/mirror/platform/platform_test.go index fd68acc2..00071f35 100644 --- a/internal/mirror/platform/platform_test.go +++ b/internal/mirror/platform/platform_test.go @@ -34,14 +34,14 @@ import ( func TestService_versionsToMirror(t *testing.T) { tests := []struct { - name string - strictTags []string - options *Options - wantVersions []string // expected versions in format "v1.72.10", etc. - wantChannels []string // expected channels - wantCustomTags []string // expected custom tags - wantErr bool - wantErrContains string + name string + strictTags []string + options *Options + wantVersions []string // expected versions in format "v1.72.10", etc. + wantChannels []string // expected channels + wantCustomTags []string // expected custom tags + wantErr bool + wantErrContains string }{ { name: "with strict tags - single semver version", @@ -85,9 +85,9 @@ func TestService_versionsToMirror(t *testing.T) { wantErr: false, }, { - name: "with strict tags - custom tag", - strictTags: []string{"pr12345"}, - options: &Options{}, + name: "with strict tags - custom tag", + strictTags: []string{"pr12345"}, + options: &Options{}, wantVersions: []string{ // No semver versions for custom tag }, diff --git a/pkg/registry/service/deckhouse_service.go b/pkg/registry/service/deckhouse_service.go index 7b21ce96..5244fbf0 100644 --- a/pkg/registry/service/deckhouse_service.go +++ b/pkg/registry/service/deckhouse_service.go @@ -56,10 +56,12 @@ func NewDeckhouseService(client registry.Client, logger *log.Logger) *DeckhouseS return &DeckhouseService{ client: client, - BasicService: NewBasicService(deckhouseServiceName, client, logger), - deckhouseReleaseChannels: NewDeckhouseReleaseService(NewBasicService(deckhouseReleaseChannelsServiceName, client.WithSegment(deckhouseReleaseChannelsSegment), logger)), - installer: NewBasicService(installerServiceName, client.WithSegment(installerSegment), logger), - standaloneInstaller: NewBasicService(standaloneInstallerServiceName, client.WithSegment(installStandaloneSegment), logger), + BasicService: NewBasicService(deckhouseServiceName, client, logger), + deckhouseReleaseChannels: NewDeckhouseReleaseService( + NewBasicService(deckhouseReleaseChannelsServiceName, client.WithSegment(deckhouseReleaseChannelsSegment), logger), + ), + installer: NewBasicService(installerServiceName, client.WithSegment(installerSegment), logger), + standaloneInstaller: NewBasicService(standaloneInstallerServiceName, client.WithSegment(installStandaloneSegment), logger), logger: logger, } diff --git a/pkg/stub/registry_client.go b/pkg/stub/registry_client.go index c6014ed3..5030dcfd 100644 --- a/pkg/stub/registry_client.go +++ b/pkg/stub/registry_client.go @@ -476,10 +476,10 @@ func (s *RegistryClientStub) initializeRegistries() { // Registry 1: dynamic source s.addRegistry(source, map[string][]string{ - "": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, // including custom tag - "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions - "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, - "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, + "": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, // including custom tag + "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, // channel names, not versions + "install": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, + "install-standalone": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, }) // Registry 2: gcr.io @@ -690,12 +690,21 @@ func (s *RegistryClientStub) createMockImageData(reg, repo, tag string) *ImageDa // WithSegment creates a new client with an additional scope path segment func (s *RegistryClientStub) WithSegment(segments ...string) registry.Client { - newRegistry := s.currentRegistry - if len(segments) > 0 { - if newRegistry == "" { - newRegistry = strings.Join(segments, "/") + // If no current registry is set, use the stub's default registry as base + base := s.currentRegistry + if base == "" { + base = s.GetRegistry() + } + + var newRegistry string + if len(segments) == 0 { + newRegistry = base + } else { + segPath := strings.Join(segments, "/") + if base == "" { + newRegistry = segPath } else { - newRegistry = newRegistry + "/" + strings.Join(segments, "/") + newRegistry = base + "/" + segPath } } From 9964a396562496c33b17d9f103829a81649f58ad Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Tue, 17 Feb 2026 11:54:14 +0300 Subject: [PATCH 12/14] fix since version Signed-off-by: Pavel Okhlopkov --- internal/mirror/platform/platform.go | 34 ++++++++++++++++++++--- internal/mirror/platform/platform_test.go | 3 +- pkg/stub/registry_client.go | 28 +++++++++++++------ 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/internal/mirror/platform/platform.go b/internal/mirror/platform/platform.go index f4bec4a9..e690ad00 100644 --- a/internal/mirror/platform/platform.go +++ b/internal/mirror/platform/platform.go @@ -262,9 +262,21 @@ func (svc *Service) versionsToMirror(ctx context.Context, tagsToMirror []string) return nil, err } + // Filter out channels that are below the minimum version (SinceVersion/rock-solid) + minVersion := svc.determineMinimumVersion(channelVersions) + filteredChannels := make([]string, 0, len(matchedChannels)) + for _, ch := range matchedChannels { + if minVersion != nil { + if v, ok := channelVersions[ch]; ok && v != nil && v.LessThan(minVersion) { + continue + } + } + filteredChannels = append(filteredChannels, ch) + } + return &VersionsToMirrorResult{ Versions: deduplicateVersions(expandedVersions), - Channels: matchedChannels, + Channels: filteredChannels, CustomTags: parsed.customTags, }, nil } @@ -389,7 +401,20 @@ func (svc *Service) expandVersionRange(ctx context.Context, channelVersions chan filteredVersions := filterVersionsBetween(minVersion, maxVersion, allTags) latestPatches := filterOnlyLatestPatches(filteredVersions) - return append(baseVersions, latestPatches...), nil + // Filter base channel versions by minVersion as well + filteredBase := baseVersions + if minVersion != nil { + nb := make([]*semver.Version, 0, len(baseVersions)) + for _, v := range baseVersions { + if v == nil || v.LessThan(minVersion) { + continue + } + nb = append(nb, v) + } + filteredBase = nb + } + + return append(filteredBase, latestPatches...), nil } // determineMinimumVersion determines the minimum version for mirroring based on configuration @@ -402,8 +427,9 @@ func (svc *Service) determineMinimumVersion(channelVersions channelVersions) *se // Use rock-solid as baseline minVersion := rockSolidVersion - // Override with SinceVersion if it's older - if svc.options.SinceVersion != nil && svc.options.SinceVersion.LessThan(rockSolidVersion) { + // If SinceVersion is provided and is newer than rock-solid, start from SinceVersion + // (user wants to mirror from a later version than rock-solid) + if svc.options.SinceVersion != nil && svc.options.SinceVersion.GreaterThan(minVersion) { minVersion = svc.options.SinceVersion } diff --git a/internal/mirror/platform/platform_test.go b/internal/mirror/platform/platform_test.go index 00071f35..162c073c 100644 --- a/internal/mirror/platform/platform_test.go +++ b/internal/mirror/platform/platform_test.go @@ -129,10 +129,9 @@ func TestService_versionsToMirror(t *testing.T) { "v1.71.0", // beta "v1.70.0", // early-access "v1.69.0", // stable - "v1.68.0", // rock-solid }, wantChannels: []string{ - "alpha", "beta", "early-access", "stable", "rock-solid", + "alpha", "beta", "early-access", "stable", }, wantCustomTags: []string{}, wantErr: false, diff --git a/pkg/stub/registry_client.go b/pkg/stub/registry_client.go index 5030dcfd..0307ea56 100644 --- a/pkg/stub/registry_client.go +++ b/pkg/stub/registry_client.go @@ -42,6 +42,8 @@ import ( type RegistryClientStub struct { registries map[string]*RegistryData currentRegistry string + // defaultRegistry holds the deterministic 'source' registry added during initialization + defaultRegistry string } // RegistryData holds data for a specific registry @@ -474,6 +476,9 @@ func getSourceFromArgs() string { func (s *RegistryClientStub) initializeRegistries() { source := getSourceFromArgs() + // record deterministic source registry so stub resolves segments consistently + s.defaultRegistry = source + // Registry 1: dynamic source s.addRegistry(source, map[string][]string{ "": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0", "pr12345"}, // including custom tag @@ -484,18 +489,18 @@ func (s *RegistryClientStub) initializeRegistries() { // Registry 2: gcr.io s.addRegistry("gcr.io/google-containers", map[string][]string{ - "": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, - "pause": {"3.9", "latest"}, - "kube-apiserver": {"v1.28.0", "v1.29.0", "latest"}, + "": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, + "pause": {"alpha", "beta", "early-access", "stable", "rock-solid", "3.9", "latest"}, + "kube-apiserver": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.28.0", "v1.29.0", "latest"}, "release-channel": {"alpha", "beta", "early-access", "stable", "rock-solid"}, - "install": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, - "install-standalone": {"v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, + "install": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, + "install-standalone": {"alpha", "beta", "early-access", "stable", "rock-solid", "v1.72.10", "v1.71.0", "v1.70.0", "v1.69.0", "v1.68.0"}, }) // Registry 3: quay.io s.addRegistry("quay.io/prometheus", map[string][]string{ - "prometheus": {"v2.45.0", "v2.46.0", "latest"}, - "alertmanager": {"v0.26.0", "v0.27.0", "latest"}, + "prometheus": {"alpha", "beta", "early-access", "stable", "rock-solid", "v2.45.0", "v2.46.0", "latest"}, + "alertmanager": {"alpha", "beta", "early-access", "stable", "rock-solid", "v0.26.0", "v0.27.0", "latest"}, }) } @@ -690,10 +695,10 @@ func (s *RegistryClientStub) createMockImageData(reg, repo, tag string) *ImageDa // WithSegment creates a new client with an additional scope path segment func (s *RegistryClientStub) WithSegment(segments ...string) registry.Client { - // If no current registry is set, use the stub's default registry as base + // If no current registry is set, use the stub's deterministic default registry as base base := s.currentRegistry if base == "" { - base = s.GetRegistry() + base = s.defaultRegistry } var newRegistry string @@ -702,6 +707,7 @@ func (s *RegistryClientStub) WithSegment(segments ...string) registry.Client { } else { segPath := strings.Join(segments, "/") if base == "" { + // fall back to segment-only (legacy behavior) newRegistry = segPath } else { newRegistry = base + "/" + segPath @@ -717,6 +723,10 @@ func (s *RegistryClientStub) WithSegment(segments ...string) registry.Client { // GetRegistry returns the full registry path func (s *RegistryClientStub) GetRegistry() string { if s.currentRegistry == "" { + if s.defaultRegistry != "" { + return s.defaultRegistry + } + for registry := range s.registries { return registry } From 80c1bc2045972be3d7edc0ef416a2548eed5a590 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Tue, 17 Feb 2026 13:22:51 +0300 Subject: [PATCH 13/14] remove legacy Signed-off-by: Pavel Okhlopkov --- internal/mirror/cmd/pull/pull.go | 2 +- .../mirror}/modules/constraints.go | 16 + .../mirror}/modules/filter.go | 0 .../mirror}/modules/filter_test.go | 0 internal/mirror/modules/modules.go | 29 +- internal/mirror/operations/pull_dkp.go | 175 ------ internal/mirror/operations/pull_dkp_test.go | 302 ---------- internal/mirror/operations/pull_modules.go | 169 ------ .../mirror/operations/pull_modules_test.go | 302 ---------- internal/mirror/operations/pull_security.go | 94 ---- internal/mirror/operations/push_dkp.go | 80 --- internal/mirror/operations/push_dkp_test.go | 336 ----------- internal/mirror/operations/push_modules.go | 110 ---- .../mirror/operations/push_modules_test.go | 524 ------------------ internal/mirror/operations/push_security.go | 79 --- .../mirror/operations/push_security_test.go | 311 ----------- internal/mirror/pull.go | 3 +- internal/mirror/releases/versions.go | 287 ---------- internal/mirror/releases/versions_test.go | 226 -------- pkg/libmirror/layouts/indexes.go | 123 ++++ pkg/libmirror/layouts/indexes_test.go | 8 + pkg/libmirror/layouts/layouts.go | 524 ------------------ pkg/libmirror/layouts/layouts_test.go | 61 -- pkg/libmirror/layouts/pull.go | 321 ----------- pkg/libmirror/layouts/pull_test.go | 228 -------- pkg/libmirror/layouts/push.go | 193 ------- pkg/libmirror/layouts/push_test.go | 141 ----- pkg/libmirror/layouts/tag_resolver.go | 126 ----- pkg/libmirror/layouts/tag_resolver_test.go | 131 ----- pkg/libmirror/modules/modules.go | 338 ----------- 30 files changed, 172 insertions(+), 5067 deletions(-) rename {pkg/libmirror => internal/mirror}/modules/constraints.go (74%) rename {pkg/libmirror => internal/mirror}/modules/filter.go (100%) rename {pkg/libmirror => internal/mirror}/modules/filter_test.go (100%) delete mode 100644 internal/mirror/operations/pull_dkp.go delete mode 100644 internal/mirror/operations/pull_dkp_test.go delete mode 100644 internal/mirror/operations/pull_modules.go delete mode 100644 internal/mirror/operations/pull_modules_test.go delete mode 100644 internal/mirror/operations/pull_security.go delete mode 100644 internal/mirror/operations/push_dkp.go delete mode 100644 internal/mirror/operations/push_dkp_test.go delete mode 100644 internal/mirror/operations/push_modules.go delete mode 100644 internal/mirror/operations/push_modules_test.go delete mode 100644 internal/mirror/operations/push_security.go delete mode 100644 internal/mirror/operations/push_security_test.go delete mode 100644 internal/mirror/releases/versions.go delete mode 100644 internal/mirror/releases/versions_test.go delete mode 100644 pkg/libmirror/layouts/layouts.go delete mode 100644 pkg/libmirror/layouts/layouts_test.go delete mode 100644 pkg/libmirror/layouts/pull.go delete mode 100644 pkg/libmirror/layouts/pull_test.go delete mode 100644 pkg/libmirror/layouts/push.go delete mode 100644 pkg/libmirror/layouts/push_test.go delete mode 100644 pkg/libmirror/layouts/tag_resolver.go delete mode 100644 pkg/libmirror/layouts/tag_resolver_test.go delete mode 100644 pkg/libmirror/modules/modules.go diff --git a/internal/mirror/cmd/pull/pull.go b/internal/mirror/cmd/pull/pull.go index d6625d35..4902571c 100644 --- a/internal/mirror/cmd/pull/pull.go +++ b/internal/mirror/cmd/pull/pull.go @@ -43,8 +43,8 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/mirror" pullflags "github.com/deckhouse/deckhouse-cli/internal/mirror/cmd/pull/flags" "github.com/deckhouse/deckhouse-cli/internal/mirror/gostsums" + "github.com/deckhouse/deckhouse-cli/internal/mirror/modules" "github.com/deckhouse/deckhouse-cli/internal/version" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/modules" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/validation" diff --git a/pkg/libmirror/modules/constraints.go b/internal/mirror/modules/constraints.go similarity index 74% rename from pkg/libmirror/modules/constraints.go rename to internal/mirror/modules/constraints.go index db85a5fb..bc539373 100644 --- a/pkg/libmirror/modules/constraints.go +++ b/internal/mirror/modules/constraints.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package modules import ( diff --git a/pkg/libmirror/modules/filter.go b/internal/mirror/modules/filter.go similarity index 100% rename from pkg/libmirror/modules/filter.go rename to internal/mirror/modules/filter.go diff --git a/pkg/libmirror/modules/filter_test.go b/internal/mirror/modules/filter_test.go similarity index 100% rename from pkg/libmirror/modules/filter_test.go rename to internal/mirror/modules/filter_test.go diff --git a/internal/mirror/modules/modules.go b/internal/mirror/modules/modules.go index e64b66fc..2c793abe 100644 --- a/internal/mirror/modules/modules.go +++ b/internal/mirror/modules/modules.go @@ -29,6 +29,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver" dkplog "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/deckhouse/pkg/registry/client" @@ -37,15 +38,31 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/mirror/puller" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - libmodules "github.com/deckhouse/deckhouse-cli/pkg/libmirror/modules" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" registryservice "github.com/deckhouse/deckhouse-cli/pkg/registry/service" ) +type Module struct { + Name string + RegistryPath string + Releases []string +} + +func (m *Module) Versions() []*semver.Version { + versions := make([]*semver.Version, 0) + for _, release := range m.Releases { + v, err := semver.NewVersion(release) + if err == nil { + versions = append(versions, v) + } + } + return versions +} + // Options contains configuration options for the modules service type Options struct { // Filter is the module filter (whitelist/blacklist) - Filter *libmodules.Filter + Filter *Filter // OnlyExtraImages pulls only extra images without main module images OnlyExtraImages bool // BundleDir is the directory to store the bundle @@ -93,7 +110,7 @@ func NewService( // Create default filter (blacklist with no items = accept all) if options.Filter == nil { - filter, _ := libmodules.NewFilter(nil, libmodules.FilterTypeBlacklist) + filter, _ := NewFilter(nil, FilterTypeBlacklist) options.Filter = filter } @@ -176,7 +193,7 @@ func (svc *Service) pullModules(ctx context.Context) error { // Filter modules according to whitelist/blacklist filteredModules := make([]moduleData, 0) for _, moduleName := range moduleNames { - mod := &libmodules.Module{ + mod := &Module{ Name: moduleName, RegistryPath: filepath.Join(svc.rootURL, "modules", moduleName), } @@ -289,7 +306,7 @@ func (svc *Service) pullSingleModule(ctx context.Context, module moduleData) err } // Check for explicit version constraints from filter - mod := &libmodules.Module{ + mod := &Module{ Name: module.name, RegistryPath: module.registryPath, } @@ -724,7 +741,7 @@ func (svc *Service) applyChannelAliases(moduleName string) error { return nil } - exact, ok := constraint.(*libmodules.ExactTagConstraint) + exact, ok := constraint.(*ExactTagConstraint) if !ok { return nil } diff --git a/internal/mirror/operations/pull_dkp.go b/internal/mirror/operations/pull_dkp.go deleted file mode 100644 index 605552b5..00000000 --- a/internal/mirror/operations/pull_dkp.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "context" - "errors" - "fmt" - "io" - "maps" - "os" - "path/filepath" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/internal" - "github.com/deckhouse/deckhouse-cli/internal/mirror/chunked" - "github.com/deckhouse/deckhouse-cli/internal/mirror/manifests" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/images" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -func PullDeckhousePlatform(pullParams *params.PullParams, channelsToMirror []string, tagsToMirror []string, client registry.Client) error { - logger := pullParams.Logger - if len(tagsToMirror) == 0 { - return fmt.Errorf("no tags to mirror") - } - - tmpDir := filepath.Join(pullParams.WorkingDir, "platform") - - logger.Infof("Creating OCI Image Layouts") - imageLayouts, err := layouts.CreateOCIImageLayoutsForDeckhouse(tmpDir, nil) - if err != nil { - return fmt.Errorf("Create OCI Image Layouts: %w", err) - } - - layouts.FillLayoutsWithBasicDeckhouseImages(pullParams, imageLayouts, channelsToMirror, tagsToMirror) - logger.Infof("Resolving tags") - if err = imageLayouts.TagsResolver.ResolveTagsDigestsForImageLayouts(&pullParams.BaseParams, imageLayouts); err != nil { - return fmt.Errorf("Resolve images tags to digests: %w", err) - } - - if err = logger.Process("Pull release channels and installers", func() error { - if err = layouts.PullDeckhouseReleaseChannels(pullParams, imageLayouts, client); err != nil { - return fmt.Errorf("Pull release channels: %w", err) - } - - if err = layouts.PullInstallers(pullParams, imageLayouts, client); err != nil { - return fmt.Errorf("Pull installers: %w", err) - } - - if err = layouts.PullStandaloneInstallers(pullParams, imageLayouts, client); err != nil { - return fmt.Errorf("Pull standalone installers: %w", err) - } - return nil - }); err != nil { - return err - } - - // We should not generate deckhousereleases.yaml manifest for tag-based pulls - if pullParams.DeckhouseTag == "" { - if err = generateDeckhouseReleaseManifests(pullParams, tagsToMirror, imageLayouts, logger); err != nil { - logger.WarnLn(err.Error()) - } - } - - logger.Infof("Searching for Deckhouse built-in modules digests") - - var prevDigests = make(map[string]struct{}, 0) - for imageTag := range imageLayouts.InstallImages { - digests, err := images.ExtractImageDigestsFromDeckhouseInstaller(pullParams, imageTag, imageLayouts.Install, prevDigests, client) - if err != nil { - return fmt.Errorf("Extract images digests: %w", err) - } - - maps.Copy(imageLayouts.DeckhouseImages, digests) - } - logger.Infof("Found %d images", len(imageLayouts.DeckhouseImages)) - - if err = logger.Process("Pull Deckhouse images", func() error { - return layouts.PullDeckhouseImages(pullParams, imageLayouts, client) - }); err != nil { - return fmt.Errorf("Pull Deckhouse images: %w", err) - } - - err = logger.Process("Processing image indexes", func() error { - if pullParams.DeckhouseTag != "" { - // If we are pulling some build by tag, propagate release channel image of it to all channels if it exists. - releaseChannel, err := layouts.FindImageDescriptorByTag(imageLayouts.ReleaseChannel, pullParams.DeckhouseTag) - switch { - case errors.Is(err, layouts.ErrImageNotFound): - logger.WarnLn("Registry does not contain release channels, release channels images will not be added to bundle") - goto sortManifests - case err != nil: - return fmt.Errorf("Find release-%s channel descriptor: %w", pullParams.DeckhouseTag, err) - } - - for _, channel := range internal.GetAllDefaultReleaseChannels() { - if err = layouts.TagImage(imageLayouts.ReleaseChannel, releaseChannel.Digest, channel); err != nil { - return fmt.Errorf("tag release channel: %w", err) - } - } - } - - sortManifests: - for _, l := range imageLayouts.AsList() { - err = layouts.SortIndexManifests(l) - if err != nil { - return fmt.Errorf("Sorting index manifests of %s: %w", l, err) - } - } - return nil - }) - if err != nil { - return fmt.Errorf("Processing image indexes: %w", err) - } - - if err = logger.Process("Pack Deckhouse images into platform.tar", func() error { - var platform io.Writer = chunked.NewChunkedFileWriter( - pullParams.BundleChunkSize, - pullParams.BundleDir, - "platform.tar", - ) - if pullParams.BundleChunkSize == 0 { - platform, err = os.Create(filepath.Join(pullParams.BundleDir, "platform.tar")) - if err != nil { - return fmt.Errorf("Create platform.tar: %w", err) - } - } - - if err = bundle.Pack(context.Background(), tmpDir, platform); err != nil { - return fmt.Errorf("Pack platform.tar: %w", err) - } - - return nil - }); err != nil { - return err - } - - return nil -} - -func generateDeckhouseReleaseManifests( - pullParams *params.PullParams, - tagsToMirror []string, - imageLayouts *layouts.ImageLayouts, - logger params.Logger, -) error { - logger.Infof("Generating DeckhouseRelease manifests") - deckhouseReleasesManifestFile := filepath.Join(pullParams.BundleDir, "deckhousereleases.yaml") - if err := manifests.GenerateDeckhouseReleaseManifestsForVersions( - tagsToMirror, - deckhouseReleasesManifestFile, - imageLayouts.ReleaseChannel, - ); err != nil { - return fmt.Errorf("Generate DeckhouseRelease manifests: %w", err) - } - return nil -} diff --git a/internal/mirror/operations/pull_dkp_test.go b/internal/mirror/operations/pull_dkp_test.go deleted file mode 100644 index d31a08a1..00000000 --- a/internal/mirror/operations/pull_dkp_test.go +++ /dev/null @@ -1,302 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "os" - "path/filepath" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - mock "github.com/deckhouse/deckhouse-cli/pkg/mock" -) - -// setupTestPullParams creates test PullParams with a mock logger -func setupTestPullParams(t testing.TB) (*params.PullParams, *mockLogger) { - t.Helper() - - tempDir := t.TempDir() - logger := &mockLogger{} - - return ¶ms.PullParams{ - BaseParams: params.BaseParams{ - RegistryAuth: authn.Anonymous, - RegistryHost: "localhost:5000", - RegistryPath: "test-repo", - ModulesPathSuffix: "modules", - DeckhouseRegistryRepo: "localhost:5000/test-repo", - BundleDir: tempDir, - WorkingDir: tempDir, - Insecure: true, - SkipTLSVerification: true, - Logger: logger, - }, - BundleChunkSize: 0, - }, logger -} - -func TestPullDeckhousePlatform_MkdirError(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - - // Set WorkingDir to a file path to cause mkdir error - tempDir := t.TempDir() - workingDir := filepath.Join(tempDir, "not-a-dir") - require.NoError(t, os.WriteFile(workingDir, []byte("content"), 0644)) - pullParams.WorkingDir = workingDir - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Create OCI Image Layouts") -} - -func TestPullDeckhousePlatform_ResolveTagsError(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - // Set invalid registry to cause resolve error - pullParams.RegistryHost = "invalid::host" - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Resolve images tags to digests") -} - -func TestPullDeckhousePlatform_PullReleaseChannelsError(t *testing.T) { - // This test is skipped because resolve tags happens first - t.Skip("Resolve tags happens before pull operations, so this error path is not reachable") -} - -func TestPullDeckhousePlatform_PullInstallersError(t *testing.T) { - // This test is skipped because resolve tags happens first - t.Skip("Resolve tags happens before pull operations, so this error path is not reachable") -} - -func TestPullDeckhousePlatform_PullStandaloneInstallersError(t *testing.T) { - // This test is skipped because resolve tags happens first - t.Skip("Resolve tags happens before pull operations, so this error path is not reachable") -} - -func TestPullDeckhousePlatform_GenerateManifestsError(t *testing.T) { - // This test is skipped because resolve tags happens first - t.Skip("Resolve tags happens before manifest generation, so this error path is not reachable") -} - -func TestPullDeckhousePlatform_ExtractDigestsError(t *testing.T) { - // This test is skipped because resolve tags happens first - t.Skip("Resolve tags happens before digest extraction, so this error path is not reachable") -} - -func TestPullDeckhousePlatform_PullImagesError(t *testing.T) { - // This test is skipped because resolve tags happens first - t.Skip("Resolve tags happens before image pulling, so this error path is not reachable") -} - -func TestPullDeckhousePlatform_SortManifestsError(t *testing.T) { - // This test is skipped because resolve tags happens first - t.Skip("Resolve tags happens before manifest sorting, so this error path is not reachable") -} - -func TestPullDeckhousePlatform_PackError(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - // Set invalid bundle dir to cause pack error - pullParams.BundleDir = "/invalid/path" - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - require.Error(t, err) - // The error will be from resolve tags first, but pack error would occur later - // For now, just verify it fails - require.Error(t, err) -} - -func TestPullDeckhousePlatform_WithDeckhouseTag(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - pullParams.DeckhouseTag = "v1.0.0" - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - // Should succeed or fail based on registry availability - // The important thing is that it doesn't fail due to tag-specific logic - if err != nil { - require.NotContains(t, err.Error(), "Generate DeckhouseRelease manifests") - } -} - -func TestPullDeckhousePlatform_EmptyTagsToMirror(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - - tagsToMirror := []string{} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - // Should fail due to registry issues, but not due to empty tags - require.Error(t, err) -} - -func TestPullDeckhousePlatform_LoggerCalls(t *testing.T) { - pullParams, logger := setupTestPullParams(t) - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - _ = PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - - // Check that expected logging occurred - adjust expectations based on actual flow - hasCreateLayoutsLog := false - for _, log := range logger.logs { - if log == "INFO: Creating OCI Image Layouts" { - hasCreateLayoutsLog = true - break - } - } - require.True(t, hasCreateLayoutsLog, "Should log creating OCI image layouts") -} - -func TestPullDeckhousePlatform_WorkingDirectoryCleanup(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - // Track if working directory is used - platformDir := filepath.Join(pullParams.WorkingDir, "platform") - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - - // The platform directory should be created and then cleaned up during execution - // Since the function fails early, check that it was attempted - _, statErr := os.Stat(platformDir) - // It might exist or not depending on when the failure occurred - _ = statErr // We don't assert here since failure timing varies - - // We expect an error due to registry issues - require.Error(t, err) -} - -func TestPullDeckhousePlatform_RegistryAuth(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - pullParams.RegistryAuth = authn.FromConfig(authn.AuthConfig{ - Username: "testuser", - Password: "testpass", - }) - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - // Should fail due to registry, but auth should be passed through - require.Error(t, err) -} - -func TestPullDeckhousePlatform_InsecureAndTLSSkip(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - pullParams.Insecure = true - pullParams.SkipTLSVerification = true - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - // Should attempt the operation with insecure settings - require.Error(t, err) // Will fail due to no registry, but should not fail due to TLS -} - -func TestPullDeckhousePlatform_BundleChunkSize(t *testing.T) { - pullParams, _ := setupTestPullParams(t) - pullParams.BundleChunkSize = 1024 * 1024 // 1MB chunks - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - // Should use chunked writer when BundleChunkSize > 0 - require.Error(t, err) // Will fail due to registry, but chunking should be attempted -} - -// Benchmark tests -func BenchmarkPullDeckhousePlatform(b *testing.B) { - pullParams, _ := setupTestPullParams(b) - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(b) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - } -} - -// Test coverage helpers - these functions help ensure we hit all code paths -func TestPullDeckhousePlatform_CodeCoverage_ProcessBlocks(t *testing.T) { - // Test that Process blocks are called correctly - this will only work if we get past resolve tags - pullParams, logger := setupTestPullParams(t) - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - _ = PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - - // Since the function fails at resolve tags, Process blocks won't be reached - // This test verifies the Process logging would work if we got that far - // For now, just check that some logging occurred - require.True(t, len(logger.logs) > 0, "Should have some logging") -} - -func TestPullDeckhousePlatform_CodeCoverage_TagPropagation(t *testing.T) { - // Test tag propagation logic for deckhouse tag pulls - pullParams, _ := setupTestPullParams(t) - pullParams.DeckhouseTag = "v1.0.0" - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - // Should succeed or fail, but tag propagation logic should be exercised - require.NotNil(t, err) // Will fail due to registry, but logic should run -} - -func TestPullDeckhousePlatform_CodeCoverage_ManifestGeneration(t *testing.T) { - // Test manifest generation logic - pullParams, _ := setupTestPullParams(t) - // Don't set DeckhouseTag so manifest generation is attempted - - tagsToMirror := []string{"v1.0.0"} - client := mock.NewRegistryClientMock(t) - - err := PullDeckhousePlatform(pullParams, []string{}, tagsToMirror, client) - // Should succeed or fail, but manifest generation should be attempted - require.NotNil(t, err) // Will fail due to registry, but manifest generation should be attempted -} - -func TestGenerateDeckhouseReleaseManifests_Success(t *testing.T) { - t.Skip("Skipping due to complexity of setting up proper image layout with release data") -} - -func TestGenerateDeckhouseReleaseManifests_InvalidBundleDir(t *testing.T) { - t.Skip("Skipping due to complexity of setting up proper image layout with release data") -} diff --git a/internal/mirror/operations/pull_modules.go b/internal/mirror/operations/pull_modules.go deleted file mode 100644 index 4eb9b02e..00000000 --- a/internal/mirror/operations/pull_modules.go +++ /dev/null @@ -1,169 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - - "github.com/google/go-containerregistry/pkg/v1/layout" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/internal" - "github.com/deckhouse/deckhouse-cli/internal/mirror/chunked" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/modules" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -func PullModules(pullParams *params.PullParams, filter *modules.Filter, client registry.Client) error { - var err error - var modulesData []modules.Module - logger := pullParams.Logger - imageLayouts := layouts.NewImageLayouts() - tmpDir := filepath.Join(pullParams.WorkingDir, "modules") - - logger.InfoLn("Fetching Deckhouse modules list") - modulesData, err = modules.ForRepo( - path.Join(pullParams.DeckhouseRegistryRepo, pullParams.ModulesPathSuffix), - pullParams.RegistryAuth, - pullParams.Insecure, pullParams.SkipTLSVerification, - client, - ) - if err != nil { - return fmt.Errorf("Find modules: %w", err) - } - - if len(modulesData) == 0 { - logger.WarnLn("Modules were not found, check your source repository address and modules path suffix") - return nil - } - printModulesList(logger, modulesData) - - logger.InfoLn("Creating OCI Layouts") - for _, module := range modulesData { - if !filter.Match(&module) { - continue - } - - moduleLayout, err := layouts.CreateEmptyImageLayout(filepath.Join(tmpDir, module.Name)) - if err != nil { - return fmt.Errorf("create OCI layout: %w", err) - } - releasesLayout, err := layouts.CreateEmptyImageLayout(filepath.Join(tmpDir, module.Name, "release")) - if err != nil { - return fmt.Errorf("create OCI layout: %w", err) - } - extraLayout, err := layouts.CreateEmptyImageLayout(filepath.Join(tmpDir, module.Name, "extra")) - if err != nil { - return fmt.Errorf("create OCI layout: %w", err) - } - - imageLayouts.Modules[module.Name] = layouts.ModuleImageLayout{ - ModuleLayout: moduleLayout, - ReleasesLayout: releasesLayout, - ExtraLayout: extraLayout, - ModuleImages: make(map[string]struct{}), - ReleaseImages: make(map[string]struct{}), - ExtraNamedLayouts: make(map[string]layout.Path), - ExtraNamedImages: make(map[string]map[string]struct{}), - } - } - - logger.InfoLn("Searching for Deckhouse external modules images") - if err = layouts.FindDeckhouseModulesImages(pullParams, imageLayouts, modulesData, filter, client); err != nil { - return fmt.Errorf("Find modules images: %w", err) - } - - if err = logger.Process("Pull images", func() error { - return layouts.PullModules(pullParams, imageLayouts, client) - }); err != nil { - return err - } - - logger.InfoLn("Processing image indexes") - for _, l := range imageLayouts.AsList() { - err = layouts.SortIndexManifests(l) - if err != nil { - return fmt.Errorf("Sorting index manifests of %s: %w", l, err) - } - } - - for name, layout := range imageLayouts.Modules { - // Skip channel aliases for --only-extra-images mode - if !pullParams.OnlyExtraImages { - if err := ApplyChannelAliasesIfNeeded(name, layout, filter); err != nil { - return fmt.Errorf("Apply channel aliases for module %s: %w", name, err) - } - } - - pkgName := "module-" + name + ".tar" - logger.Infof("Packing %s", pkgName) - - var pkg io.Writer = chunked.NewChunkedFileWriter(pullParams.BundleChunkSize, pullParams.BundleDir, pkgName) - if pullParams.BundleChunkSize == 0 { - pkg, err = os.Create(filepath.Join(pullParams.BundleDir, pkgName)) - if err != nil { - return fmt.Errorf("Create %s: %w", pkgName, err) - } - } - - if err = bundle.Pack(context.Background(), string(layout.ModuleLayout), pkg); err != nil { - return fmt.Errorf("Pack module %s: %w", pkgName, err) - } - } - - return nil -} - -func ApplyChannelAliasesIfNeeded(name string, layout layouts.ModuleImageLayout, filter *modules.Filter) error { - c, ok := filter.GetConstraint(name) - if ok && c.IsExact() { - ex := c.(*modules.ExactTagConstraint) - - desc, err := layouts.FindImageDescriptorByTag(layout.ReleasesLayout, ex.Tag()) - if err != nil { - return err - } - - if ex.HasChannelAlias() { - if err := layouts.TagImage(layout.ReleasesLayout, desc.Digest, ex.Channel()); err != nil { - return err - } - } else { - for _, channel := range append(internal.GetAllDefaultReleaseChannels(), internal.LTSChannel) { - if err := layouts.TagImage(layout.ReleasesLayout, desc.Digest, channel); err != nil { - return err - } - } - } - } - return nil -} - -func printModulesList(logger params.Logger, modulesData []modules.Module) { - logger.Infof("Repo contains %d modules:", len(modulesData)) - for i, module := range modulesData { - logger.Infof("%d:\t%s", i+1, module.Name) - } -} diff --git a/internal/mirror/operations/pull_modules_test.go b/internal/mirror/operations/pull_modules_test.go deleted file mode 100644 index f0257556..00000000 --- a/internal/mirror/operations/pull_modules_test.go +++ /dev/null @@ -1,302 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/modules" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - mock "github.com/deckhouse/deckhouse-cli/pkg/mock" -) - -// setupTestPullParamsForModules creates test PullParams with a mock logger -func setupTestPullParamsForModules(t testing.TB) (*params.PullParams, *mockLogger) { - t.Helper() - - tempDir := t.TempDir() - logger := &mockLogger{} - - return ¶ms.PullParams{ - BaseParams: params.BaseParams{ - RegistryAuth: authn.Anonymous, - RegistryHost: "localhost:5000", - RegistryPath: "test-repo", - ModulesPathSuffix: "modules", - DeckhouseRegistryRepo: "localhost:5000/test-repo", - BundleDir: tempDir, - WorkingDir: tempDir, - Insecure: true, - SkipTLSVerification: true, - Logger: logger, - }, - BundleChunkSize: 0, - }, logger -} - -func TestPullModules_FindModulesError(t *testing.T) { - pullParams, _ := setupTestPullParamsForModules(t) - // Set invalid registry to cause find modules error - pullParams.RegistryHost = "invalid::host" - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return(nil, fmt.Errorf("invalid registry")) - client.WithSegmentMock.Optional().Return(client) - err = PullModules(pullParams, filter, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Find modules") -} - -func TestPullModules_NoModulesFound(t *testing.T) { - pullParams, _ := setupTestPullParamsForModules(t) - // Set registry that would return no modules - pullParams.RegistryHost = "empty-registry" - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - err = PullModules(pullParams, filter, client) - require.NoError(t, err) // Succeeds with no modules -} - -func TestPullModules_CreateLayoutError(t *testing.T) { - // This test is skipped because find modules happens first - t.Skip("Find modules happens before layout creation, so this error path is not reachable") -} - -func TestPullModules_FindImagesError(t *testing.T) { - // This test is skipped because find modules happens first - t.Skip("Find modules happens before find images, so this error path is not reachable") -} - -func TestPullModules_PullImagesError(t *testing.T) { - // This test is skipped because find modules happens first - t.Skip("Find modules happens before pull images, so this error path is not reachable") -} - -func TestPullModules_SortManifestsError(t *testing.T) { - // This test is skipped because find modules happens first - t.Skip("Find modules happens before sort manifests, so this error path is not reachable") -} - -func TestPullModules_PackError(t *testing.T) { - // This test is skipped because find modules happens first - t.Skip("Find modules happens before packing, so this error path is not reachable") -} - -func TestPullModules_OnlyExtraImages(t *testing.T) { - pullParams, _ := setupTestPullParamsForModules(t) - pullParams.OnlyExtraImages = true - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - err = PullModules(pullParams, filter, client) - require.NoError(t, err) // Succeeds with no modules -} - -func TestPullModules_BundleChunkSize(t *testing.T) { - pullParams, _ := setupTestPullParamsForModules(t) - pullParams.BundleChunkSize = 1024 * 1024 // 1MB chunks - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - err = PullModules(pullParams, filter, client) - require.NoError(t, err) // Succeeds with no modules -} - -func TestPullModules_LoggerCalls(t *testing.T) { - pullParams, logger := setupTestPullParamsForModules(t) - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - _ = PullModules(pullParams, filter, client) - - // Check that expected logging occurred - only the first log since it fails early - hasFetchLog := false - - for _, log := range logger.logs { - if log == "INFO: Fetching Deckhouse modules list" { - hasFetchLog = true - } - } - - require.True(t, hasFetchLog, "Should log fetching modules list") -} - -func TestPullModules_WorkingDirectoryCleanup(t *testing.T) { - pullParams, _ := setupTestPullParamsForModules(t) - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - // Track if working directory is used - modulesDir := filepath.Join(pullParams.WorkingDir, "modules") - - err = PullModules(pullParams, filter, client) - - // The modules directory should be created during execution - // Since the function fails early, check that it was attempted - _, statErr := os.Stat(modulesDir) - // It might exist or not depending on when the failure occurred - _ = statErr // We don't assert here since failure timing varies - - require.NoError(t, err) // Succeeds with no modules -} - -func TestPullModules_RegistryAuth(t *testing.T) { - pullParams, _ := setupTestPullParamsForModules(t) - pullParams.RegistryAuth = authn.FromConfig(authn.AuthConfig{ - Username: "testuser", - Password: "testpass", - }) - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - err = PullModules(pullParams, filter, client) - require.NoError(t, err) // Succeeds with no modules -} - -func TestPullModules_InsecureAndTLSSkip(t *testing.T) { - pullParams, _ := setupTestPullParamsForModules(t) - pullParams.Insecure = true - pullParams.SkipTLSVerification = true - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - err = PullModules(pullParams, filter, client) - require.NoError(t, err) // Succeeds with no modules -} - -// Benchmark tests -func BenchmarkPullModules(b *testing.B) { - pullParams, _ := setupTestPullParamsForModules(b) - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(b, err) - - client := mock.NewRegistryClientMock(b) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = PullModules(pullParams, filter, client) - } -} - -// Test coverage helpers - these functions help ensure we hit all code paths -func TestPullModules_CodeCoverage_ProcessBlocks(t *testing.T) { - // Test that Process blocks are called correctly - pullParams, logger := setupTestPullParamsForModules(t) - - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - client := mock.NewRegistryClientMock(t) - client.ListTagsMock.Return([]string{}, nil) - client.WithSegmentMock.Optional().Return(client) - _ = PullModules(pullParams, filter, client) - - // Since the function fails at find modules, Process blocks won't be reached - // This test verifies the Process logging would work if we got that far - // For now, just check that some logging occurred - require.True(t, len(logger.logs) > 0, "Should have some logging") -} - -func TestApplyChannelAliasesIfNeeded_NoConstraint(t *testing.T) { - filter, err := modules.NewFilter([]string{}, modules.FilterTypeWhitelist) - require.NoError(t, err) - - // Create a mock layout - this is complex, so let's skip for now - layout := layouts.ModuleImageLayout{} - - err = ApplyChannelAliasesIfNeeded("test-module", layout, filter) - // Should succeed with no constraint - require.NoError(t, err) -} - -func TestApplyChannelAliasesIfNeeded_WithConstraint(t *testing.T) { - // This test would require setting up proper image layouts with tags - // For now, skip due to complexity - t.Skip("Skipping due to complexity of setting up proper image layout with tags") -} - -func TestPrintModulesList(t *testing.T) { - logger := &mockLogger{} - - modulesData := []modules.Module{ - {Name: "module1"}, - {Name: "module2"}, - {Name: "module3"}, - } - - printModulesList(logger, modulesData) - - // Check that the list was logged - expectedLogs := []string{ - "INFO: Repo contains 3 modules:", - "INFO: 1:\tmodule1", - "INFO: 2:\tmodule2", - "INFO: 3:\tmodule3", - } - - for _, expectedLog := range expectedLogs { - found := false - for _, log := range logger.logs { - if log == expectedLog { - found = true - break - } - } - require.True(t, found, "Should log: %s", expectedLog) - } -} diff --git a/internal/mirror/operations/pull_security.go b/internal/mirror/operations/pull_security.go deleted file mode 100644 index 17a5786a..00000000 --- a/internal/mirror/operations/pull_security.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/internal/mirror/chunked" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -func PullSecurityDatabases(pullParams *params.PullParams, client registry.Client) error { - var err error - logger := pullParams.Logger - tmpDir := filepath.Join(pullParams.WorkingDir, "security") - - imageLayouts := &layouts.ImageLayouts{} - imageLayouts.TrivyDB, err = layouts.CreateEmptyImageLayout(filepath.Join(tmpDir, "trivy-db")) - if err != nil { - return fmt.Errorf("setup trivy db layout: %w", err) - } - imageLayouts.TrivyBDU, err = layouts.CreateEmptyImageLayout(filepath.Join(tmpDir, "trivy-bdu")) - if err != nil { - return fmt.Errorf("setup bdu layout: %w", err) - } - imageLayouts.TrivyJavaDB, err = layouts.CreateEmptyImageLayout(filepath.Join(tmpDir, "trivy-java-db")) - if err != nil { - return fmt.Errorf("setup java db layout: %w", err) - } - imageLayouts.TrivyChecks, err = layouts.CreateEmptyImageLayout(filepath.Join(tmpDir, "trivy-checks")) - if err != nil { - return fmt.Errorf("setup trivy checks layout: %w", err) - } - - if err := layouts.PullTrivyVulnerabilityDatabasesImages(pullParams, imageLayouts, client); err != nil { - return fmt.Errorf("Pull Secutity Databases: %w", err) - } - - logger.InfoLn("Processing image indexes") - - for _, l := range imageLayouts.AsList() { - err = layouts.SortIndexManifests(l) - if err != nil { - return fmt.Errorf("Sorting index manifests of %s: %w", l, err) - } - } - - if err = logger.Process("Pack security databases to security.tar", func() error { - var securityDB io.Writer = chunked.NewChunkedFileWriter( - pullParams.BundleChunkSize, - pullParams.BundleDir, - "security.tar", - ) - - if pullParams.BundleChunkSize == 0 { - securityDB, err = os.Create(filepath.Join(pullParams.BundleDir, "security.tar")) - if err != nil { - return fmt.Errorf("create security.tar: %w", err) - } - } - - if err = bundle.Pack(context.Background(), tmpDir, securityDB); err != nil { - return fmt.Errorf("pack security.tar: %w", err) - } - - return nil - }); err != nil { - return err - } - - return nil -} diff --git a/internal/mirror/operations/push_dkp.go b/internal/mirror/operations/push_dkp.go deleted file mode 100644 index 85645bbb..00000000 --- a/internal/mirror/operations/push_dkp.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - - "github.com/google/go-containerregistry/pkg/v1/layout" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -func PushDeckhousePlatform(pushParams *params.PushParams, pkg io.Reader, client registry.Client) error { - packageDir := filepath.Join(pushParams.WorkingDir, "platform") - if err := os.MkdirAll(packageDir, 0755); err != nil { - return fmt.Errorf("mkdir: %w", err) - } - defer os.RemoveAll(packageDir) - - pushParams.Logger.InfoLn("Unpacking platform package") - if err := bundle.Unpack(context.Background(), pkg, packageDir); err != nil { - return fmt.Errorf("Unpack package: %w", err) - } - - pushParams.Logger.InfoLn("Validating platform package") - if err := bundle.ValidateUnpackedPackage(bundle.MandatoryLayoutsForPlatform(packageDir)); err != nil { - return fmt.Errorf("Invalid platform package: %w", err) - } - - // These are layouts within platform.tar - layoutsToPush := []string{ - "", // Root layout - "install", // Installer images - "install-standalone", // Standalone installer bundles - "release-channel", // Release channels - } - - for _, repo := range layoutsToPush { - repoRef := path.Join(pushParams.RegistryHost, pushParams.RegistryPath, repo) - pushParams.Logger.InfoLn("Pushing", repoRef) - if err := layouts.PushLayoutToRepoContext( - context.Background(), - client.WithSegment(repo), - layout.Path(filepath.Join(packageDir, repo)), - repoRef, - pushParams.RegistryAuth, - pushParams.Logger, - pushParams.Parallelism, - pushParams.Insecure, - pushParams.SkipTLSVerification, - ); err != nil { - return fmt.Errorf("Push platform package: %w", err) - } - } - - return nil -} diff --git a/internal/mirror/operations/push_dkp_test.go b/internal/mirror/operations/push_dkp_test.go deleted file mode 100644 index 4544d148..00000000 --- a/internal/mirror/operations/push_dkp_test.go +++ /dev/null @@ -1,336 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "archive/tar" - "bytes" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -func createValidPlatformPackage(t testing.TB) io.Reader { - t.Helper() - - var buf bytes.Buffer - tarWriter := tar.NewWriter(&buf) - - // Create index.json for each required layout - indexContent := `{ - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.index.v1+json", - "manifests": [] - }` - - // Add oci-layout file - layoutContent := `{"imageLayoutVersion": "1.0.0"}` - - // Create directories and files for each layout that gets pushed - layouts := []string{"", "install", "install-standalone", "release-channel"} - - for _, layoutName := range layouts { - var prefix string - if layoutName != "" { - prefix = layoutName + "/" - } - - // Add index.json - hdr := &tar.Header{ - Name: prefix + "index.json", - Mode: 0644, - Size: int64(len(indexContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err := tarWriter.Write([]byte(indexContent)) - require.NoError(t, err) - - // Add oci-layout - hdr = &tar.Header{ - Name: prefix + "oci-layout", - Mode: 0644, - Size: int64(len(layoutContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err = tarWriter.Write([]byte(layoutContent)) - require.NoError(t, err) - } - - require.NoError(t, tarWriter.Close()) - return &buf -} - -// createInvalidPlatformPackage creates a tar archive missing required layouts -func createInvalidPlatformPackage(t testing.TB) io.Reader { - t.Helper() - - var buf bytes.Buffer - tarWriter := tar.NewWriter(&buf) - - // Add some random file but not the required layouts - content := "some content" - hdr := &tar.Header{ - Name: "some-file.txt", - Mode: 0644, - Size: int64(len(content)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err := tarWriter.Write([]byte(content)) - require.NoError(t, err) - - require.NoError(t, tarWriter.Close()) - return &buf -} - -func TestPushDeckhousePlatform_MkdirError(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set WorkingDir to a file path to cause mkdir error - tempDir := t.TempDir() - workingDir := filepath.Join(tempDir, "not-a-dir") - require.NoError(t, os.WriteFile(workingDir, []byte("content"), 0644)) - pushParams.WorkingDir = workingDir - - pkg := createValidPlatformPackage(t) - - err := PushDeckhousePlatform(pushParams, pkg, client) - require.Error(t, err) - require.Contains(t, err.Error(), "mkdir") -} - -func TestPushDeckhousePlatform_UnpackError(t *testing.T) { - t.Skip("Skipping due to bug in bundle.Unpack - it doesn't handle tar reader errors properly") - - pushParams, _, client := setupTestPushParams(t) - - // Create a reader that returns an error - errReader := &errorReader{err: errors.New("read error")} - err := PushDeckhousePlatform(pushParams, errReader, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Unpack package") -} - -func TestPushDeckhousePlatform_ValidationError(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pkg := createInvalidPlatformPackage(t) - - err := PushDeckhousePlatform(pushParams, pkg, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Invalid platform package") -} - -func TestPushDeckhousePlatform_NilReader(t *testing.T) { - t.Skip("Skipping due to nil pointer issues with tar reader") - - pushParams, _, client := setupTestPushParams(t) - - err := PushDeckhousePlatform(pushParams, nil, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Unpack package") -} - -func TestPushDeckhousePlatform_LayoutPaths(t *testing.T) { - pushParams, logger, client := setupTestPushParams(t) - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushDeckhousePlatform(pushParams, pkg, client) - - // Even if it fails due to registry, we can check that the correct paths were logged - if err == nil || err != nil { - // Check that the expected repo paths were logged - expectedRepos := []string{ - "localhost:5000/test-repo/", - "localhost:5000/test-repo/install", - "localhost:5000/test-repo/install-standalone", - "localhost:5000/test-repo/release-channel", - } - - for _, repo := range expectedRepos { - found := false - for _, log := range logger.logs { - if strings.Contains(log, "Pushing"+repo) { - found = true - break - } - } - require.True(t, found, "Should log pushing repo: %s", repo) - } - } -} - -func TestPushDeckhousePlatform_LoggerCalls(t *testing.T) { - pushParams, logger, client := setupTestPushParams(t) - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - _ = PushDeckhousePlatform(pushParams, pkg, client) - - // Check that unpacking and validation logging occurred - hasUnpackLog := false - hasValidationLog := false - for _, log := range logger.logs { - if strings.Contains(log, "Unpacking platform package") { - hasUnpackLog = true - } - if strings.Contains(log, "Validating platform package") { - hasValidationLog = true - } - } - require.True(t, hasUnpackLog, "Should log unpacking platform package") - require.True(t, hasValidationLog, "Should log validating platform package") -} - -func TestPushDeckhousePlatform_WorkingDirectoryCleanup(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - // Track if cleanup occurred by checking directory existence - packageDir := filepath.Join(pushParams.WorkingDir, "platform") - - err := PushDeckhousePlatform(pushParams, pkg, client) - - // Even after success, the directory should be cleaned up - _, statErr := os.Stat(packageDir) - require.True(t, os.IsNotExist(statErr), "Working directory should be cleaned up") - - // For empty layouts, the function should succeed - require.NoError(t, err) -} - -func TestPushDeckhousePlatform_RegistryAuth(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pushParams.RegistryAuth = authn.FromConfig(authn.AuthConfig{ - Username: "testuser", - Password: "testpass", - }) - - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushDeckhousePlatform(pushParams, pkg, client) - // For empty layouts, should succeed (auth is configured but not used) - require.NoError(t, err) -} - -func TestPushDeckhousePlatform_InsecureAndTLSSkip(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pushParams.Insecure = true - pushParams.SkipTLSVerification = true - - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushDeckhousePlatform(pushParams, pkg, client) - // For empty layouts, should succeed (insecure settings are configured but not used) - require.NoError(t, err) -} - -func TestPushDeckhousePlatform_ParallelismConfig(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pushParams.Parallelism = params.ParallelismConfig{ - Blobs: 2, - Images: 2, - } - - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushDeckhousePlatform(pushParams, pkg, client) - // For empty layouts, should succeed (parallelism is configured but not used) - require.NoError(t, err) -} - -// Benchmark tests -func BenchmarkPushDeckhousePlatform(b *testing.B) { - pushParams, _, client := setupTestPushParams(b) - pkg := createValidPlatformPackage(b) - - client.WithSegmentMock.Return(client) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = PushDeckhousePlatform(pushParams, pkg, client) - } -} - -// Test coverage helpers - these functions help ensure we hit all code paths -func TestPushDeckhousePlatform_CodeCoverage_LayoutsToPush(t *testing.T) { - // This test ensures we cover the layoutsToPush slice creation - pushParams, _, client := setupTestPushParams(t) - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - // The layoutsToPush slice should be created with correct paths - expectedLayouts := []string{"", "install", "install-standalone", "release-channel"} - - // We can't directly test the slice, but we can verify the function runs - // and check that the expected repos are constructed correctly - err := PushDeckhousePlatform(pushParams, pkg, client) - - // Verify that logs contain the expected repo constructions - for _, layoutName := range expectedLayouts { - var expectedRepo string - if layoutName == "" { - expectedRepo = "localhost:5000/test-repo/" - } else { - expectedRepo = fmt.Sprintf("localhost:5000/test-repo/%s", layoutName) - } - found := false - for _, log := range pushParams.Logger.(*mockLogger).logs { - if strings.Contains(log, "Pushing"+expectedRepo) { - found = true - break - } - } - require.True(t, found, "Should construct repo path for layout %s: %s", layoutName, expectedRepo) - } - - require.NoError(t, err) // Expected to succeed for empty layouts -} - -func TestPushDeckhousePlatform_CodeCoverage_AuthOptions(t *testing.T) { - // Test that auth options are constructed correctly - pushParams, _, client := setupTestPushParams(t) - pushParams.RegistryAuth = authn.Anonymous - pushParams.Insecure = true - pushParams.SkipTLSVerification = true - - pkg := createValidPlatformPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushDeckhousePlatform(pushParams, pkg, client) - require.NoError(t, err) // Will succeed for empty layouts -} diff --git a/internal/mirror/operations/push_modules.go b/internal/mirror/operations/push_modules.go deleted file mode 100644 index 33543f0d..00000000 --- a/internal/mirror/operations/push_modules.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - - "github.com/google/go-containerregistry/pkg/v1/layout" - "github.com/google/go-containerregistry/pkg/v1/random" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -func PushModule(pushParams *params.PushParams, moduleName string, pkg io.Reader, client registry.Client) error { - logger := pushParams.Logger - - packageDir := filepath.Join(pushParams.WorkingDir, "modules", moduleName) - if err := os.MkdirAll(packageDir, 0755); err != nil { - return fmt.Errorf("mkdir: %w", err) - } - defer os.RemoveAll(packageDir) - - if err := bundle.Unpack(context.Background(), pkg, packageDir); err != nil { - return fmt.Errorf("Unpack package: %w", err) - } - - if err := bundle.ValidateUnpackedPackage(bundle.MandatoryLayoutsForModule(packageDir)); err != nil { - return fmt.Errorf("Invalid module package: %w", err) - } - - // These are layouts within module-ABC.tar mapped to paths they belong to in the deckhouse registry. - // Registry paths are relative to root of deckhouse repo. - layoutsToPush := map[string]string{ - "": path.Join(pushParams.ModulesPathSuffix, moduleName), - "release": path.Join(pushParams.ModulesPathSuffix, moduleName, "release"), - "extra": path.Join(pushParams.ModulesPathSuffix, moduleName, "extra"), - } - - // automatically discover extra layouts in the extra/ directory - dirEntries, err := os.ReadDir(filepath.Join(packageDir, "extra")) - if err != nil { - if os.IsNotExist(err) { - logger.Debugf("No extra dir for module %s", moduleName) - } else { - logger.Warnf("Error reading extra dir for module %s: %v", moduleName, err) - } - } - - // add discovered extra layouts to the list of layouts to push - for _, de := range dirEntries { - logger.Debugf("Found extra layout %s for module %s", de.Name(), moduleName) - if de.IsDir() { - layoutsToPush[path.Join("extra", de.Name())] = path.Join(pushParams.ModulesPathSuffix, moduleName, "extra", de.Name()) - } - } - - for layoutPathSuffix, repo := range layoutsToPush { - repoRef := path.Join(pushParams.RegistryHost, pushParams.RegistryPath, repo) - pushParams.Logger.InfoLn("Pushing", repoRef) - if err := layouts.PushLayoutToRepoContext( - context.Background(), - client.WithSegment(moduleName).WithSegment(layoutPathSuffix), - layout.Path(filepath.Join(packageDir, layoutPathSuffix)), - repoRef, - pushParams.RegistryAuth, - pushParams.Logger, - pushParams.Parallelism, - pushParams.Insecure, - pushParams.SkipTLSVerification, - ); err != nil { - return fmt.Errorf("Push module package: %w", err) - } - } - - pushParams.Logger.Infof("Pushing module tag for %s", moduleName) - - img, err := random.Image(32, 1) - if err != nil { - return fmt.Errorf("random.Image: %w", err) - } - - if err = client.PushImage(context.Background(), moduleName, img); err != nil { - return fmt.Errorf("Write module index tag: %w", err) - } - - return nil -} diff --git a/internal/mirror/operations/push_modules_test.go b/internal/mirror/operations/push_modules_test.go deleted file mode 100644 index 86d5e6d4..00000000 --- a/internal/mirror/operations/push_modules_test.go +++ /dev/null @@ -1,524 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "archive/tar" - "bytes" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/mock" -) - -// mockLogger implements params.Logger for testing -type mockLogger struct { - logs []string -} - -func (m *mockLogger) Debugf(format string, a ...interface{}) { - m.logs = append(m.logs, fmt.Sprintf("DEBUG: "+format, a...)) -} - -func (m *mockLogger) DebugLn(a ...interface{}) { - m.logs = append(m.logs, fmt.Sprintf("DEBUG: %s", fmt.Sprint(a...))) -} - -func (m *mockLogger) Infof(format string, a ...interface{}) { - m.logs = append(m.logs, fmt.Sprintf("INFO: "+format, a...)) -} - -func (m *mockLogger) InfoLn(a ...interface{}) { - m.logs = append(m.logs, fmt.Sprintf("INFO: %s", fmt.Sprint(a...))) -} - -func (m *mockLogger) Warnf(format string, a ...interface{}) { - m.logs = append(m.logs, fmt.Sprintf("WARN: "+format, a...)) -} - -func (m *mockLogger) WarnLn(a ...interface{}) { - m.logs = append(m.logs, fmt.Sprintf("WARN: %s", fmt.Sprint(a...))) -} - -func (m *mockLogger) Process(topic string, run func() error) error { - m.logs = append(m.logs, fmt.Sprintf("PROCESS: %s", topic)) - return run() -} - -// setupTestPushParams creates test PushParams with a mock logger -func setupTestPushParams(t testing.TB) (*params.PushParams, *mockLogger, *mock.RegistryClientMock) { - t.Helper() - - tempDir := t.TempDir() - logger := &mockLogger{} - - pushParams := ¶ms.PushParams{ - BaseParams: params.BaseParams{ - RegistryAuth: authn.Anonymous, - RegistryHost: "localhost:5000", - RegistryPath: "test-repo", - ModulesPathSuffix: "modules", - BundleDir: tempDir, - WorkingDir: tempDir, - Insecure: true, - SkipTLSVerification: true, - Logger: logger, - }, - Parallelism: params.ParallelismConfig{ - Blobs: 1, - Images: 1, - }, - } - - // Create registry client mock for tests - client := mock.NewRegistryClientMock(t) - - // Note: We set expectations for WithSegment and PushImage in individual tests - // because some tests expect early errors before these methods are called. - // Tests that need these methods should set expectations explicitly. - - return pushParams, logger, client -} - -// createValidModulePackage creates a tar archive that mimics a valid module package -func createValidModulePackage(t testing.TB) io.Reader { - t.Helper() - - var buf bytes.Buffer - tarWriter := tar.NewWriter(&buf) - - // Create index.json for the root layout - indexContent := `{ - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.index.v1+json", - "manifests": [] - }` - - // Add index.json - hdr := &tar.Header{ - Name: "index.json", - Mode: 0644, - Size: int64(len(indexContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err := tarWriter.Write([]byte(indexContent)) - require.NoError(t, err) - - // Add oci-layout file - layoutContent := `{"imageLayoutVersion": "1.0.0"}` - hdr = &tar.Header{ - Name: "oci-layout", - Mode: 0644, - Size: int64(len(layoutContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err = tarWriter.Write([]byte(layoutContent)) - require.NoError(t, err) - - // Add release/index.json - hdr = &tar.Header{ - Name: "release/index.json", - Mode: 0644, - Size: int64(len(indexContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err = tarWriter.Write([]byte(indexContent)) - require.NoError(t, err) - - // Add release/oci-layout - hdr = &tar.Header{ - Name: "release/oci-layout", - Mode: 0644, - Size: int64(len(layoutContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err = tarWriter.Write([]byte(layoutContent)) - require.NoError(t, err) - - // Add extra/index.json - hdr = &tar.Header{ - Name: "extra/index.json", - Mode: 0644, - Size: int64(len(indexContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err = tarWriter.Write([]byte(indexContent)) - require.NoError(t, err) - - // Add extra/oci-layout - hdr = &tar.Header{ - Name: "extra/oci-layout", - Mode: 0644, - Size: int64(len(layoutContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err = tarWriter.Write([]byte(layoutContent)) - require.NoError(t, err) - - require.NoError(t, tarWriter.Close()) - return &buf -} - -// createInvalidModulePackage creates a tar archive without required layouts -func createInvalidModulePackage(t testing.TB) io.Reader { - t.Helper() - - var buf bytes.Buffer - tarWriter := tar.NewWriter(&buf) - - // Add some random file but not the required layouts - content := "some content" - hdr := &tar.Header{ - Name: "some-file.txt", - Mode: 0644, - Size: int64(len(content)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err := tarWriter.Write([]byte(content)) - require.NoError(t, err) - - require.NoError(t, tarWriter.Close()) - return &buf -} - -func TestPushModule_MkdirError(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set WorkingDir to a file path to cause mkdir error - tempDir := t.TempDir() - workingDir := filepath.Join(tempDir, "not-a-dir") - require.NoError(t, os.WriteFile(workingDir, []byte("content"), 0644)) - pushParams.WorkingDir = workingDir - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - require.Error(t, err) - require.Contains(t, err.Error(), "mkdir") -} - -func TestPushModule_UnpackError(t *testing.T) { - t.Skip("Skipping due to bug in bundle.Unpack - it doesn't handle tar reader errors properly") - - pushParams, _, client := setupTestPushParams(t) - moduleName := "test-module" - - // Create a reader that returns an error - errReader := &errorReader{err: errors.New("read error")} - err := PushModule(pushParams, moduleName, errReader, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Unpack package") -} - -func TestPushModule_ValidationError(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - moduleName := "test-module" - pkg := createInvalidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Invalid module package") -} - -func TestPushModule_NilReader(t *testing.T) { - t.Skip("Skipping due to nil pointer issues with tar reader") - - pushParams, _, client := setupTestPushParams(t) - moduleName := "test-module" - - err := PushModule(pushParams, moduleName, nil, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Unpack package") -} - -func TestPushModule_EmptyModuleName(t *testing.T) { - t.Skip("Skipping empty module name test") - - pushParams, _, client := setupTestPushParams(t) - moduleName := "" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - require.Error(t, err) - // Should still attempt to create directory and unpack - require.Contains(t, err.Error(), "Unpack package") -} - -func TestPushModule_LayoutPaths(t *testing.T) { - pushParams, logger, client := setupTestPushParams(t) - - // Set up mock expectations for this test - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(nil) - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - - // Even if it fails due to registry, we can check that the correct paths were logged - if err != nil { - // Check that the expected repo paths were logged - expectedRepos := []string{ - "localhost:5000/test-repo/modules/test-module", - "localhost:5000/test-repo/modules/test-module/release", - "localhost:5000/test-repo/modules/test-module/extra", - } - - for _, repo := range expectedRepos { - found := false - for _, log := range logger.logs { - if strings.Contains(log, "Pushing"+repo) { - found = true - break - } - } - require.True(t, found, "Should log pushing repo: %s", repo) - } - } -} - -func TestPushModule_LoggerCalls(t *testing.T) { - pushParams, logger, client := setupTestPushParams(t) - - // Set up mock expectations for this test - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(nil) - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - _ = PushModule(pushParams, moduleName, pkg, client) - - // Check that module tag logging occurred - hasModuleTagLog := false - for _, log := range logger.logs { - if strings.Contains(log, "Pushing module tag for test-module") { - hasModuleTagLog = true - break - } - } - require.True(t, hasModuleTagLog, "Should log pushing module tag") -} - -func TestPushModule_WorkingDirectoryCleanup(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("registry connection failed")) - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - // Track if cleanup occurred by checking directory existence - packageDir := filepath.Join(pushParams.WorkingDir, "modules", moduleName) - - err := PushModule(pushParams, moduleName, pkg, client) - - // Even after error, the directory should be cleaned up - _, statErr := os.Stat(packageDir) - require.True(t, os.IsNotExist(statErr), "Working directory should be cleaned up") - - // We expect an error due to registry issues, but cleanup should still happen - require.Error(t, err) -} - -func TestPushModule_RegistryAuth(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("registry connection failed")) - - pushParams.RegistryAuth = authn.FromConfig(authn.AuthConfig{ - Username: "testuser", - Password: "testpass", - }) - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - // Should fail due to registry, but auth should be passed through - require.Error(t, err) -} - -func TestPushModule_ParseReferenceError(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("invalid reference")) - - // Set invalid registry host to cause parse error - pushParams.RegistryHost = "invalid::host" - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Write module index tag") -} - -func TestPushModule_InsecureAndTLSSkip(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("registry connection failed")) - - pushParams.Insecure = true - pushParams.SkipTLSVerification = true - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - // Should attempt the operation with insecure settings - require.Error(t, err) // Will fail due to no registry, but should not fail due to TLS -} - -func TestPushModule_ParallelismConfig(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("registry connection failed")) - - pushParams.Parallelism = params.ParallelismConfig{ - Blobs: 2, - Images: 2, - } - - moduleName := "test-module" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - require.Error(t, err) // Will fail due to no registry, but parallelism should be passed through -} - -// errorReader implements io.Reader that always returns an error -type errorReader struct { - err error -} - -func (e *errorReader) Read(p []byte) (n int, err error) { - return 0, e.err -} - -// Benchmark tests -func BenchmarkPushModule(b *testing.B) { - pushParams, _, client := setupTestPushParams(b) - - // Set up mock expectations for benchmark - PushImage should succeed for performance testing - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(nil) - - moduleName := "bench-module" - pkg := createValidModulePackage(b) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = PushModule(pushParams, moduleName, pkg, client) - } -} - -// Test coverage helpers - these functions help ensure we hit all code paths -func TestPushModule_CodeCoverage_LayoutsToPush(t *testing.T) { - // This test ensures we cover the layoutsToPush map creation - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("registry connection failed")) - - moduleName := "coverage-test" - pkg := createValidModulePackage(t) - - // The layoutsToPush map should be created with correct paths - expectedPaths := map[string]string{ - "": "modules/coverage-test", - "release": "modules/coverage-test/release", - "extra": "modules/coverage-test/extra", - } - - // We can't directly test the map, but we can verify the function runs - // and check that the expected repos are constructed correctly - err := PushModule(pushParams, moduleName, pkg, client) - - // Verify that logs contain the expected repo constructions - for layoutPath, expectedSuffix := range expectedPaths { - expectedRepo := fmt.Sprintf("localhost:5000/test-repo/%s", expectedSuffix) - found := false - for _, log := range pushParams.Logger.(*mockLogger).logs { - if strings.Contains(log, "Pushing"+expectedRepo) { - found = true - break - } - } - require.True(t, found, "Should construct repo path for layout %s: %s", layoutPath, expectedRepo) - } - - require.Error(t, err) // Expected to fail due to registry -} - -func TestPushModule_CodeCoverage_RandomImage(t *testing.T) { - // Test that random.Image is called correctly - // This is hard to test directly, but we can ensure the code path is exercised - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("registry connection failed")) - - moduleName := "random-image-test" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - require.Error(t, err) // Will fail at Client.PushImage, but random.Image should be called -} - -func TestPushModule_CodeCoverage_AuthOptions(t *testing.T) { - // Test that auth options are constructed correctly - pushParams, _, client := setupTestPushParams(t) - - // Set up mock expectations for this test - expect PushImage to fail - client.WithSegmentMock.Return(client) - client.PushImageMock.Return(errors.New("registry connection failed")) - - pushParams.RegistryAuth = authn.Anonymous - pushParams.Insecure = true - pushParams.SkipTLSVerification = true - - moduleName := "auth-options-test" - pkg := createValidModulePackage(t) - - err := PushModule(pushParams, moduleName, pkg, client) - require.Error(t, err) // Will fail, but auth options should be constructed -} diff --git a/internal/mirror/operations/push_security.go b/internal/mirror/operations/push_security.go deleted file mode 100644 index 1c34b2fa..00000000 --- a/internal/mirror/operations/push_security.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - - "github.com/google/go-containerregistry/pkg/v1/layout" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/bundle" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/layouts" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -func PushSecurityDatabases(pushParams *params.PushParams, pkg io.Reader, client registry.Client) error { - packageDir := filepath.Join(pushParams.WorkingDir, "security") - if err := os.MkdirAll(packageDir, 0755); err != nil { - return fmt.Errorf("mkdir: %w", err) - } - defer os.RemoveAll(packageDir) - - if err := bundle.Unpack(context.Background(), pkg, packageDir); err != nil { - return fmt.Errorf("Unpack package: %w", err) - } - - if err := bundle.ValidateUnpackedPackage(bundle.MandatoryLayoutsForSecurityDatabase(packageDir)); err != nil { - return fmt.Errorf("Invalid security database package: %w", err) - } - - // These are layouts within security.tar mapped to paths they belong to in the deckhouse registry. - // Registry paths are relative to root of deckhouse repo. - layoutsToPush := map[string]string{ - "trivy-db": "security/trivy-db", - "trivy-java-db": "security/trivy-java-db", - "trivy-bdu": "security/trivy-bdu", - "trivy-checks": "security/trivy-checks", - } - - for layoutPathSuffix, repo := range layoutsToPush { - repoRef := path.Join(pushParams.RegistryHost, pushParams.RegistryPath, repo) - pushParams.Logger.InfoLn("Pushing", repoRef) - if err := layouts.PushLayoutToRepoContext( - context.Background(), - client.WithSegment("security").WithSegment(layoutPathSuffix), - layout.Path(filepath.Join(packageDir, layoutPathSuffix)), - repoRef, - pushParams.RegistryAuth, - pushParams.Logger, - pushParams.Parallelism, - pushParams.Insecure, - pushParams.SkipTLSVerification, - ); err != nil { - return fmt.Errorf("Push security package: %w", err) - } - } - - return nil -} diff --git a/internal/mirror/operations/push_security_test.go b/internal/mirror/operations/push_security_test.go deleted file mode 100644 index 1b19149c..00000000 --- a/internal/mirror/operations/push_security_test.go +++ /dev/null @@ -1,311 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/lice err := PushSecurityDatabases(pushParams, pkg) - require.Error(t, err) // Will fail, but auth options should be constructed -}-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operations - -import ( - "archive/tar" - "bytes" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" -) - -// createValidSecurityPackage creates a tar archive that mimics a valid security database package -func createValidSecurityPackage(t testing.TB) io.Reader { - t.Helper() - - var buf bytes.Buffer - tarWriter := tar.NewWriter(&buf) - - // Create index.json for each required layout - indexContent := `{ - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.index.v1+json", - "manifests": [] - }` - - // Add oci-layout file - layoutContent := `{"imageLayoutVersion": "1.0.0"}` - - // Create directories and files for each required layout - layouts := []string{"trivy-db", "trivy-bdu", "trivy-java-db", "trivy-checks"} - - for _, layoutName := range layouts { - // Add index.json - hdr := &tar.Header{ - Name: layoutName + "/index.json", - Mode: 0644, - Size: int64(len(indexContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err := tarWriter.Write([]byte(indexContent)) - require.NoError(t, err) - - // Add oci-layout - hdr = &tar.Header{ - Name: layoutName + "/oci-layout", - Mode: 0644, - Size: int64(len(layoutContent)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err = tarWriter.Write([]byte(layoutContent)) - require.NoError(t, err) - } - - require.NoError(t, tarWriter.Close()) - return &buf -} - -// createInvalidSecurityPackage creates a tar archive missing required layouts -func createInvalidSecurityPackage(t testing.TB) io.Reader { - t.Helper() - - var buf bytes.Buffer - tarWriter := tar.NewWriter(&buf) - - // Add some random file but not the required layouts - content := "some content" - hdr := &tar.Header{ - Name: "some-file.txt", - Mode: 0644, - Size: int64(len(content)), - } - require.NoError(t, tarWriter.WriteHeader(hdr)) - _, err := tarWriter.Write([]byte(content)) - require.NoError(t, err) - - require.NoError(t, tarWriter.Close()) - return &buf -} - -func TestPushSecurityDatabases_MkdirError(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - - // Set WorkingDir to a file path to cause mkdir error - tempDir := t.TempDir() - workingDir := filepath.Join(tempDir, "not-a-dir") - require.NoError(t, os.WriteFile(workingDir, []byte("content"), 0644)) - pushParams.WorkingDir = workingDir - - pkg := createValidSecurityPackage(t) - - err := PushSecurityDatabases(pushParams, pkg, client) - require.Error(t, err) - require.Contains(t, err.Error(), "mkdir") -} - -func TestPushSecurityDatabases_UnpackError(t *testing.T) { - t.Skip("Skipping due to bug in bundle.Unpack - it doesn't handle tar reader errors properly") - - pushParams, _, client := setupTestPushParams(t) - - // Create a reader that returns an error - errReader := &errorReader{err: errors.New("read error")} - err := PushSecurityDatabases(pushParams, errReader, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Unpack package") -} - -func TestPushSecurityDatabases_ValidationError(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pkg := createInvalidSecurityPackage(t) - - err := PushSecurityDatabases(pushParams, pkg, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Invalid security database package") -} - -func TestPushSecurityDatabases_NilReader(t *testing.T) { - t.Skip("Skipping due to nil pointer issues with tar reader") - - pushParams, _, client := setupTestPushParams(t) - - err := PushSecurityDatabases(pushParams, nil, client) - require.Error(t, err) - require.Contains(t, err.Error(), "Unpack package") -} - -func TestPushSecurityDatabases_LayoutPaths(t *testing.T) { - pushParams, logger, client := setupTestPushParams(t) - pkg := createValidSecurityPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushSecurityDatabases(pushParams, pkg, client) - - // Even if it fails due to registry, we can check that the correct paths were logged - if err != nil { - // Check that the expected repo paths were logged - expectedRepos := []string{ - "localhost:5000/test-repo/security/trivy-db", - "localhost:5000/test-repo/security/trivy-java-db", - "localhost:5000/test-repo/security/trivy-bdu", - "localhost:5000/test-repo/security/trivy-checks", - } - - for _, repo := range expectedRepos { - found := false - for _, log := range logger.logs { - if strings.Contains(log, "Pushing"+repo) { - found = true - break - } - } - require.True(t, found, "Should log pushing repo: %s", repo) - } - } -} - -func TestPushSecurityDatabases_WorkingDirectoryCleanup(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pkg := createValidSecurityPackage(t) - - client.WithSegmentMock.Return(client) - - // Track if cleanup occurred by checking directory existence - packageDir := filepath.Join(pushParams.WorkingDir, "security") - - err := PushSecurityDatabases(pushParams, pkg, client) - - // Even after success, the directory should be cleaned up - _, statErr := os.Stat(packageDir) - require.True(t, os.IsNotExist(statErr), "Working directory should be cleaned up") - - // For empty layouts, the function should succeed - require.NoError(t, err) -} - -func TestPushSecurityDatabases_RegistryAuth(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pushParams.RegistryAuth = authn.FromConfig(authn.AuthConfig{ - Username: "testuser", - Password: "testpass", - }) - - pkg := createValidSecurityPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushSecurityDatabases(pushParams, pkg, client) - // For empty layouts, should succeed (auth is configured but not used) - require.NoError(t, err) -} - -func TestPushSecurityDatabases_InsecureAndTLSSkip(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pushParams.Insecure = true - pushParams.SkipTLSVerification = true - - pkg := createValidSecurityPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushSecurityDatabases(pushParams, pkg, client) - // For empty layouts, should succeed (insecure settings are configured but not used) - require.NoError(t, err) -} - -func TestPushSecurityDatabases_ParallelismConfig(t *testing.T) { - pushParams, _, client := setupTestPushParams(t) - pushParams.Parallelism = params.ParallelismConfig{ - Blobs: 2, - Images: 2, - } - - pkg := createValidSecurityPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushSecurityDatabases(pushParams, pkg, client) - // For empty layouts, should succeed (parallelism is configured but not used) - require.NoError(t, err) -} - -// Benchmark tests -func BenchmarkPushSecurityDatabases(b *testing.B) { - pushParams, _, client := setupTestPushParams(b) - pkg := createValidSecurityPackage(b) - - client.WithSegmentMock.Return(client) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = PushSecurityDatabases(pushParams, pkg, client) - } -} - -// Test coverage helpers - these functions help ensure we hit all code paths -func TestPushSecurityDatabases_CodeCoverage_LayoutsToPush(t *testing.T) { - // This test ensures we cover the layoutsToPush map creation - pushParams, _, client := setupTestPushParams(t) - pkg := createValidSecurityPackage(t) - - client.WithSegmentMock.Return(client) - - // The layoutsToPush map should be created with correct paths - expectedPaths := map[string]string{ - "trivy-db": "security/trivy-db", - "trivy-java-db": "security/trivy-java-db", - "trivy-bdu": "security/trivy-bdu", - "trivy-checks": "security/trivy-checks", - } - - // We can't directly test the map, but we can verify the function runs - // and check that the expected repos are constructed correctly - err := PushSecurityDatabases(pushParams, pkg, client) - - // Verify that logs contain the expected repo constructions - for layoutPath, expectedSuffix := range expectedPaths { - expectedRepo := fmt.Sprintf("localhost:5000/test-repo/%s", expectedSuffix) - found := false - for _, log := range pushParams.Logger.(*mockLogger).logs { - if strings.Contains(log, "Pushing"+expectedRepo) { - found = true - break - } - } - require.True(t, found, "Should construct repo path for layout %s: %s", layoutPath, expectedRepo) - } - - require.NoError(t, err) // Expected to succeed for empty layouts -} - -func TestPushSecurityDatabases_CodeCoverage_AuthOptions(t *testing.T) { - // Test that auth options are constructed correctly - pushParams, _, client := setupTestPushParams(t) - pushParams.RegistryAuth = authn.Anonymous - pushParams.Insecure = true - pushParams.SkipTLSVerification = true - - pkg := createValidSecurityPackage(t) - - client.WithSegmentMock.Return(client) - - err := PushSecurityDatabases(pushParams, pkg, client) - require.NoError(t, err) // Will succeed for empty layouts -} diff --git a/internal/mirror/pull.go b/internal/mirror/pull.go index 0c9ab239..60c21377 100644 --- a/internal/mirror/pull.go +++ b/internal/mirror/pull.go @@ -25,7 +25,6 @@ import ( "github.com/deckhouse/deckhouse-cli/internal/mirror/modules" "github.com/deckhouse/deckhouse-cli/internal/mirror/platform" "github.com/deckhouse/deckhouse-cli/internal/mirror/security" - libmodules "github.com/deckhouse/deckhouse-cli/pkg/libmirror/modules" "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" registryservice "github.com/deckhouse/deckhouse-cli/pkg/registry/service" ) @@ -43,7 +42,7 @@ type PullServiceOptions struct { // IgnoreSuspend allows mirroring even if release channels are suspended IgnoreSuspend bool // ModuleFilter is the filter for module selection (whitelist/blacklist) - ModuleFilter *libmodules.Filter + ModuleFilter *modules.Filter // BundleDir is the directory to store the bundle BundleDir string // BundleChunkSize is the max size of bundle chunks in bytes (0 = no chunking) diff --git a/internal/mirror/releases/versions.go b/internal/mirror/releases/versions.go deleted file mode 100644 index be503698..00000000 --- a/internal/mirror/releases/versions.go +++ /dev/null @@ -1,287 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package releases - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/Masterminds/semver/v3" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - "golang.org/x/exp/maps" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/internal" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/images" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/errorutil" -) - -type releaseChannelVersionResult struct { - ver *semver.Version - err error -} - -func VersionsToMirror(pullParams *params.PullParams, client registry.Client, tagsToMirror []string) ([]semver.Version, []string, error) { - logger := pullParams.Logger - - if len(tagsToMirror) > 0 { - logger.Infof("Skipped releases lookup as tag %q is specifically requested with --deckhouse-tag", pullParams.DeckhouseTag) - } - - vers := make([]*semver.Version, 0, 1) - - for _, tag := range tagsToMirror { - v, err := semver.NewVersion(tag) - if err != nil { - continue - } - - vers = append(vers, v) - } - - releaseChannelsToCopy := internal.GetAllDefaultReleaseChannels() - releaseChannelsToCopy = append(releaseChannelsToCopy, internal.LTSChannel) - - releaseChannelsVersionsResult := make(map[string]releaseChannelVersionResult, len(releaseChannelsToCopy)) - for _, channel := range releaseChannelsToCopy { - v, err := getReleaseChannelVersionFromRegistry(pullParams, channel) - if channel == internal.LTSChannel { - if err != nil { - logger.Warnf("Skipping LTS channel: %v", err) - continue - } - } - - releaseChannelsVersionsResult[channel] = releaseChannelVersionResult{ver: v, err: err} - } - - releaseChannelsVersions := make(map[string]*semver.Version, len(releaseChannelsToCopy)) - - _, ltsChannelFound := releaseChannelsVersionsResult[internal.LTSChannel] - for channel, res := range releaseChannelsVersionsResult { - if !ltsChannelFound && res.err != nil { - return nil, nil, fmt.Errorf("get %s release version from registry: %w", channel, res.err) - } - - if res.err == nil { - releaseChannelsVersions[channel] = res.ver - } - } - - mappedChannels := make(map[string]struct{}, len(releaseChannelsVersions)) - for channel, v := range releaseChannelsVersions { - if len(tagsToMirror) == 0 { - vers = append(vers, v) - mappedChannels[channel] = struct{}{} - continue - } - - for _, tag := range tagsToMirror { - if tag == "v"+v.String() || tag == channel { - vers = append(vers, v) - mappedChannels[channel] = struct{}{} - } - } - } - - channels := make([]string, 0, len(mappedChannels)) - for channel := range mappedChannels { - channels = append(channels, channel) - } - - if len(tagsToMirror) > 0 { - return deduplicateVersions(vers), channels, nil - } - - var mirrorFromVersion *semver.Version - rockSolidVersion, found := releaseChannelsVersions[internal.RockSolidChannel] - if found { - mirrorFromVersion = rockSolidVersion - if pullParams.SinceVersion != nil { - mirrorFromVersion = pullParams.SinceVersion - if rockSolidVersion.LessThan(pullParams.SinceVersion) { - mirrorFromVersion = rockSolidVersion - } - } - } - - tags, err := getReleasedTagsFromRegistry(pullParams, client.WithSegment("release-channel")) - if err != nil { - return nil, nil, fmt.Errorf("get releases from github: %w", err) - } - - alphaChannelVersion, found := releaseChannelsVersions[internal.AlphaChannel] - if found { - versionsAboveMinimal := parseAndFilterVersionsAboveMinimalAndBelowAlpha(mirrorFromVersion, tags, alphaChannelVersion) - versionsAboveMinimal = FilterOnlyLatestPatches(versionsAboveMinimal) - - return deduplicateVersions(append(vers, versionsAboveMinimal...)), channels, nil - } - - return deduplicateVersions(vers), channels, nil -} - -func getReleasedTagsFromRegistry(pullParams *params.PullParams, client registry.Client) ([]string, error) { - logger := pullParams.Logger - - nameOpts, _ := auth.MakeRemoteRegistryRequestOptionsFromMirrorParams(&pullParams.BaseParams) - repo, err := name.NewRepository(pullParams.DeckhouseRegistryRepo+"/release-channel", nameOpts...) - if err != nil { - return nil, fmt.Errorf("parsing repo: %v", err) - } - - logger.Debugf("listing: %s", repo.String()) - - tags, err := client.ListTags(context.TODO()) - if err != nil { - return nil, fmt.Errorf("get tags from Deckhouse registry: %w", err) - } - - return tags, nil -} - -func parseAndFilterVersionsAboveMinimalAndBelowAlpha( - minVersion *semver.Version, - tags []string, - alphaChannelVersion *semver.Version, -) []*semver.Version { - versionsAboveMinimal := make([]*semver.Version, 0) - for _, tag := range tags { - version, err := semver.NewVersion(tag) - if err != nil || minVersion.GreaterThan(version) || version.GreaterThan(alphaChannelVersion) { - continue - } - versionsAboveMinimal = append(versionsAboveMinimal, version) - } - return versionsAboveMinimal -} - -func FilterOnlyLatestPatches(versions []*semver.Version) []*semver.Version { - type majorMinor [2]uint64 - patches := map[majorMinor]uint64{} - for _, version := range versions { - release := majorMinor{version.Major(), version.Minor()} - if patch := patches[release]; patch <= version.Patch() { - patches[release] = version.Patch() - } - } - - topPatches := make([]*semver.Version, 0, len(patches)) - for majMin, patch := range patches { - // Use of semver.MustParse instead of semver.New is important here since we use those versions as map keys, - // structs must be comparable via == operator and semver.New does not provide structs identical to semver.MustParse. - topPatches = append(topPatches, semver.MustParse(fmt.Sprintf("v%d.%d.%d", majMin[0], majMin[1], patch))) - } - return topPatches -} - -func getReleaseChannelVersionFromRegistry(mirrorCtx *params.PullParams, releaseChannel string) (*semver.Version, error) { - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptionsFromMirrorParams(&mirrorCtx.BaseParams) - nameOpts = append(nameOpts, name.StrictValidation) - - ref, err := name.ParseReference(mirrorCtx.DeckhouseRegistryRepo+"/release-channel:"+releaseChannel, nameOpts...) - if err != nil { - return nil, fmt.Errorf("parse rock solid release ref: %w", err) - } - - rockSolidReleaseImage, err := remote.Image(ref, remoteOpts...) - if err != nil { - return nil, fmt.Errorf("get %s release channel data: %w", releaseChannel, err) - } - - versionJSON, err := images.ExtractFileFromImage(rockSolidReleaseImage, "version.json") - if err != nil { - return nil, fmt.Errorf("cannot get %s release channel version: %w", releaseChannel, err) - } - - releaseInfo := &struct { - Version string `json:"version"` - Suspended bool `json:"suspend"` - }{} - if err = json.Unmarshal(versionJSON.Bytes(), releaseInfo); err != nil { - return nil, fmt.Errorf("cannot find release channel version: %w", err) - } - - if releaseInfo.Suspended && !mirrorCtx.IgnoreSuspend { - return nil, fmt.Errorf("cannot mirror Deckhouse: source registry contains suspended release channel %q, try again later (use --ignore-suspend to override)", releaseChannel) - } - - ver, err := semver.NewVersion(releaseInfo.Version) - if err != nil { - return nil, fmt.Errorf("cannot find release channel version: %w", err) - } - - return ver, nil -} - -func deduplicateVersions(versions []*semver.Version) []semver.Version { - m := map[semver.Version]struct{}{} - for _, v := range versions { - m[*v] = struct{}{} - } - - return maps.Keys(m) -} - -func FetchVersionsFromModuleReleaseChannels( - releaseChannelImages map[string]struct{}, - authProvider authn.Authenticator, - insecure, skipVerifyTLS bool, - client registry.Client, -) (map[string]string, error) { - nameOpts, _ := auth.MakeRemoteRegistryRequestOptions(authProvider, insecure, skipVerifyTLS) - channelVersions := map[string]string{} - for imageTag := range releaseChannelImages { - ref, err := name.ParseReference(imageTag, nameOpts...) - if err != nil { - return nil, fmt.Errorf("pull %q release channel: %w", imageTag, err) - } - - // Extract repository path and tag - tag := ref.Identifier() - - img, err := client.GetImage(context.Background(), tag) - if err != nil { - if errorutil.IsImageNotFoundError(err) { - continue - } - return nil, fmt.Errorf("pull %q release channel: %w", imageTag, err) - } - - versionJSON, err := images.ExtractFileFromImage(img, "version.json") - if err != nil { - return nil, fmt.Errorf("read version.json from %q: %w", imageTag, err) - } - - tmp := &struct { - Version string `json:"version"` - }{} - if err = json.Unmarshal(versionJSON.Bytes(), tmp); err != nil { - return nil, fmt.Errorf("parse version.json: %w", err) - } - - channelVersions[imageTag] = tmp.Version - } - - return channelVersions, nil -} diff --git a/internal/mirror/releases/versions_test.go b/internal/mirror/releases/versions_test.go deleted file mode 100644 index cf241a45..00000000 --- a/internal/mirror/releases/versions_test.go +++ /dev/null @@ -1,226 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package releases - -import ( - "testing" - - "github.com/Masterminds/semver/v3" - "github.com/stretchr/testify/require" -) - -func TestParseAndFilterVersionsAboveMinimalAndBelowAlpha(t *testing.T) { - minVersion := semver.MustParse("v1.50.0") - alphaVersion := semver.MustParse("v1.60.0") - - tests := []struct { - name string - tags []string - expected []string - }{ - { - name: "empty tags", - tags: []string{}, - expected: []string{}, - }, - { - name: "tags below minimum", - tags: []string{"v1.49.0", "v1.48.0"}, - expected: []string{}, - }, - { - name: "tags above alpha", - tags: []string{"v1.61.0", "v1.62.0"}, - expected: []string{}, - }, - { - name: "tags in range", - tags: []string{"v1.50.0", "v1.51.0", "v1.52.0", "v1.59.0"}, - expected: []string{"v1.50.0", "v1.51.0", "v1.52.0", "v1.59.0"}, - }, - { - name: "mixed valid and invalid tags", - tags: []string{"v1.49.0", "v1.50.0", "invalid", "v1.51.0", "v1.61.0"}, - expected: []string{"v1.50.0", "v1.51.0"}, - }, - { - name: "exact boundary values", - tags: []string{"v1.50.0", "v1.60.0"}, - expected: []string{"v1.50.0", "v1.60.0"}, // alpha version is included (not GreaterThan) - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := parseAndFilterVersionsAboveMinimalAndBelowAlpha(minVersion, tt.tags, alphaVersion) - - resultStrs := make([]string, len(result)) - for i, v := range result { - resultStrs[i] = "v" + v.String() - } - - require.Equal(t, tt.expected, resultStrs) - }) - } -} - -func TestFilterOnlyLatestPatches(t *testing.T) { - tests := []struct { - name string - input []string - expected []string - }{ - { - name: "empty input", - input: []string{}, - expected: []string{}, - }, - { - name: "single version", - input: []string{"v1.50.0"}, - expected: []string{"v1.50.0"}, - }, - { - name: "multiple patches same major.minor", - input: []string{"v1.50.0", "v1.50.1", "v1.50.2", "v1.50.3"}, - expected: []string{"v1.50.3"}, - }, - { - name: "different major.minor versions", - input: []string{"v1.50.0", "v1.51.0", "v1.52.0", "v2.0.0"}, - expected: []string{"v1.50.0", "v1.51.0", "v1.52.0", "v2.0.0"}, - }, - { - name: "mixed patches", - input: []string{"v1.50.0", "v1.50.1", "v1.51.0", "v1.51.2", "v1.51.1"}, - expected: []string{"v1.50.1", "v1.51.2"}, - }, - { - name: "unsorted input", - input: []string{"v1.51.1", "v1.50.0", "v1.51.0", "v1.50.2"}, - expected: []string{"v1.50.2", "v1.51.1"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - inputVersions := make([]*semver.Version, len(tt.input)) - for i, v := range tt.input { - inputVersions[i] = semver.MustParse(v) - } - - result := FilterOnlyLatestPatches(inputVersions) - - resultStrs := make([]string, len(result)) - for i, v := range result { - resultStrs[i] = "v" + v.String() - } - - // Sort both slices for comparison since map iteration order is not guaranteed - require.ElementsMatch(t, tt.expected, resultStrs) - }) - } -} - -func TestDeduplicateVersions(t *testing.T) { - tests := []struct { - name string - input []string - expected []string - }{ - { - name: "empty input", - input: []string{}, - expected: []string{}, - }, - { - name: "no duplicates", - input: []string{"v1.50.0", "v1.51.0", "v1.52.0"}, - expected: []string{"v1.50.0", "v1.51.0", "v1.52.0"}, - }, - { - name: "with duplicates", - input: []string{"v1.50.0", "v1.51.0", "v1.50.0", "v1.51.0", "v1.52.0"}, - expected: []string{"v1.50.0", "v1.51.0", "v1.52.0"}, - }, - { - name: "all duplicates", - input: []string{"v1.50.0", "v1.50.0", "v1.50.0"}, - expected: []string{"v1.50.0"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - inputVersions := make([]*semver.Version, len(tt.input)) - for i, v := range tt.input { - inputVersions[i] = semver.MustParse(v) - } - - result := deduplicateVersions(inputVersions) - - resultStrs := make([]string, len(result)) - for i, v := range result { - resultStrs[i] = "v" + v.String() - } - - require.ElementsMatch(t, tt.expected, resultStrs) - }) - } -} - -// Benchmark tests -func BenchmarkParseAndFilterVersionsAboveMinimalAndBelowAlpha(b *testing.B) { - minVersion := semver.MustParse("v1.50.0") - alphaVersion := semver.MustParse("v1.60.0") - tags := []string{"v1.49.0", "v1.50.0", "v1.51.0", "v1.52.0", "v1.59.0", "v1.60.0", "v1.61.0"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - parseAndFilterVersionsAboveMinimalAndBelowAlpha(minVersion, tags, alphaVersion) - } -} - -func BenchmarkFilterOnlyLatestPatches(b *testing.B) { - versions := []*semver.Version{ - semver.MustParse("v1.50.0"), - semver.MustParse("v1.50.1"), - semver.MustParse("v1.51.0"), - semver.MustParse("v1.51.2"), - semver.MustParse("v1.52.0"), - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - FilterOnlyLatestPatches(versions) - } -} - -func BenchmarkDeduplicateVersions(b *testing.B) { - versions := []*semver.Version{ - semver.MustParse("v1.50.0"), - semver.MustParse("v1.51.0"), - semver.MustParse("v1.50.0"), - semver.MustParse("v1.51.0"), - semver.MustParse("v1.52.0"), - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - deduplicateVersions(versions) - } -} diff --git a/pkg/libmirror/layouts/indexes.go b/pkg/libmirror/layouts/indexes.go index 02bd820a..d0c0f5f4 100644 --- a/pkg/libmirror/layouts/indexes.go +++ b/pkg/libmirror/layouts/indexes.go @@ -19,9 +19,12 @@ package layouts import ( "bytes" "encoding/json" + "errors" "fmt" "os" + "path/filepath" "sort" + "strings" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" @@ -92,3 +95,123 @@ func SortIndexManifests(l layout.Path) error { return nil } + +var ErrImageNotFound = errors.New("image not found") + +func FindImageByTag(l layout.Path, tag string) (v1.Image, error) { + index, err := l.ImageIndex() + if err != nil { + return nil, err + } + indexManifest, err := index.IndexManifest() + if err != nil { + return nil, err + } + + for _, imageDescriptor := range indexManifest.Manifests { + for key, value := range imageDescriptor.Annotations { + if key == "org.opencontainers.image.ref.name" && strings.HasSuffix(value, ":"+tag) { + return index.Image(imageDescriptor.Digest) + } + } + } + + return nil, ErrImageNotFound +} + +type indexSchema struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []struct { + MediaType string `json:"mediaType,omitempty"` + Size int `json:"size,omitempty"` + Digest string `json:"digest,omitempty"` + } `json:"manifests"` +} + +type ociLayout struct { + ImageLayoutVersion string `json:"imageLayoutVersion"` +} + +func CreateEmptyImageLayout(path string) (layout.Path, error) { + layoutFilePath := filepath.Join(path, "oci-layout") + indexFilePath := filepath.Join(path, "index.json") + blobsPath := filepath.Join(path, "blobs") + + if err := os.MkdirAll(blobsPath, 0o755); err != nil { + return "", fmt.Errorf("mkdir for blobs: %w", err) + } + + layoutContents := ociLayout{ImageLayoutVersion: "1.0.0"} + indexContents := indexSchema{ + SchemaVersion: 2, + MediaType: "application/vnd.oci.image.index.v1+json", + } + + rawJSON, err := json.MarshalIndent(indexContents, "", " ") + if err != nil { + return "", fmt.Errorf("create index.json: %w", err) + } + if err = os.WriteFile(indexFilePath, rawJSON, 0o644); err != nil { + return "", fmt.Errorf("create index.json: %w", err) + } + + rawJSON, err = json.MarshalIndent(layoutContents, "", " ") + if err != nil { + return "", fmt.Errorf("create oci-layout: %w", err) + } + if err = os.WriteFile(layoutFilePath, rawJSON, 0o644); err != nil { + return "", fmt.Errorf("create oci-layout: %w", err) + } + + return layout.Path(path), nil +} + +func FindImageDescriptorByTag(l layout.Path, tag string) (v1.Descriptor, error) { + index, err := l.ImageIndex() + if err != nil { + return v1.Descriptor{}, err + } + indexManifest, err := index.IndexManifest() + if err != nil { + return v1.Descriptor{}, err + } + + for _, imageDescriptor := range indexManifest.Manifests { + for key, value := range imageDescriptor.Annotations { + if key == "org.opencontainers.image.ref.name" && strings.HasSuffix(value, ":"+tag) { + return imageDescriptor, nil + } + } + } + + return v1.Descriptor{}, ErrImageNotFound +} + +func TagImage(l layout.Path, imageDigest v1.Hash, tag string) error { + index, err := l.ImageIndex() + if err != nil { + return err + } + indexManifest, err := index.IndexManifest() + if err != nil { + return err + } + + for _, imageDescriptor := range indexManifest.Manifests { + if imageDescriptor.Digest == imageDigest { + imageRepo, _, found := strings.Cut(imageDescriptor.Annotations["org.opencontainers.image.ref.name"], ":") + // If there is no ":" symbol in the image reference, then it must be a reference by digest and those are fine as is + if found { + imageDescriptor.Annotations["org.opencontainers.image.ref.name"] = imageRepo + ":" + tag + } + imageDescriptor.Annotations["io.deckhouse.image.short_tag"] = tag + if err = l.AppendDescriptor(imageDescriptor); err != nil { + return fmt.Errorf("append descriptor %s: %w", tag, err) + } + return nil + } + } + + return ErrImageNotFound +} diff --git a/pkg/libmirror/layouts/indexes_test.go b/pkg/libmirror/layouts/indexes_test.go index f634c403..9b801453 100644 --- a/pkg/libmirror/layouts/indexes_test.go +++ b/pkg/libmirror/layouts/indexes_test.go @@ -93,3 +93,11 @@ func Test_indexManifestAnnotations_MarshalJSON(t *testing.T) { }) } } + +func createEmptyOCILayout(t *testing.T) layout.Path { + t.Helper() + + l, err := CreateEmptyImageLayout(t.TempDir()) + require.NoError(t, err) + return l +} diff --git a/pkg/libmirror/layouts/layouts.go b/pkg/libmirror/layouts/layouts.go deleted file mode 100644 index 3280b0f5..00000000 --- a/pkg/libmirror/layouts/layouts.go +++ /dev/null @@ -1,524 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path" - "path/filepath" - "reflect" - "strings" - - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/layout" - "github.com/google/go-containerregistry/pkg/v1/remote" - "golang.org/x/exp/maps" - - "github.com/deckhouse/deckhouse/pkg/registry" - regclient "github.com/deckhouse/deckhouse/pkg/registry/client" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/modules" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" -) - -type ModuleImageLayout struct { - ModuleLayout layout.Path - ModuleImages map[string]struct{} - - ReleasesLayout layout.Path - ReleaseImages map[string]struct{} - - ExtraLayout layout.Path - ExtraNamedLayouts map[string]layout.Path - - ExtraNamedImages map[string]map[string]struct{} -} - -type ImageLayouts struct { - Deckhouse layout.Path - DeckhouseImages map[string]struct{} - - Install layout.Path - InstallImages map[string]struct{} - - InstallStandalone layout.Path - InstallStandaloneImages map[string]struct{} - - ReleaseChannel layout.Path - ReleaseChannelImages map[string]struct{} - - TrivyDB layout.Path - TrivyDBImages map[string]struct{} - TrivyBDU layout.Path - TrivyBDUImages map[string]struct{} - TrivyJavaDB layout.Path - TrivyJavaDBImages map[string]struct{} - TrivyChecks layout.Path - TrivyChecksImages map[string]struct{} - - Modules map[string]ModuleImageLayout - - TagsResolver *TagsResolver -} - -func NewImageLayouts() *ImageLayouts { - return &ImageLayouts{ - TagsResolver: NewTagsResolver(), - Modules: make(map[string]ModuleImageLayout), - } -} - -// AsList returns a list of layout.Path's in it. Undefined path's are not included in the list. -func (l *ImageLayouts) AsList() []layout.Path { - layoutsValue := reflect.ValueOf(l).Elem() - layoutPathType := reflect.TypeOf(layout.Path("")) - - paths := make([]layout.Path, 0) - for i := 0; i < layoutsValue.NumField(); i++ { - if layoutsValue.Field(i).Type() != layoutPathType { - continue - } - - if pathValue := layoutsValue.Field(i).String(); pathValue != "" { - paths = append(paths, layout.Path(pathValue)) - } - } - - for _, moduleImageLayout := range l.Modules { - if moduleImageLayout.ModuleLayout != "" { - paths = append(paths, moduleImageLayout.ModuleLayout) - } - if moduleImageLayout.ReleasesLayout != "" { - paths = append(paths, moduleImageLayout.ReleasesLayout) - } - if moduleImageLayout.ExtraLayout != "" { - paths = append(paths, moduleImageLayout.ExtraLayout) - } - } - - return paths -} - -func CreateOCIImageLayoutsForDeckhouse( - rootFolder string, - modules []modules.Module, -) (*ImageLayouts, error) { - var err error - layouts := NewImageLayouts() - - fsPaths := map[*layout.Path]string{ - &layouts.Deckhouse: rootFolder, - &layouts.Install: filepath.Join(rootFolder, "install"), - &layouts.InstallStandalone: filepath.Join(rootFolder, "install-standalone"), - &layouts.ReleaseChannel: filepath.Join(rootFolder, "release-channel"), - } - - for layoutPtr, fsPath := range fsPaths { - *layoutPtr, err = CreateEmptyImageLayout(fsPath) - if err != nil { - return nil, fmt.Errorf("create OCI Image Layout at %s: %w", fsPath, err) - } - } - - for _, module := range modules { - path := filepath.Join(rootFolder, "modules", module.Name) - moduleLayout, err := CreateEmptyImageLayout(path) - if err != nil { - return nil, fmt.Errorf("create OCI Image Layout at %s: %w", path, err) - } - - path = filepath.Join(rootFolder, "modules", module.Name, "release") - moduleReleasesLayout, err := CreateEmptyImageLayout(path) - if err != nil { - return nil, fmt.Errorf("create OCI Image Layout at %s: %w", path, err) - } - - path = filepath.Join(rootFolder, "modules", module.Name, "extra") - moduleExtraLayout, err := CreateEmptyImageLayout(path) - if err != nil { - return nil, fmt.Errorf("create OCI Image Layout at %s: %w", path, err) - } - - layouts.Modules[module.Name] = ModuleImageLayout{ - ModuleLayout: moduleLayout, - ModuleImages: map[string]struct{}{}, - ReleasesLayout: moduleReleasesLayout, - ReleaseImages: map[string]struct{}{}, - ExtraLayout: moduleExtraLayout, - ExtraNamedLayouts: map[string]layout.Path{}, - ExtraNamedImages: map[string]map[string]struct{}{}, - } - } - - return layouts, nil -} - -func CreateEmptyImageLayout(path string) (layout.Path, error) { - layoutFilePath := filepath.Join(path, "oci-layout") - indexFilePath := filepath.Join(path, "index.json") - blobsPath := filepath.Join(path, "blobs") - - if err := os.MkdirAll(blobsPath, 0o755); err != nil { - return "", fmt.Errorf("mkdir for blobs: %w", err) - } - - layoutContents := ociLayout{ImageLayoutVersion: "1.0.0"} - indexContents := indexSchema{ - SchemaVersion: 2, - MediaType: "application/vnd.oci.image.index.v1+json", - } - - rawJSON, err := json.MarshalIndent(indexContents, "", " ") - if err != nil { - return "", fmt.Errorf("create index.json: %w", err) - } - if err = os.WriteFile(indexFilePath, rawJSON, 0o644); err != nil { - return "", fmt.Errorf("create index.json: %w", err) - } - - rawJSON, err = json.MarshalIndent(layoutContents, "", " ") - if err != nil { - return "", fmt.Errorf("create oci-layout: %w", err) - } - if err = os.WriteFile(layoutFilePath, rawJSON, 0o644); err != nil { - return "", fmt.Errorf("create oci-layout: %w", err) - } - - return layout.Path(path), nil -} - -type indexSchema struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Manifests []struct { - MediaType string `json:"mediaType,omitempty"` - Size int `json:"size,omitempty"` - Digest string `json:"digest,omitempty"` - } `json:"manifests"` -} - -type ociLayout struct { - ImageLayoutVersion string `json:"imageLayoutVersion"` -} - -func FillLayoutsWithBasicDeckhouseImages( - pullParams *params.PullParams, - layouts *ImageLayouts, - channelsToMirror []string, - deckhouseVersions []string, -) { - layouts.DeckhouseImages = map[string]struct{}{} - layouts.InstallImages = map[string]struct{}{} - layouts.InstallStandaloneImages = map[string]struct{}{} - layouts.ReleaseChannelImages = map[string]struct{}{} - - // todo(mvasl) need to check if trivy must be here anymore - layouts.TrivyDBImages = map[string]struct{}{ - pullParams.DeckhouseRegistryRepo + "/security/trivy-db:2": {}, - pullParams.DeckhouseRegistryRepo + "/security/trivy-bdu:1": {}, - pullParams.DeckhouseRegistryRepo + "/security/trivy-java-db:1": {}, - pullParams.DeckhouseRegistryRepo + "/security/trivy-checks:0": {}, - } - - for _, version := range deckhouseVersions { - layouts.DeckhouseImages[fmt.Sprintf("%s:%s", pullParams.DeckhouseRegistryRepo, version)] = struct{}{} - layouts.InstallImages[fmt.Sprintf("%s/install:%s", pullParams.DeckhouseRegistryRepo, version)] = struct{}{} - layouts.InstallStandaloneImages[fmt.Sprintf("%s/install-standalone:%s", pullParams.DeckhouseRegistryRepo, version)] = struct{}{} - layouts.ReleaseChannelImages[fmt.Sprintf("%s/release-channel:%s", pullParams.DeckhouseRegistryRepo, version)] = struct{}{} - } - - for _, channel := range channelsToMirror { - layouts.DeckhouseImages[pullParams.DeckhouseRegistryRepo+":"+channel] = struct{}{} - layouts.InstallImages[pullParams.DeckhouseRegistryRepo+"/install:"+channel] = struct{}{} - layouts.InstallStandaloneImages[pullParams.DeckhouseRegistryRepo+"/install-standalone:"+channel] = struct{}{} - layouts.ReleaseChannelImages[pullParams.DeckhouseRegistryRepo+"/release-channel:"+channel] = struct{}{} - } -} - -func FindDeckhouseModulesImages( - params *params.PullParams, - layouts *ImageLayouts, - modulesData []modules.Module, - filter *modules.Filter, - client registry.Client, -) error { - logger := params.Logger - - tmpDir := filepath.Join(params.WorkingDir, "modules") - - counter := 0 - for _, module := range modulesData { - if !filter.Match(&module) { - continue - } - - counter++ - - moduleImageLayouts := layouts.Modules[module.Name] - moduleImageLayouts.ReleaseImages = map[string]struct{}{} - if filter.ShouldMirrorReleaseChannels(module.Name) { - moduleImageLayouts.ReleaseImages = map[string]struct{}{ - path.Join(params.DeckhouseRegistryRepo, params.ModulesPathSuffix, module.Name, "release") + ":alpha": {}, - path.Join(params.DeckhouseRegistryRepo, params.ModulesPathSuffix, module.Name, "release") + ":beta": {}, - path.Join(params.DeckhouseRegistryRepo, params.ModulesPathSuffix, module.Name, "release") + ":early-access": {}, - path.Join(params.DeckhouseRegistryRepo, params.ModulesPathSuffix, module.Name, "release") + ":stable": {}, - path.Join(params.DeckhouseRegistryRepo, params.ModulesPathSuffix, module.Name, "release") + ":rock-solid": {}, - } - } - - logger.Infof("%d:\t%s - find external module images", counter, module.Name) - - moduleImages, moduleImagesWithExternal, releaseImages, err := modules.FindExternalModuleImages( - params, - &module, - filter, - params.RegistryAuth, - params.Insecure, - params.SkipTLSVerification, - client.WithSegment(module.Name), - ) - if err != nil { - return fmt.Errorf("Find images of %s: %w", module.Name, err) - } - - moduleImageLayouts.ModuleImages = moduleImagesWithExternal - maps.Copy(moduleImageLayouts.ReleaseImages, releaseImages) - - logger.Infof("%d:\t%s - find module extra images", counter, module.Name) - - // Find extra images if any exist - extraImages, err := modules.FindModuleExtraImages( - params, - &module, - moduleImages, - params.RegistryAuth, - params.Insecure, - params.SkipTLSVerification, - client.WithSegment(module.Name), - ) - if err != nil { - return fmt.Errorf("Find extra images of %s: %w", module.Name, err) - } - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptionsFromMirrorParams(¶ms.BaseParams) - - logger.Infof("Searching for VEX images") - - for digest := range moduleImagesWithExternal { - vexImageName, err := FindVexImage( - params, - module.RegistryPath, - nameOpts, - remoteOpts, - digest, - client, - ) - - if err != nil { - return fmt.Errorf("Find VEX image for digest %q: %w", digest, err) - } - - if vexImageName != "" { - logger.Debugf("Vex image found %s", vexImageName) - moduleImagesWithExternal[vexImageName] = struct{}{} - } - } - - for digest := range extraImages { - refWoTag, _ := splitImageRefByRepoAndTag(digest) - - _, extraName := splitExtraName(refWoTag) - - _, ok := layouts.Modules[module.Name].ExtraNamedLayouts[extraName] - if !ok { - extraLayout, err := CreateEmptyImageLayout(filepath.Join(tmpDir, module.Name, "extra", extraName)) - if err != nil { - return fmt.Errorf("create OCI layout: %w", err) - } - - layouts.Modules[module.Name].ExtraNamedLayouts[extraName] = extraLayout - } - - _, ok = layouts.Modules[module.Name].ExtraNamedImages[extraName] - if !ok { - layouts.Modules[module.Name].ExtraNamedImages[extraName] = make(map[string]struct{}, 1) - } - - images := layouts.Modules[module.Name].ExtraNamedImages[extraName] - images[digest] = struct{}{} - - vexImageName, err := FindVexImage( - params, - module.RegistryPath, - nameOpts, - remoteOpts, - digest, - client, - ) - - if err != nil { - return fmt.Errorf("Find VEX image for digest %q: %w", digest, err) - } - - if vexImageName != "" { - logger.Debugf("Vex image found %s", vexImageName) - extraImages[vexImageName] = struct{}{} - images[vexImageName] = struct{}{} - } - - layouts.Modules[module.Name].ExtraNamedImages[extraName] = images - } - - if len(moduleImageLayouts.ModuleImages) == 0 { - return fmt.Errorf("found no releases for %s", module.Name) - } - - layouts.Modules[module.Name] = moduleImageLayouts - - logger.Infof("%d:\t%s", counter, module.Name) - } - - return nil -} - -var ErrImageNotFound = errors.New("image not found") - -func FindImageByTag(l layout.Path, tag string) (v1.Image, error) { - index, err := l.ImageIndex() - if err != nil { - return nil, err - } - indexManifest, err := index.IndexManifest() - if err != nil { - return nil, err - } - - for _, imageDescriptor := range indexManifest.Manifests { - for key, value := range imageDescriptor.Annotations { - if key == "org.opencontainers.image.ref.name" && strings.HasSuffix(value, ":"+tag) { - return index.Image(imageDescriptor.Digest) - } - } - } - - return nil, ErrImageNotFound -} - -func FindImageDescriptorByTag(l layout.Path, tag string) (v1.Descriptor, error) { - index, err := l.ImageIndex() - if err != nil { - return v1.Descriptor{}, err - } - indexManifest, err := index.IndexManifest() - if err != nil { - return v1.Descriptor{}, err - } - - for _, imageDescriptor := range indexManifest.Manifests { - for key, value := range imageDescriptor.Annotations { - if key == "org.opencontainers.image.ref.name" && strings.HasSuffix(value, ":"+tag) { - return imageDescriptor, nil - } - } - } - - return v1.Descriptor{}, ErrImageNotFound -} - -func TagImage(l layout.Path, imageDigest v1.Hash, tag string) error { - index, err := l.ImageIndex() - if err != nil { - return err - } - indexManifest, err := index.IndexManifest() - if err != nil { - return err - } - - for _, imageDescriptor := range indexManifest.Manifests { - if imageDescriptor.Digest == imageDigest { - imageRepo, _, found := strings.Cut(imageDescriptor.Annotations["org.opencontainers.image.ref.name"], ":") - // If there is no ":" symbol in the image reference, then it must be a reference by digest and those are fine as is - if found { - imageDescriptor.Annotations["org.opencontainers.image.ref.name"] = imageRepo + ":" + tag - } - imageDescriptor.Annotations["io.deckhouse.image.short_tag"] = tag - if err = l.AppendDescriptor(imageDescriptor); err != nil { - return fmt.Errorf("append descriptor %s: %w", tag, err) - } - return nil - } - } - - return ErrImageNotFound -} - -func FindVexImage( - params *params.PullParams, - _ string, - nameOpts []name.Option, - _ []remote.Option, - digest string, - client registry.Client, -) (string, error) { - logger := params.Logger - - // vex image reference check - vexImageName := strings.Replace(strings.Replace(digest, "@sha256:", "@sha256-", 1), "@sha256", ":sha256", 1) + ".att" - - logger.Debugf("Checking vex image from %s", vexImageName) - - _, err := name.ParseReference(vexImageName, nameOpts...) - if err != nil { - return "", fmt.Errorf("parse reference: %w", err) - } - - // Use LastIndex to correctly handle URLs with port (e.g., localhost:443/repo:tag) - splitIndex := strings.LastIndex(vexImageName, ":") - if splitIndex == -1 { - return "", fmt.Errorf("invalid vex image name format: %s", vexImageName) - } - imagePath := vexImageName[:splitIndex] - tag := vexImageName[splitIndex+1:] - - // Add missing path segments to client if VEX image is in a subpath - imageSegmentsRaw := strings.TrimPrefix(imagePath, client.GetRegistry()) - imageSegmentsRaw = strings.TrimPrefix(imageSegmentsRaw, "/") - if imageSegmentsRaw != "" { - for _, segment := range strings.Split(imageSegmentsRaw, "/") { - client = client.WithSegment(segment) - logger.Debugf("Segment: %s", segment) - } - } - - err = client.CheckImageExists(context.TODO(), tag) - if errors.Is(err, regclient.ErrImageNotFound) { - // Image not found, which is expected for non-vulnerable images - return "", nil - } - if err != nil { - return "", fmt.Errorf("check VEX image exists: %w", err) - } - - return vexImageName, nil -} diff --git a/pkg/libmirror/layouts/layouts_test.go b/pkg/libmirror/layouts/layouts_test.go deleted file mode 100644 index 2bcc0ace..00000000 --- a/pkg/libmirror/layouts/layouts_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCreateEmptyImageLayoutAtPath(t *testing.T) { - p, err := os.MkdirTemp(os.TempDir(), "create_layout_test") - require.NoError(t, err) - t.Cleanup(func() { - _ = os.RemoveAll(p) - }) - - _, err = CreateEmptyImageLayout(p) - require.NoError(t, err) - require.DirExists(t, p) - require.FileExists(t, filepath.Join(p, "oci-layout")) - require.FileExists(t, filepath.Join(p, "index.json")) -} - -func TestImagesLayoutsAllLayouts(t *testing.T) { - il := &ImageLayouts{ - Deckhouse: createEmptyOCILayout(t), - ReleaseChannel: createEmptyOCILayout(t), - InstallStandalone: createEmptyOCILayout(t), - Modules: map[string]ModuleImageLayout{ - "commander-agent": {ModuleLayout: createEmptyOCILayout(t), ReleasesLayout: createEmptyOCILayout(t)}, - "commander": {ModuleLayout: createEmptyOCILayout(t), ReleasesLayout: createEmptyOCILayout(t)}, - }, - } - - layouts := il.AsList() - require.Len(t, layouts, 7, "AsList should return exactly the number of non-empty layouts defined within it") - require.Contains(t, layouts, il.Deckhouse, "All non-empty layouts should be returned") - require.Contains(t, layouts, il.ReleaseChannel, "All non-empty layouts should be returned") - require.Contains(t, layouts, il.InstallStandalone, "All non-empty layouts should be returned") - require.Contains(t, layouts, il.Modules["commander"].ModuleLayout, "All non-empty layouts should be returned") - require.Contains(t, layouts, il.Modules["commander"].ReleasesLayout, "All non-empty layouts should be returned") - require.Contains(t, layouts, il.Modules["commander-agent"].ModuleLayout, "All non-empty layouts should be returned") - require.Contains(t, layouts, il.Modules["commander-agent"].ReleasesLayout, "All non-empty layouts should be returned") -} diff --git a/pkg/libmirror/layouts/pull.go b/pkg/libmirror/layouts/pull.go deleted file mode 100644 index cb9e7bde..00000000 --- a/pkg/libmirror/layouts/pull.go +++ /dev/null @@ -1,321 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "context" - "errors" - "fmt" - "path" - "strings" - "time" - - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/layout" - - "github.com/deckhouse/deckhouse/pkg/registry" - regclient "github.com/deckhouse/deckhouse/pkg/registry/client" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/retry" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/retry/task" -) - -func PullInstallers(pullParams *params.PullParams, layouts *ImageLayouts, client registry.Client) error { - pullParams.Logger.InfoLn("Beginning to pull installers") - if err := PullImageSet( - pullParams, - layouts.Install, - layouts.InstallImages, - client, - WithTagToDigestMapper(layouts.TagsResolver.GetTagDigest), - ); err != nil { - return err - } - pullParams.Logger.InfoLn("All required installers are pulled!") - return nil -} - -func PullStandaloneInstallers(pullParams *params.PullParams, layouts *ImageLayouts, client registry.Client) error { - pullParams.Logger.InfoLn("Beginning to pull standalone installers") - if err := PullImageSet( - pullParams, - layouts.InstallStandalone, - layouts.InstallStandaloneImages, - client, - WithTagToDigestMapper(layouts.TagsResolver.GetTagDigest), - WithAllowMissingTags(true), - ); err != nil { - return err - } - pullParams.Logger.InfoLn("All required standalone installers are pulled!") - return nil -} - -func PullDeckhouseReleaseChannels(pullParams *params.PullParams, layouts *ImageLayouts, client registry.Client) error { - pullParams.Logger.InfoLn("Beginning to pull Deckhouse release channels information") - if err := PullImageSet( - pullParams, - layouts.ReleaseChannel, - layouts.ReleaseChannelImages, - client, - WithTagToDigestMapper(layouts.TagsResolver.GetTagDigest), - WithAllowMissingTags(pullParams.DeckhouseTag != ""), - ); err != nil { - return err - } - pullParams.Logger.InfoLn("Deckhouse release channels are pulled!") - return nil -} - -func PullDeckhouseImages(pullParams *params.PullParams, layouts *ImageLayouts, client registry.Client) error { - pullParams.Logger.InfoLn("Beginning to pull Deckhouse, this may take a while") - if err := PullImageSet( - pullParams, - layouts.Deckhouse, - layouts.DeckhouseImages, - client, - WithTagToDigestMapper(layouts.TagsResolver.GetTagDigest), - ); err != nil { - return err - } - pullParams.Logger.InfoLn("All required Deckhouse images are pulled!") - return nil -} - -func PullModules(pullParams *params.PullParams, layouts *ImageLayouts, client registry.Client) error { - for moduleName, moduleData := range layouts.Modules { - // Skip main module images if only pulling extra images - if !pullParams.OnlyExtraImages { - pullParams.Logger.InfoLn(moduleName, "images") - - if err := PullImageSet( - pullParams, - moduleData.ModuleLayout, - moduleData.ModuleImages, - client, - WithTagToDigestMapper(layouts.TagsResolver.GetTagDigest), - ); err != nil { - return fmt.Errorf("pull %q module: %w", moduleName, err) - } - - pullParams.Logger.InfoLn(moduleName, "release channels") - - if err := PullImageSet( - pullParams, - moduleData.ReleasesLayout, - moduleData.ReleaseImages, - client, - WithTagToDigestMapper(layouts.TagsResolver.GetTagDigest), - WithAllowMissingTags(true), - ); err != nil { - return fmt.Errorf("pull %q module release information: %w", moduleName, err) - } - } - - // Always pull extra images if they exist - if len(moduleData.ExtraNamedImages) > 0 { - for extraName, imageSet := range moduleData.ExtraNamedImages { - l := moduleData.ExtraNamedLayouts[extraName] - - pullParams.Logger.InfoLn(moduleName, "extra images") - - if err := PullImageSet( - pullParams, - l, - imageSet, - client, - WithTagToDigestMapper(layouts.TagsResolver.GetTagDigest), - WithAllowMissingTags(true), - ); err != nil { - return fmt.Errorf("pull %q module extra images: %w", moduleName, err) - } - } - } - } - - message := "Deckhouse modules pulled!" - if pullParams.OnlyExtraImages { - message = "Extra images pulled!" - } - pullParams.Logger.InfoLn(message) - return nil -} - -func PullTrivyVulnerabilityDatabasesImages( - pullParams *params.PullParams, - layouts *ImageLayouts, - client registry.Client, -) error { - nameOpts, _ := auth.MakeRemoteRegistryRequestOptionsFromMirrorParams(&pullParams.BaseParams) - - dbImages := map[layout.Path]string{ - layouts.TrivyDB: path.Join(pullParams.DeckhouseRegistryRepo, "security", "trivy-db:2"), - layouts.TrivyBDU: path.Join(pullParams.DeckhouseRegistryRepo, "security", "trivy-bdu:1"), - layouts.TrivyJavaDB: path.Join(pullParams.DeckhouseRegistryRepo, "security", "trivy-java-db:1"), - layouts.TrivyChecks: path.Join(pullParams.DeckhouseRegistryRepo, "security", "trivy-checks:0"), - } - - for dbImageLayout, imageRef := range dbImages { - ref, err := name.ParseReference(imageRef, nameOpts...) - if err != nil { - return fmt.Errorf("parse trivy-db reference %q: %w", imageRef, err) - } - - if err = PullImageSet( - pullParams, - dbImageLayout, - map[string]struct{}{ref.String(): {}}, - client, - WithTagToDigestMapper(NopTagToDigestMappingFunc), - WithAllowMissingTags(true), // SE edition does not contain images for trivy - ); err != nil { - return fmt.Errorf("pull vulnerability database: %w", err) - } - } - - return nil -} - -func PullImageSet( - pullParams *params.PullParams, - targetLayout layout.Path, - imageSet map[string]struct{}, - client registry.Client, - opts ...func(opts *pullImageSetOptions), -) error { - logger := pullParams.Logger - - pullOpts := &pullImageSetOptions{} - for _, o := range opts { - o(pullOpts) - } - - nameOpts, _ := auth.MakeRemoteRegistryRequestOptions( - pullParams.RegistryAuth, - pullParams.Insecure, - pullParams.SkipTLSVerification, - ) - - pullCount, totalCount := 1, len(imageSet) - for imageReferenceString := range imageSet { - imageRepo, originalTag := splitImageRefByRepoAndTag(imageReferenceString) - - // If we already know the digest of the tagged image, we should pull it by this digest instead of pulling by tag - // to avoid race-conditions between mirroring and releasing new builds on release channels. - pullReference := imageReferenceString - if pullOpts.tagToDigestMapper != nil { - if mapping := pullOpts.tagToDigestMapper(imageReferenceString); mapping != nil { - pullReference = imageRepo + "@" + mapping.String() - } - } - - ref, err := name.ParseReference(pullReference, nameOpts...) - if err != nil { - return fmt.Errorf("parse image reference %q: %w", pullReference, err) - } - - logger.Debugf("reference here: %s", ref.String()) - - // tag here can be ussually tag or digest - // thats why we need original tag calculated upper - imagePath, tag := splitImageRefByRepoAndTag(pullReference) - - scopedClient := client - imageSegmentsRaw := strings.TrimPrefix(imagePath, scopedClient.GetRegistry()) - imageSegments := strings.Split(imageSegmentsRaw, "/") - - for i, segment := range imageSegments { - scopedClient = scopedClient.WithSegment(segment) - logger.Debugf("Segment %d: %s", i, segment) - } - - err = retry.RunTask( - context.TODO(), - pullParams.Logger, - fmt.Sprintf("[%d / %d] Pulling %s ", pullCount, totalCount, imageReferenceString), - task.WithConstantRetries(5, 10*time.Second, func(_ context.Context) error { - img, err := scopedClient.GetImage(context.TODO(), tag) - if err != nil { - if errors.Is(err, regclient.ErrImageNotFound) && pullOpts.allowMissingTags { - logger.WarnLn("⚠️ Not found in registry, skipping pull") - return nil - } - - logger.Debugf("failed to pull image %s:%s: %v", imageReferenceString, tag, err) - - return fmt.Errorf("pull image metadata: %w", err) - } - - err = targetLayout.AppendImage(img, - layout.WithPlatform(v1.Platform{Architecture: "amd64", OS: "linux"}), - layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": imageReferenceString, - // original tag here to have tagged releases as an alias - "io.deckhouse.image.short_tag": originalTag, - }), - ) - if err != nil { - return fmt.Errorf("write image to index: %w", err) - } - - return nil - })) - if err != nil { - return fmt.Errorf("pull image %q: %w", imageReferenceString, err) - } - pullCount++ - } - return nil -} - -func splitImageRefByRepoAndTag(imageReferenceString string) (string, string) { - splitIndex := strings.LastIndex(imageReferenceString, ":") - repo := imageReferenceString[:splitIndex] - tag := imageReferenceString[splitIndex+1:] - - return repo, tag -} - -func splitExtraName(imageReferenceStringWOTag string) (string, string) { - splitIndex := strings.LastIndex(imageReferenceStringWOTag, "/") - repo := imageReferenceStringWOTag[:splitIndex] - extra := imageReferenceStringWOTag[splitIndex+1:] - - return repo, extra -} - -type pullImageSetOptions struct { - tagToDigestMapper TagToDigestMappingFunc - allowMissingTags bool -} - -func WithAllowMissingTags(allow bool) func(opts *pullImageSetOptions) { - return func(opts *pullImageSetOptions) { - opts.allowMissingTags = allow - } -} - -type TagToDigestMappingFunc func(imageRef string) *v1.Hash - -func WithTagToDigestMapper(fn TagToDigestMappingFunc) func(opts *pullImageSetOptions) { - return func(opts *pullImageSetOptions) { - opts.tagToDigestMapper = fn - } -} diff --git a/pkg/libmirror/layouts/pull_test.go b/pkg/libmirror/layouts/pull_test.go deleted file mode 100644 index 9b592079..00000000 --- a/pkg/libmirror/layouts/pull_test.go +++ /dev/null @@ -1,228 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "context" - "fmt" - "io" - "log/slog" - "net/http/httptest" - "strings" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/registry" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/layout" - "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" - mock "github.com/deckhouse/deckhouse-cli/pkg/mock" - "github.com/deckhouse/deckhouse-cli/pkg/registry/image" - dregistry "github.com/deckhouse/deckhouse/pkg/registry" -) - -type mockRegistryImage struct { - v1.Image - ref string -} - -func (m *mockRegistryImage) Extract() io.ReadCloser { - return io.NopCloser(strings.NewReader("")) -} - -func (m *mockRegistryImage) GetMetadata() (pkg.ImageMeta, error) { - return image.NewImageMeta(m.ref, "", nil), nil -} - -func (m *mockRegistryImage) SetMetadata(meta pkg.ImageMeta) { - // No-op for mock -} - -var testLogger = log.NewSLogger(slog.LevelDebug) - -func TestPullTrivyVulnerabilityDatabaseImageSuccessSkipTLS(t *testing.T) { - blobHandler := registry.NewInMemoryBlobHandler() - registryHandler := registry.New(registry.WithBlobHandler(blobHandler)) - server := httptest.NewTLSServer(registryHandler) - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authn.Anonymous, false, true) - - deckhouseRepo := strings.TrimPrefix(server.URL, "https://") + "/deckhouse/ee" - images := []string{ - deckhouseRepo + "/security/trivy-db:2", - deckhouseRepo + "/security/trivy-bdu:1", - deckhouseRepo + "/security/trivy-java-db:1", - deckhouseRepo + "/security/trivy-checks:0", - } - - wantImages := make([]v1.Image, 0) - for _, image := range images { - ref, err := name.ParseReference(image, nameOpts...) - require.NoError(t, err) - wantImage, err := random.Image(256, 1) - require.NoError(t, err) - require.NoError(t, remote.Write(ref, wantImage, remoteOpts...)) - wantImages = append(wantImages, wantImage) - } - - wantRegistryImages := make([]pkg.RegistryImage, 0) - for i, img := range wantImages { - wantRegistryImages = append(wantRegistryImages, &mockRegistryImage{Image: img, ref: images[i]}) - } - - layouts := &ImageLayouts{ - TrivyDB: createEmptyOCILayout(t), - TrivyBDU: createEmptyOCILayout(t), - TrivyJavaDB: createEmptyOCILayout(t), - TrivyChecks: createEmptyOCILayout(t), - } - - client := mock.NewRegistryClientMock(t) - client.GetRegistryMock.Return(strings.TrimPrefix(server.URL, "https://")) - client.WithSegmentMock.Return(client) - callCount := 0 - client.GetImageMock.Set(func(ctx context.Context, tag string, opts ...dregistry.ImageGetOption) (dregistry.Image, error) { - switch tag { - case "2": - return wantRegistryImages[0], nil - case "1": - if callCount == 0 { - callCount++ - return wantRegistryImages[1], nil - } else { - return wantRegistryImages[2], nil - } - case "0": - return wantRegistryImages[3], nil - default: - return nil, fmt.Errorf("unexpected tag %s", tag) - } - }) - err := PullTrivyVulnerabilityDatabasesImages( - ¶ms.PullParams{BaseParams: params.BaseParams{ - Logger: testLogger, - RegistryAuth: authn.Anonymous, - DeckhouseRegistryRepo: deckhouseRepo, - SkipTLSVerification: true, - }}, - layouts, - client, - ) - require.NoError(t, err) -} - -func TestPullTrivyVulnerabilityDatabaseImageSuccessInsecure(t *testing.T) { - blobHandler := registry.NewInMemoryBlobHandler() - registryHandler := registry.New(registry.WithBlobHandler(blobHandler)) - server := httptest.NewServer(registryHandler) - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authn.Anonymous, true, false) - - deckhouseRepo := strings.TrimPrefix(server.URL, "http://") + "/deckhouse/ee" - images := []string{ - deckhouseRepo + "/security/trivy-db:2", - deckhouseRepo + "/security/trivy-bdu:1", - deckhouseRepo + "/security/trivy-java-db:1", - deckhouseRepo + "/security/trivy-checks:0", - } - - wantImages := make([]v1.Image, 0) - for _, image := range images { - ref, err := name.ParseReference(image, nameOpts...) - require.NoError(t, err) - wantImage, err := random.Image(256, 1) - require.NoError(t, err) - require.NoError(t, remote.Write(ref, wantImage, remoteOpts...)) - wantImages = append(wantImages, wantImage) - } - - wantRegistryImages := make([]pkg.RegistryImage, 0) - for i, img := range wantImages { - wantRegistryImages = append(wantRegistryImages, &mockRegistryImage{Image: img, ref: images[i]}) - } - - layouts := &ImageLayouts{ - TrivyDB: createEmptyOCILayout(t), - TrivyBDU: createEmptyOCILayout(t), - TrivyJavaDB: createEmptyOCILayout(t), - TrivyChecks: createEmptyOCILayout(t), - } - - client := mock.NewRegistryClientMock(t) - client.GetRegistryMock.Return(strings.TrimPrefix(server.URL, "http://")) - client.WithSegmentMock.Return(client) - callCount := 0 - client.GetImageMock.Set(func(ctx context.Context, tag string, opts ...dregistry.ImageGetOption) (dregistry.Image, error) { - switch tag { - case "2": - return wantRegistryImages[0], nil - case "1": - if callCount == 0 { - callCount++ - return wantRegistryImages[1], nil - } else { - return wantRegistryImages[2], nil - } - case "0": - return wantRegistryImages[3], nil - default: - return nil, fmt.Errorf("unexpected tag %s", tag) - } - }) - err := PullTrivyVulnerabilityDatabasesImages( - ¶ms.PullParams{BaseParams: params.BaseParams{ - Logger: testLogger, - RegistryAuth: authn.Anonymous, - DeckhouseRegistryRepo: deckhouseRepo, - Insecure: true, - }}, - layouts, - client, - ) - require.NoError(t, err) -} - -func layoutByIndex(t *testing.T, layouts *ImageLayouts, idx int) layout.Path { - t.Helper() - switch idx { - case 0: - return layouts.TrivyDB - case 1: - return layouts.TrivyBDU - case 2: - return layouts.TrivyJavaDB - case 3: - return layouts.TrivyChecks - default: - t.Fatalf("Unexpected layout index, expected only [0-3], but got %d", idx) - return "" - } -} - -func createEmptyOCILayout(t *testing.T) layout.Path { - t.Helper() - - l, err := CreateEmptyImageLayout(t.TempDir()) - require.NoError(t, err) - return l -} diff --git a/pkg/libmirror/layouts/push.go b/pkg/libmirror/layouts/push.go deleted file mode 100644 index 2563c901..00000000 --- a/pkg/libmirror/layouts/push.go +++ /dev/null @@ -1,193 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/layout" - "github.com/hashicorp/go-multierror" - "github.com/samber/lo" - "github.com/samber/lo/parallel" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/errorutil" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/retry" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/retry/task" -) - -func PushLayoutToRepo( - client registry.Client, - imagesLayout layout.Path, - registryRepo string, - authProvider authn.Authenticator, - logger params.Logger, - parallelismConfig params.ParallelismConfig, - insecure, skipVerifyTLS bool, -) error { - return PushLayoutToRepoContext( - context.Background(), - client, - imagesLayout, - registryRepo, - authProvider, - logger, - parallelismConfig, - insecure, - skipVerifyTLS, - ) -} - -func PushLayoutToRepoContext( - ctx context.Context, - client registry.Client, - imagesLayout layout.Path, - registryRepo string, - authProvider authn.Authenticator, - logger params.Logger, - parallelismConfig params.ParallelismConfig, - insecure, skipVerifyTLS bool, -) error { - refOpts, _ := auth.MakeRemoteRegistryRequestOptions(authProvider, insecure, skipVerifyTLS) - - index, err := imagesLayout.ImageIndex() - if err != nil { - return fmt.Errorf("Read OCI Image Index: %w", err) - } - - indexManifest, err := index.IndexManifest() - if err != nil { - return fmt.Errorf("Parse OCI Image Index Manifest: %w", err) - } - - if len(indexManifest.Manifests) == 0 { - return nil - } - - batches := lo.Chunk(indexManifest.Manifests, parallelismConfig.Images) - batchesCount, imagesCount := 1, 1 - - for _, manifestSet := range batches { - if parallelismConfig.Images == 1 { - cfg := &pushImageConfig{ - client: client, - registryRepo: registryRepo, - index: index, - manifest: manifestSet[0], - refOpts: refOpts, - logger: logger, - imageNum: imagesCount, - totalImages: len(indexManifest.Manifests), - } - if err = pushImage(ctx, cfg); err != nil { - return fmt.Errorf("Push Image: %w", err) - } - imagesCount++ - continue - } - - err = logger.Process(fmt.Sprintf("Pushing batch %d / %d", batchesCount, len(batches)), func() error { - logger.InfoLn("Images in batch:") - for _, manifest := range manifestSet { - tag := manifest.Annotations["io.deckhouse.image.short_tag"] - logger.Infof("- %s", registryRepo+":"+tag) - } - - errMu := &sync.Mutex{} - merr := &multierror.Error{} - currentImagesCount := imagesCount - parallel.ForEach(manifestSet, func(item v1.Descriptor, idx int) { - imageNum := currentImagesCount + idx - cfg := &pushImageConfig{ - client: client, - registryRepo: registryRepo, - index: index, - manifest: item, - refOpts: refOpts, - logger: logger, - imageNum: imageNum, - totalImages: len(indexManifest.Manifests), - } - if err = pushImage(ctx, cfg); err != nil { - errMu.Lock() - defer errMu.Unlock() - merr = multierror.Append(merr, err) - } - }) - - return merr.ErrorOrNil() - }) - if err != nil { - return fmt.Errorf("Push batch of images: %w", err) - } - batchesCount++ - imagesCount += len(manifestSet) - } - - return nil -} - -type pushImageConfig struct { - client registry.Client - registryRepo string - index v1.ImageIndex - manifest v1.Descriptor - refOpts []name.Option - logger params.Logger - imageNum int - totalImages int -} - -func pushImage(ctx context.Context, cfg *pushImageConfig) error { - tag := cfg.manifest.Annotations["io.deckhouse.image.short_tag"] - imageRef := cfg.registryRepo + ":" + tag - img, err := cfg.index.Image(cfg.manifest.Digest) - if err != nil { - return fmt.Errorf("Read image: %v", err) - } - ref, err := name.ParseReference(imageRef, cfg.refOpts...) - if err != nil { - return fmt.Errorf("Parse image reference: %v", err) - } - - err = retry.RunTask( - ctx, - cfg.logger, - fmt.Sprintf("[%d / %d] Pushing %s", cfg.imageNum, cfg.totalImages, imageRef), - task.WithConstantRetries(4, 3*time.Second, func(ctx context.Context) error { - if err = cfg.client.PushImage(ctx, tag, img); err != nil { - if errorutil.IsTrivyMediaTypeNotAllowedError(err) { - return fmt.Errorf(errorutil.CustomTrivyMediaTypesWarning) - } - return fmt.Errorf("Write %s to registry: %w", ref.String(), err) - } - return nil - })) - if err != nil { - return fmt.Errorf("Run push task: %v", err) - } - return nil -} diff --git a/pkg/libmirror/layouts/push_test.go b/pkg/libmirror/layouts/push_test.go deleted file mode 100644 index 3e2f3311..00000000 --- a/pkg/libmirror/layouts/push_test.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "log/slog" - "math/rand/v2" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/layout" - "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" - "github.com/deckhouse/deckhouse-cli/pkg/mock" - mirrorTestUtils "github.com/deckhouse/deckhouse-cli/testing/util/mirror" -) - -func TestPushLayoutToRepoWithParallelism(t *testing.T) { - s := require.New(t) - - const totalImages, layersPerImage = 10, 3 - imagesLayout := createEmptyOCILayout(t) - host, repoPath, _ := mirrorTestUtils.SetupEmptyRegistryRepo(false) - - client := mock.NewRegistryClientMock(t) - client.PushImageMock.Return(nil) - - platformOpt := layout.WithPlatform(v1.Platform{OS: "linux", Architecture: "amd64"}) - for range [totalImages]struct{}{} { - img, err := random.Image(rand.Int64N(513), layersPerImage) - s.NoError(err) - digest, err := img.Digest() - s.NoError(err) - err = imagesLayout.AppendImage(img, platformOpt, layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": host + repoPath + "@" + digest.String(), - "io.deckhouse.image.short_tag": digest.Hex, - })) - s.NoError(err) - } - - err := PushLayoutToRepo( - client, - imagesLayout, - host+repoPath, // Images repo - authn.Anonymous, - log.NewSLogger(slog.LevelDebug), - params.ParallelismConfig{ - Blobs: 4, - Images: 5, - }, - true, // Use plain insecure HTTP - false, // TLS verification irrelevant to HTTP requests - ) - - s.NoError(err, "Push should not fail") - - // Verify that PushImage was called for each image - s.Len(client.PushImageMock.Calls(), totalImages, "PushImage should be called for each image") -} - -func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { - s := require.New(t) - - const totalImages, layersPerImage = 10, 3 - imagesLayout := createEmptyOCILayout(t) - host, repoPath, _ := mirrorTestUtils.SetupEmptyRegistryRepo(false) - - client := mock.NewRegistryClientMock(t) - client.PushImageMock.Return(nil) - - platformOpt := layout.WithPlatform(v1.Platform{OS: "linux", Architecture: "amd64"}) - for range [totalImages]struct{}{} { - img, err := random.Image(rand.Int64N(513), layersPerImage) - s.NoError(err) - digest, err := img.Digest() - s.NoError(err) - err = imagesLayout.AppendImage(img, platformOpt, layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": host + repoPath + "@" + digest.String(), - "io.deckhouse.image.short_tag": digest.Hex, - })) - s.NoError(err) - } - - err := PushLayoutToRepo( - client, - imagesLayout, - host+repoPath, // Images repo - authn.Anonymous, - log.NewSLogger(slog.LevelDebug), - params.ParallelismConfig{ - Blobs: 4, - Images: 1, - }, - true, // Use plain insecure HTTP - false, // TLS verification irrelevant to HTTP requests - ) - - s.NoError(err, "Push should not fail") - - // Verify that PushImage was called for each image - s.Len(client.PushImageMock.Calls(), totalImages, "PushImage should be called for each image") -} - -func TestPushEmptyLayoutToRepo(t *testing.T) { - s := require.New(t) - host, repoPath, blobHandler := mirrorTestUtils.SetupEmptyRegistryRepo(false) - - client := mock.NewRegistryClientMock(t) - - emptyLayout := createEmptyOCILayout(t) - err := PushLayoutToRepo( - client, - emptyLayout, - host+repoPath, - authn.Anonymous, - log.NewSLogger(slog.LevelDebug), - params.DefaultParallelism, - true, // Use plain insecure HTTP - false, // TLS verification irrelevant to HTTP requests - ) - s.NoError(err, "Push should not fail") - s.Len(blobHandler.ListBlobs(), 0, "No blobs should be pushed to registry") -} diff --git a/pkg/libmirror/layouts/tag_resolver.go b/pkg/libmirror/layouts/tag_resolver.go deleted file mode 100644 index 315585c0..00000000 --- a/pkg/libmirror/layouts/tag_resolver.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "fmt" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/images" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/errorutil" -) - -// TagsResolver is responsible for resolving tag to digest mappings for images. -// It holds a mapping of tags to their corresponding digests. -// Example usage: -// resolver := NewTagsResolver() -// err := resolver.ResolveTagsDigestsForImageLayouts(mirrorCtx, layouts) -type TagsResolver struct { - tagsDigestsMapping map[string]v1.Hash -} - -// NewTagsResolver initializes a new TagsResolver with an empty mapping. -// Example usage: -// resolver := NewTagsResolver() -func NewTagsResolver() *TagsResolver { - return &TagsResolver{tagsDigestsMapping: make(map[string]v1.Hash)} -} - -// TODO no-op must be the default, this should not exist -// NopTagToDigestMappingFunc is a no-operation function that returns nil for any input. -// This is used as a placeholder when no mapping is needed. -// Example usage: -// digest := NopTagToDigestMappingFunc("some-tag") -func NopTagToDigestMappingFunc(_ string) *v1.Hash { - return nil -} - -// ResolveTagsDigestsForImageLayouts resolves the digests for the given image layouts. -// It takes a mirror context and the layouts to resolve. -// Example usage: -// err := resolver.ResolveTagsDigestsForImageLayouts(mirrorCtx, layouts) -func (r *TagsResolver) ResolveTagsDigestsForImageLayouts(mirrorCtx *params.BaseParams, layouts *ImageLayouts) error { - imageSets := make([]map[string]struct{}, 0, 4+len(layouts.Modules)*2) - imageSets = append(imageSets, - layouts.DeckhouseImages, - layouts.ReleaseChannelImages, - layouts.InstallImages, - layouts.InstallStandaloneImages, - ) - - for _, moduleImageLayout := range layouts.Modules { - imageSets = append(imageSets, moduleImageLayout.ModuleImages) - imageSets = append(imageSets, moduleImageLayout.ReleaseImages) - } - - for _, imageSet := range imageSets { - if err := r.ResolveTagsDigestsFromImageSet( - imageSet, - mirrorCtx.RegistryAuth, - mirrorCtx.Insecure, - mirrorCtx.SkipTLSVerification, - ); err != nil { - return err - } - } - - return nil -} - -func (r *TagsResolver) ResolveTagsDigestsFromImageSet( - imageSet map[string]struct{}, - authProvider authn.Authenticator, - insecure, skipTLSVerification bool, -) error { - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authProvider, insecure, skipTLSVerification) - for imageRef := range imageSet { - if images.IsValidImageDigestString(imageRef) { - continue - } - - ref, err := name.ParseReference(imageRef, nameOpts...) - if err != nil { - return fmt.Errorf("parse %q image reference: %w", imageRef, err) - } - desc, err := remote.Head(ref, remoteOpts...) - if err != nil { - if errorutil.IsImageNotFoundError(err) { - continue - } - - return fmt.Errorf("get image descriptor for %q: %w", imageRef, err) - } - - r.tagsDigestsMapping[imageRef] = desc.Digest - } - - return nil -} - -func (r *TagsResolver) GetTagDigest(imageRef string) *v1.Hash { - digest, found := r.tagsDigestsMapping[imageRef] - if !found { - return nil - } - return &digest -} diff --git a/pkg/libmirror/layouts/tag_resolver_test.go b/pkg/libmirror/layouts/tag_resolver_test.go deleted file mode 100644 index 6bd1a08b..00000000 --- a/pkg/libmirror/layouts/tag_resolver_test.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package layouts - -import ( - "io" - "log" - "maps" - "math/rand" - "net/http/httptest" - "strings" - "testing" - - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/registry" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/stretchr/testify/require" - - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" -) - -func TestTagsResolver_GetTagDigest_HappyPath(t *testing.T) { - const imageReference = "registry.deckhouse.io/deckhouse/ee/install:stable" - - want := v1.Hash{ - Algorithm: "sha256", - Hex: "77af4d6b9913e693e8d0b4b294fa62ade6054e6b2f1ffb617ac955dd63fb0182", - } - resolver := &TagsResolver{tagsDigestsMapping: map[string]v1.Hash{ - imageReference: want, - }} - - got := resolver.GetTagDigest(imageReference) - require.Equal(t, want, *got) -} - -func TestTagsResolver_GetTagDigest_UnknownTag(t *testing.T) { - const imageReference = "registry.deckhouse.io/deckhouse/ee/install:stable" - resolver := &TagsResolver{tagsDigestsMapping: map[string]v1.Hash{}} - - got := resolver.GetTagDigest(imageReference) - require.Nil(t, got) -} - -func TestTagsResolver_ResolveTagsDigestsFromImageSet(t *testing.T) { - registryHost, registryRepoPath := setupEmptyRegistryRepo(false) - - taggedImages := map[string]struct{}{ - registryHost + registryRepoPath + ":alpha": {}, - registryHost + registryRepoPath + ":beta": {}, - } - - untaggedImages := map[string]struct{}{ - registryHost + registryRepoPath + "@sha256:77af4d6b9913e693e8d0b4b294fa62ade6054e6b2f1ffb617ac955dd63fb0182": {}, - registryHost + registryRepoPath + "@sha256:09ea463141cd441da365200b2933cf9863141cd441da365200b2933cf98b2f1f": {}, - } - - digests := map[string]string{} - imageSet := map[string]struct{}{} - maps.Copy(imageSet, taggedImages) - maps.Copy(imageSet, untaggedImages) - - for imageRef := range imageSet { - digests[imageRef] = createRandomImageInRegistry(t, imageRef) - } - - r := NewTagsResolver() - err := r.ResolveTagsDigestsFromImageSet(imageSet, nil, true, false) - require.NoError(t, err) - - for imageRef := range taggedImages { - digest := r.GetTagDigest(imageRef) - require.NotNil(t, digest) - require.Equal(t, digests[imageRef], digest.String()) - } -} - -func setupEmptyRegistryRepo(useTLS bool) (host, repoPath string) { - bh := registry.NewInMemoryBlobHandler() - registryHandler := registry.New(registry.WithBlobHandler(bh), registry.Logger(log.New(io.Discard, "", 0))) - - server := httptest.NewUnstartedServer(registryHandler) - if useTLS { - server.StartTLS() - } else { - server.Start() - } - - host = strings.TrimPrefix(server.URL, "http://") - repoPath = "/deckhouse/ee" - if useTLS { - host = strings.TrimPrefix(server.URL, "https://") - } - - return host, repoPath -} - -func createRandomImageInRegistry(t *testing.T, imageRef string) (digest string) { - t.Helper() - - img, err := random.Image(int64(rand.Intn(1024)+1), int64(rand.Intn(5)+1)) - require.NoError(t, err) - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(nil, true, false) - ref, err := name.ParseReference(imageRef, nameOpts...) - require.NoError(t, err) - - err = remote.Write(ref, img, remoteOpts...) - require.NoError(t, err) - - digestHash, err := img.Digest() - require.NoError(t, err) - - return digestHash.String() -} diff --git a/pkg/libmirror/modules/modules.go b/pkg/libmirror/modules/modules.go deleted file mode 100644 index 4a31ac4e..00000000 --- a/pkg/libmirror/modules/modules.go +++ /dev/null @@ -1,338 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package modules - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/fs" - "path" - "strings" - - "github.com/Masterminds/semver/v3" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - - "github.com/deckhouse/deckhouse/pkg/registry" - - "github.com/deckhouse/deckhouse-cli/internal" - "github.com/deckhouse/deckhouse-cli/internal/mirror/releases" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/images" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/errorutil" -) - -type Module struct { - Name string - RegistryPath string - Releases []string -} - -func (m *Module) Versions() []*semver.Version { - versions := make([]*semver.Version, 0) - for _, release := range m.Releases { - v, err := semver.NewVersion(release) - if err == nil { - versions = append(versions, v) - } - } - return versions -} - -func ForRepo(repo string, registryAuth authn.Authenticator, insecure, skipVerifyTLS bool, client registry.Client) ([]Module, error) { - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(registryAuth, insecure, skipVerifyTLS) - result, err := getModulesForRepo(repo, nameOpts, remoteOpts, client) - if err != nil { - return nil, fmt.Errorf("Get external modules: %w", err) - } - - return result, nil -} - -func getModulesForRepo( - repo string, - nameOpts []name.Option, - remoteOpts []remote.Option, - client registry.Client, -) ([]Module, error) { - modules, err := client.ListTags(context.TODO()) - if err != nil { - if errorutil.IsRepoNotFoundError(err) { - return []Module{}, nil - } - return nil, fmt.Errorf("Get Deckhouse modules list from %s: %w", repo, err) - } - - result := make([]Module, 0, len(modules)) - for _, module := range modules { - // if len(flags.ModulesWhitelist) > 0 { - // isWhitelisted := false - - // for _, whitelisted := range flags.ModulesWhitelist { - // if module == whitelisted { - // isWhitelisted = true - // } - // } - - // if !isWhitelisted { - // continue - // } - // } - - m := Module{ - Name: module, - RegistryPath: path.Join(repo, module), - Releases: []string{}, - } - - repo, err := name.NewRepository(path.Join(m.RegistryPath, "release"), nameOpts...) - if err != nil { - return nil, fmt.Errorf("Parsing repo: %v", err) - } - m.Releases, err = remote.List(repo, remoteOpts...) - if err != nil { - return nil, fmt.Errorf("Get releases for module %q: %w", m.RegistryPath, err) - } - result = append(result, m) - } - return result, nil -} - -func FindExternalModuleImages( - params *params.PullParams, - mod *Module, - filter *Filter, - authProvider authn.Authenticator, - insecure, skipVerifyTLS bool, - client registry.Client, -) ( /*moduleImages*/ []string /*moduleImagesWithExternal*/, map[string]struct{} /*releaseImages*/, map[string]struct{}, error) { - logger := params.Logger - - var moduleImages []string - var moduleImagesWithExternal, releaseImages map[string]struct{} - - moduleImagesWithExternal, releaseImages = map[string]struct{}{}, map[string]struct{}{} - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authProvider, insecure, skipVerifyTLS) - - // Check if specific versions are requested (explicit tags) - versionsToMirror := filter.VersionsToMirror(mod) - - // Check if this is the default ">=0.0.0" constraint (no version specified) - isDefaultConstraint := false - if constraint, found := filter.GetConstraint(mod.Name); found { - if sc, ok := constraint.(*SemanticVersionConstraint); ok { - // Detect the default >=0.0.0 constraint - constraintStr := sc.constraint.String() - if constraintStr == ">= 0.0.0" || constraintStr == ">=0.0.0" { - isDefaultConstraint = true - } - } - } - - if len(versionsToMirror) > 0 && !isDefaultConstraint { - // Explicit versions specified (e.g., neuvector@=v1.2.3 or neuvector@~1.2.0) - for _, tag := range versionsToMirror { - moduleImages = append(moduleImages, mod.RegistryPath+":"+tag) - moduleImagesWithExternal[mod.RegistryPath+":"+tag] = struct{}{} - releaseImages[path.Join(mod.RegistryPath, "release")+":"+tag] = struct{}{} - } - } - - if filter.ShouldMirrorReleaseChannels(mod.Name) { - // No explicit versions - use release channels - channelImgs, err := getAvailableReleaseChannelsImagesForModule(mod, nameOpts, remoteOpts) - if err != nil { - return nil, nil, nil, fmt.Errorf("get release channels: %w", err) - } - - for img := range channelImgs { - releaseImages[img] = struct{}{} - } - - channelVers, err := releases.FetchVersionsFromModuleReleaseChannels(channelImgs, authProvider, insecure, skipVerifyTLS, client.WithSegment("release")) - if err != nil { - return nil, nil, nil, fmt.Errorf("fetch channel versions: %w", err) - } - - for _, version := range channelVers { - moduleImages = append(moduleImages, mod.RegistryPath+":"+version) - moduleImagesWithExternal[mod.RegistryPath+":"+version] = struct{}{} - releaseImages[path.Join(mod.RegistryPath, "release")+":"+version] = struct{}{} - } - - versionsToMirror = make([]string, 0) - - for moduleTag := range moduleImagesWithExternal { - tag := strings.SplitN(moduleTag, ":", 2)[1] - - semverTag, err := semver.NewVersion(tag) - if err == nil { - versionsToMirror = append(versionsToMirror, semverTag.Original()) - } - } - } - - // remove duplicate from versionsToMirror - seen := make(map[string]struct{}) - uniqueTags := make([]string, 0, len(versionsToMirror)) - for _, tag := range versionsToMirror { - if _, ok := seen[tag]; !ok { - seen[tag] = struct{}{} - uniqueTags = append(uniqueTags, tag) - } - } - - logger.Debugf("Finding module extra images for %s", mod.Name) - - for _, tag := range uniqueTags { - logger.Debugf("Checking module image %s for extra images", tag) - - img, err := client.GetImage(context.TODO(), tag) - if err != nil { - if errorutil.IsImageNotFoundError(err) { - continue - } - return nil, nil, nil, fmt.Errorf("Get digests for %q version: %w", tag, err) - } - - logger.Debugf("Extracting images_digests.json from %s", tag) - - imagesDigestsJSON, err := images.ExtractFileFromImage(img, "images_digests.json") - switch { - case errors.Is(err, fs.ErrNotExist): - continue - case err != nil: - return nil, nil, nil, fmt.Errorf("Extract digests for %q version: %w", tag, err) - } - - logger.Debugf("Parsing images_digests.json from %s", tag) - - digests := images.ExtractDigestsFromJSONFile(imagesDigestsJSON.Bytes()) - for _, digest := range digests { - extraImageName := mod.RegistryPath + "@" + digest - logger.Debugf("Adding extra image %s to module images", extraImageName) - moduleImagesWithExternal[extraImageName] = struct{}{} - } - } - - return moduleImages, moduleImagesWithExternal, releaseImages, nil -} - -func getAvailableReleaseChannelsImagesForModule(mod *Module, refOpts []name.Option, remoteOpts []remote.Option) (map[string]struct{}, error) { - releasesRegistryPath := path.Join(mod.RegistryPath, "release") - result := make(map[string]struct{}) - for _, imageTag := range []string{ - releasesRegistryPath + ":" + internal.AlphaChannel, - releasesRegistryPath + ":" + internal.BetaChannel, - releasesRegistryPath + ":" + internal.EarlyAccessChannel, - releasesRegistryPath + ":" + internal.StableChannel, - releasesRegistryPath + ":" + internal.RockSolidChannel, - releasesRegistryPath + ":" + internal.LTSChannel, - } { - imageRef, err := name.ParseReference(imageTag, refOpts...) - if err != nil { - return nil, fmt.Errorf("Parse release channel reference: %w", err) - } - - _, err = remote.Head(imageRef, remoteOpts...) - if err != nil { - if errorutil.IsImageNotFoundError(err) { - continue - } - return nil, fmt.Errorf("Check if release channel is present: %w", err) - } - result[imageTag] = struct{}{} - } - - return result, nil -} - -// FindModuleExtraImages extracts extra_images.json from module images and returns extra images map -func FindModuleExtraImages( - params *params.PullParams, - mod *Module, - moduleImages []string, - _ authn.Authenticator, - _, _ bool, - client registry.Client, -) ( /*extraImages*/ map[string]struct{}, error) { - logger := params.Logger - - extraImages := map[string]struct{}{} - - // Try to extract extra_images.json from any available module version - for _, imageTag := range moduleImages { - if strings.Contains(imageTag, "@sha256:") { - logger.Debugf("Skipping digest reference %s for extra_images.json extraction", imageTag) - continue // Skip digest references - } - - logger.Debugf("Checking module image %s for extra_images.json", imageTag) - - img, err := client.GetImage(context.TODO(), strings.Split(imageTag, ":")[1]) - if err != nil { - continue - } - - logger.Debugf("Extracting extra_images.json from %s", imageTag) - - extraImagesJSON, err := images.ExtractFileFromImage(img, "extra_images.json") - if errors.Is(err, fs.ErrNotExist) { - continue // No extra_images.json in this version, try next - } - if err != nil { - return nil, fmt.Errorf("Extract extra_images.json from %q: %w", imageTag, err) - } - - // Parse extra_images.json - it should contain image_name:tag mappings - // Support numeric tag values like {"scanner": 3} - var extraImagesRaw map[string]interface{} - if err := json.Unmarshal(extraImagesJSON.Bytes(), &extraImagesRaw); err != nil { - return nil, fmt.Errorf("Parse extra_images.json from %q: %w", imageTag, err) - } - - // Convert to full registry paths with tags - for imageName, tagValue := range extraImagesRaw { - logger.Debugf("Found extra image %s with tag %v", imageName, tagValue) - - var imageTag string - - switch v := tagValue.(type) { - case float64: - imageTag = fmt.Sprintf("%.0f", v) - case int: - imageTag = fmt.Sprintf("%d", v) - default: - return nil, fmt.Errorf("Invalid tag type for %q in extra_images.json: %T", imageName, tagValue) - } - - fullImagePath := path.Join(mod.RegistryPath, "extra", imageName) + ":" + imageTag - - logger.Debugf("Constructed full extra image path: %s", fullImagePath) - - extraImages[fullImagePath] = struct{}{} - } - - // Continue checking other versions to collect all possible extra images - } - - return extraImages, nil -} From 1e875a757316caac0284a1e05b1c8fdf2abce78d Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Tue, 17 Feb 2026 18:12:39 +0300 Subject: [PATCH 14/14] fix Signed-off-by: Pavel Okhlopkov --- internal/mirror/modules/modules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mirror/modules/modules.go b/internal/mirror/modules/modules.go index 2c793abe..a22d2ba8 100644 --- a/internal/mirror/modules/modules.go +++ b/internal/mirror/modules/modules.go @@ -29,7 +29,7 @@ import ( "strings" "time" - "github.com/Masterminds/semver" + "github.com/Masterminds/semver/v3" dkplog "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/deckhouse/pkg/registry/client"