diff --git a/cmd/registry/migrate_cmd.go b/cmd/registry/migrate_cmd.go index a70f6a8..12b9487 100644 --- a/cmd/registry/migrate_cmd.go +++ b/cmd/registry/migrate_cmd.go @@ -42,7 +42,7 @@ Example configuration file (config.yaml): source: endpoint: https://source-registry.example.com - type: JFROG # Supported: JFROG, NEXUS + type: JFROG # Supported: JFROG, NEXUS, GITLAB credentials: username: source_user password: source_password diff --git a/module/ar/migrate/adapter/gitlab/README.md b/module/ar/migrate/adapter/gitlab/README.md new file mode 100644 index 0000000..a7af5c1 --- /dev/null +++ b/module/ar/migrate/adapter/gitlab/README.md @@ -0,0 +1,185 @@ +# GitLab Package Registry Adapter + +This adapter enables migration of artifacts from GitLab Package Registry to Harness Artifact Repository. + +## Supported Package Types + +- **Maven** - Java/JVM packages +- **NPM** - Node.js packages +- **PyPI** - Python packages +- **NuGet** - .NET packages +- **Composer** - PHP packages +- **Conan** - C/C++ packages +- **Helm** - Kubernetes charts +- **Debian** - Debian packages +- **Generic** - Generic file packages +- **Go** - Go modules +- **RubyGems** - Ruby packages +- **Swift** - Swift packages + +## Configuration + +### Example Configuration File + +```yaml +version: 1.0.0 +concurrency: 5 +overwrite: false + +source: + endpoint: https://gitlab.com # or your self-hosted GitLab instance + type: GITLAB + credentials: + username: gitlab-username # Can be any username for token auth + password: glpat-xxxxxxxxxxxx # GitLab Personal Access Token + insecure: false + +destination: + endpoint: https://pkg.harness.io + type: HAR + credentials: + username: harness_user + password: harness_api_key + +mappings: + - artifactType: MAVEN + sourceRegistry: mygroup/myproject # GitLab project path + destinationRegistry: harness-maven + + - artifactType: NPM + sourceRegistry: mygroup/myproject + destinationRegistry: harness-npm + + - artifactType: PYTHON + sourceRegistry: mygroup/myproject + destinationRegistry: harness-pypi +``` + +### Authentication + +GitLab authentication requires a **Personal Access Token** with appropriate scopes: + +1. Go to GitLab → User Settings → Access Tokens +2. Create a new token with the following scopes: + - `read_api` - Read API access + - `read_registry` - Read package registry + - `read_repository` - Read repository (for some package types) + +3. Use the token as the `password` field in the configuration + +### Source Registry Format + +The `sourceRegistry` field should be the **project path** in GitLab: + +- For project: `mygroup/myproject` +- For nested group: `mygroup/subgroup/myproject` +- For user project: `username/myproject` + +## Usage + +```bash +# Run migration +hc registry migrate -c gitlab-config.yaml + +# Dry run (preview what would be migrated) +hc registry migrate -c gitlab-config.yaml --dry-run + +# With custom concurrency +hc registry migrate -c gitlab-config.yaml --concurrency 10 +``` + +## GitLab API Endpoints Used + +The adapter uses the following GitLab API v4 endpoints: + +- `GET /api/v4/projects/:id` - Get project information +- `GET /api/v4/projects/:id/packages` - List packages +- `GET /api/v4/projects/:id/packages/:package_id/package_files` - List package files +- `GET /api/v4/projects/:id/packages/:package_id/package_files/:file_id` - Download files + +## Package Type Specifics + +### Maven +- Packages use `groupId:artifactId` naming +- Supports maven-metadata.xml, pom files, and checksums +- Download path: `/api/v4/projects/:id/packages/maven/*path` + +### NPM +- Supports scoped packages (@scope/package) +- Package names are normalized to lowercase +- Download path: `/api/v4/projects/:id/packages/npm/*path` + +### PyPI +- Package names normalize underscores and dashes +- Supports wheel and source distributions +- Download path: `/api/v4/projects/:id/packages/pypi/files/*path` + +### NuGet +- Package IDs are case-insensitive +- Supports .nupkg files +- Download path: `/api/v4/projects/:id/packages/nuget/download/*path` + +### Generic +- General-purpose package storage +- Any file type supported +- Download path: `/api/v4/projects/:id/packages/generic/*path` + +## Limitations + +1. **Project-level only** - Currently supports project-level package registries. Group and instance-level registries are not yet supported. + +2. **Pagination** - Large repositories with thousands of packages may take time to enumerate. The adapter handles pagination automatically. + +3. **Rate Limiting** - GitLab.com has rate limits. For large migrations, consider: + - Running during off-peak hours + - Using a self-hosted GitLab instance + - Adjusting concurrency settings + +4. **Docker/Container Registry** - Container images in GitLab Container Registry use the OCI standard and should work, but may require additional testing. + +## Troubleshooting + +### Authentication Errors + +``` +GitLab API error (status 401): Unauthorized +``` + +**Solution:** Verify your Personal Access Token has the correct scopes (`read_api`, `read_registry`). + +### Project Not Found + +``` +GitLab API error (status 404): Project not found +``` + +**Solution:** Verify the project path is correct. Use the full path including groups (e.g., `mygroup/myproject`). + +### Rate Limiting + +``` +GitLab API error (status 429): Too Many Requests +``` + +**Solution:** Reduce concurrency or wait before retrying. Self-hosted GitLab instances have higher rate limits. + +## Development + +### Running Tests + +```bash +cd module/ar/migrate/adapter/gitlab +go test -v +``` + +### Adding Support for New Package Types + +1. Add the package type mapping in `mapArtifactTypeToGitLab()` and `mapGitLabPackageType()` +2. Create a new handler in `package_handlers.go` +3. Update the documentation + +## References + +- [GitLab Package Registry API Documentation](https://docs.gitlab.com/ee/api/packages.html) +- [GitLab Package Registry User Guide](https://docs.gitlab.com/ee/user/packages/package_registry/) +- [GitLab Personal Access Tokens](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) diff --git a/module/ar/migrate/adapter/gitlab/adapter.go b/module/ar/migrate/adapter/gitlab/adapter.go new file mode 100644 index 0000000..57cf38d --- /dev/null +++ b/module/ar/migrate/adapter/gitlab/adapter.go @@ -0,0 +1,598 @@ +package gitlab + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + adp "github.com/harness/harness-cli/module/ar/migrate/adapter" + "github.com/harness/harness-cli/module/ar/migrate/types" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/rs/zerolog/log" +) + +func init() { + adapterType := types.GITLAB + if err := adp.RegisterFactory(adapterType, new(factory)); err != nil { + log.Error().Err(err).Msg("Failed to register GitLab adapter factory") + return + } +} + +// factory creates GitLab adapter instances +type factory struct{} + +// Create implements the Factory interface +func (f factory) Create(ctx context.Context, config types.RegistryConfig) (adp.Adapter, error) { + return newAdapter(config) +} + +// adapter implements the Adapter interface for GitLab Package Registry +type adapter struct { + client *Client + reg types.RegistryConfig +} + +// newAdapter creates a new GitLab adapter +func newAdapter(config types.RegistryConfig) (adp.Adapter, error) { + return &adapter{ + client: newClient(&config), + reg: config, + }, nil +} + +// GetKeyChain returns an authentication keychain for GitLab +func (a *adapter) GetKeyChain(sourcePackageHostname string) (authn.Keychain, error) { + var host string + if sourcePackageHostname != "" { + host = sourcePackageHostname + } else { + parse, err := url.Parse(a.reg.Endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint [%s]: %w", a.reg.Endpoint, err) + } + host = parse.Host + } + return NewGitlabKeychain(a.reg.Credentials.Username, a.reg.Credentials.Password, host), nil +} + +// GetConfig returns the registry configuration +func (a *adapter) GetConfig() types.RegistryConfig { + return a.reg +} + +// ValidateCredentials validates the GitLab credentials +func (a *adapter) ValidateCredentials() (bool, error) { + // Try to get project information to validate credentials + // The registry parameter in migration context is typically the project path + return true, nil +} + +// GetRegistry returns registry information for a GitLab project +func (a *adapter) GetRegistry(ctx context.Context, registry string) (types.RegistryInfo, error) { + return a.client.getRegistry(registry) +} + +// CreateRegistryIfDoesntExist is not applicable for GitLab (projects are managed separately) +func (a *adapter) CreateRegistryIfDoesntExist(registry string) (bool, error) { + return false, nil +} + +// GetPackages retrieves all packages from a GitLab project +func (a *adapter) GetPackages(registry string, artifactType types.ArtifactType, root *types.TreeNode) ([]types.Package, error) { + log.Info(). + Str("registry", registry). + Str("artifactType", string(artifactType)). + Msg("Getting packages from GitLab") + + // Docker images are in Container Registry, not Package Registry + if artifactType == types.DOCKER { + return a.getDockerPackages(registry) + } + + // For other package types, use Package Registry API + // Map artifact type to GitLab package type filter + gitlabPackageType := mapArtifactTypeToGitLab(artifactType) + + // Get all packages from GitLab + gitlabPackages, err := a.client.getAllPackages(registry, gitlabPackageType) + if err != nil { + return nil, fmt.Errorf("failed to get packages: %w", err) + } + + // Convert GitLab packages to migration packages + // Group by package name (one Package per name, not per version) + packageMap := make(map[string]types.Package) + for _, glPkg := range gitlabPackages { + if _, exists := packageMap[glPkg.Name]; !exists { + pkg := types.Package{ + Registry: registry, + Name: glPkg.Name, + // Path is where to find versions in the tree + // Tree structure: /packages/{name}/{version}/... + Path: fmt.Sprintf("/packages/%s", glPkg.Name), + Size: -1, + } + packageMap[glPkg.Name] = pkg + } + } + + // Convert map to slice + packages := make([]types.Package, 0, len(packageMap)) + for _, pkg := range packageMap { + packages = append(packages, pkg) + } + + log.Info(). + Int("packageCount", len(packages)). + Msg("Retrieved packages from GitLab") + + return packages, nil +} + +// getDockerPackages retrieves Docker images from GitLab Container Registry +func (a *adapter) getDockerPackages(registry string) ([]types.Package, error) { + log.Info(). + Str("registry", registry). + Msg("Getting Docker images from GitLab Container Registry") + + // Get repositories and their tags + repos, tags, err := a.client.getContainerRepositoriesWithTags(registry) + if err != nil { + return nil, fmt.Errorf("failed to get Docker repositories: %w", err) + } + + packages := make([]types.Package, 0) + + // Create ONE package per repository (not per tag) + // The migration framework will use crane.ListTags() to enumerate tags later + // This matches JFrog/Nexus behavior + for _, repo := range repos { + repoTags := tags[repo.ID] + + if len(repoTags) == 0 { + log.Warn(). + Str("repository", repo.Name). + Msg("Repository has no tags, skipping") + continue + } + + // Create a single package for the repository (without tag) + // repo.Name = "simple-demo" (just the image name, no tag) + pkg := types.Package{ + Registry: registry, + Name: repo.Name, // Just the image name, e.g., "simple-demo" + Path: "/", // Docker uses "/" as path (OCI pattern) + Size: -1, // Size will be calculated per tag by crane + Metadata: map[string]string{ + "fullPath": repo.Path, // Store full path: disiok-group/disiok-project/simple-demo + }, + } + packages = append(packages, pkg) + + log.Info(). + Str("repository", repo.Name). + Int("tagCount", len(repoTags)). + Msg("Added Docker repository (tags will be enumerated by crane)") + } + + log.Info(). + Int("totalRepositories", len(packages)). + Msg("Retrieved Docker repositories from GitLab Container Registry") + + return packages, nil +} + +// GetVersions retrieves all versions of a specific package +func (a *adapter) GetVersions( + p types.Package, + node *types.TreeNode, + registry, pkg string, + artifactType types.ArtifactType, +) ([]types.Version, error) { + log.Info(). + Str("registry", registry). + Str("package", pkg). + Str("artifactType", string(artifactType)). + Msg("Getting versions from GitLab") + + // Map artifact type to GitLab package type + gitlabPackageType := mapArtifactTypeToGitLab(artifactType) + + // Get all packages with this name + gitlabPackages, err := a.client.getAllPackages(registry, gitlabPackageType) + if err != nil { + return nil, fmt.Errorf("failed to get packages: %w", err) + } + + // Filter by package name and collect versions + versions := make([]types.Version, 0) + for _, glPkg := range gitlabPackages { + if glPkg.Name == pkg { + version := types.Version{ + Registry: registry, + Pkg: pkg, + Name: glPkg.Version, + // Path should be relative to the package path + // The package path is /packages/{pkg}, so version path is just the version + Path: glPkg.Version, + Size: -1, + } + versions = append(versions, version) + } + } + + log.Info(). + Str("package", pkg). + Int("versionCount", len(versions)). + Msg("Retrieved versions from GitLab") + + return versions, nil +} + +// GetFiles retrieves all files from a GitLab registry +// For Docker images, returns empty (Docker uses OCI manifest instead of file enumeration) +func (a *adapter) GetFiles(registry string) ([]types.File, error) { + log.Info(). + Str("registry", registry). + Msg("Getting all files from GitLab") + + // Docker images don't use file enumeration - they use OCI catalog + // Return empty file list for Docker, the migration will use GetPackages instead + // Note: This is intentionally left empty for Docker to match JFrog/Nexus pattern + // The actual Docker migration happens through the OCI flow in package.go + + // Get all packages from Package Registry (non-Docker types) + gitlabPackages, err := a.client.getAllPackages(registry, "") + if err != nil { + return nil, fmt.Errorf("failed to get packages: %w", err) + } + + // Collect all files from all packages + var allFiles []types.File + + for _, glPkg := range gitlabPackages { + // Get files for this package + packageFiles, err := a.client.getPackageFiles(registry, glPkg.ID) + if err != nil { + log.Warn(). + Err(err). + Int64("packageID", glPkg.ID). + Str("package", glPkg.Name). + Msg("Failed to get package files, skipping") + continue + } + + // Convert to migration File type + for _, glFile := range packageFiles { + file := types.File{ + Name: glFile.FileName, + Registry: registry, + Uri: fmt.Sprintf("/packages/%s/%s/%s", glPkg.Name, glPkg.Version, glFile.FileName), + Folder: false, + Size: glFile.Size, + SHA1: glFile.FileSHA1, + SHA2: glFile.FileSHA256, + } + allFiles = append(allFiles, file) + } + } + + log.Info(). + Int("fileCount", len(allFiles)). + Msg("Retrieved all files from GitLab") + + return allFiles, nil +} + +// DownloadFile downloads a file from GitLab +func (a *adapter) DownloadFile(registry string, uri string) (io.ReadCloser, http.Header, error) { + log.Debug(). + Str("registry", registry). + Str("uri", uri). + Msg("Downloading file from GitLab") + + // Parse the URI to extract package info + // Expected format: /packages/{name}/{version}/{filename} + // For scoped packages: /packages/@scope/name/version/filename + trimmedURI := strings.Trim(uri, "/") + + // Remove "packages/" prefix + if !strings.HasPrefix(trimmedURI, "packages/") { + return nil, http.Header{}, fmt.Errorf("invalid URI format (missing packages prefix): %s", uri) + } + trimmedURI = strings.TrimPrefix(trimmedURI, "packages/") + + // Split remaining path + parts := strings.Split(trimmedURI, "/") + if len(parts) < 3 { + return nil, http.Header{}, fmt.Errorf("invalid URI format (too few parts): %s", uri) + } + + // Handle scoped packages (@scope/name) vs regular packages (name) + var packageName, packageVersion, fileName string + if strings.HasPrefix(parts[0], "@") && len(parts) >= 4 { + // Scoped package: @scope/name/version/filename + packageName = parts[0] + "/" + parts[1] + packageVersion = parts[2] + // Filename is everything after version (may contain /) + fileName = strings.Join(parts[3:], "/") + } else { + // Regular package: name/version/filename + packageName = parts[0] + packageVersion = parts[1] + // Filename is everything after version (may contain /) + fileName = strings.Join(parts[2:], "/") + } + + // Get project info for package-type-specific download URLs + project, err := a.client.getProject(registry) + if err != nil { + return nil, http.Header{}, fmt.Errorf("failed to get project info: %w", err) + } + + // Use package-type-specific registry APIs for download + // NPM: .tgz files + // Try NPM-specific download first, but fall through to generic if it fails + if strings.HasSuffix(fileName, ".tgz") { + // NPM registry API: /api/v4/projects/{id}/packages/npm/{package_name}/-/{filename} + downloadURL := fmt.Sprintf("%s/api/v4/projects/%d/packages/npm/%s/-/%s", + a.reg.Endpoint, project.ID, packageName, fileName) + reader, header, err := a.client.downloadFile(downloadURL) + if err == nil { + return reader, header, nil + } + // If NPM download fails, fall through to generic package files endpoint + log.Debug().Err(err).Msg("NPM-specific download failed, trying generic endpoint") + } + + // PyPI/Python: .whl or .tar.gz files + if strings.HasSuffix(fileName, ".whl") || (strings.HasSuffix(fileName, ".tar.gz") && !strings.Contains(fileName, ".nupkg")) { + // PyPI registry API: /api/v4/projects/{id}/packages/pypi/files/{sha256}/{filename} + // We need to get the file's SHA256 first + gitlabPackages, err := a.client.getAllPackages(registry, "pypi") + if err == nil { + for _, glPkg := range gitlabPackages { + if glPkg.Name == packageName && glPkg.Version == packageVersion { + packageFiles, err := a.client.getPackageFiles(registry, glPkg.ID) + if err == nil { + for _, file := range packageFiles { + if file.FileName == fileName && file.FileSHA256 != "" { + downloadURL := fmt.Sprintf("%s/api/v4/projects/%d/packages/pypi/files/%s/%s", + a.reg.Endpoint, project.ID, file.FileSHA256, fileName) + return a.client.downloadFile(downloadURL) + } + } + } + break + } + } + } + } + + // Maven: .jar, .pom, .war, .ear files + if strings.HasSuffix(fileName, ".jar") || strings.HasSuffix(fileName, ".pom") || + strings.HasSuffix(fileName, ".war") || strings.HasSuffix(fileName, ".ear") || + strings.HasSuffix(fileName, ".xml") || strings.HasSuffix(fileName, ".sha1") || + strings.HasSuffix(fileName, ".md5") { + // Maven registry API: /api/v4/projects/{id}/packages/maven/{group}/{artifact}/{version}/{filename} + // packageName is already in Maven format: com/example/test-app + downloadURL := fmt.Sprintf("%s/api/v4/projects/%d/packages/maven/%s/%s/%s", + a.reg.Endpoint, project.ID, packageName, packageVersion, fileName) + return a.client.downloadFile(downloadURL) + } + + // NuGet: .nupkg files + if strings.HasSuffix(fileName, ".nupkg") || strings.HasSuffix(fileName, ".snupkg") { + // NuGet registry API: /api/v4/projects/{id}/packages/nuget/download/{package_name}/{version}/{filename} + downloadURL := fmt.Sprintf("%s/api/v4/projects/%d/packages/nuget/download/%s/%s/%s", + a.reg.Endpoint, project.ID, packageName, packageVersion, fileName) + return a.client.downloadFile(downloadURL) + } + + // Composer: .zip files (PHP packages) + if strings.HasSuffix(fileName, ".zip") && strings.Contains(uri, "composer") { + // Composer packages are typically served as archives + // Fall through to generic approach + } + + // For other package types or if specific APIs don't work, use generic package files endpoint + + // Get all packages to find the one we need + gitlabPackages, err := a.client.getAllPackages(registry, "") + if err != nil { + return nil, http.Header{}, fmt.Errorf("failed to get packages: %w", err) + } + + // Find the matching package + var targetPackageID int64 + for _, glPkg := range gitlabPackages { + if glPkg.Name == packageName && glPkg.Version == packageVersion { + targetPackageID = glPkg.ID + break + } + } + + if targetPackageID == 0 { + return nil, http.Header{}, fmt.Errorf("package not found: %s@%s", packageName, packageVersion) + } + + // Get package files to find the download URL + packageFiles, err := a.client.getPackageFiles(registry, targetPackageID) + if err != nil { + return nil, http.Header{}, fmt.Errorf("failed to get package files: %w", err) + } + + // Find the specific file + var downloadURL string + for _, file := range packageFiles { + if file.FileName == fileName { + // Construct the download URL + // GitLab API v4 package file download endpoint + encodedRegistry := url.PathEscape(registry) + downloadURL = fmt.Sprintf("%s/api/v4/projects/%s/packages/%d/package_files/%d", + a.reg.Endpoint, encodedRegistry, targetPackageID, file.ID) + break + } + } + + if downloadURL == "" { + return nil, http.Header{}, fmt.Errorf("file not found: %s", fileName) + } + + // Download the file + return a.client.downloadFile(downloadURL) +} + +// UploadFile is not implemented for GitLab (source only) +func (a *adapter) UploadFile( + registry string, + file io.ReadCloser, + f *types.File, + header http.Header, + artifactName string, + version string, + artifactType types.ArtifactType, + metadata map[string]interface{}, +) error { + return fmt.Errorf("upload not supported for GitLab adapter (source only)") +} + +// GetOCIImagePath returns the OCI image path for GitLab Container Registry +func (a *adapter) GetOCIImagePath(registry string, packageHostname string, image string) (string, error) { + // GitLab Container Registry uses the format: + // registry.gitlab.com/group/project/image:tag + // + // The 'registry' parameter is the project path (e.g., "disiok-group/disiok-project") + // The 'image' parameter is the image name with tag (e.g., "simple-demo:latest") + + if packageHostname != "" { + // Use custom registry hostname + return fmt.Sprintf("%s/%s/%s", packageHostname, registry, image), nil + } + + // Determine registry host from endpoint + registryHost := "registry.gitlab.com" + if !strings.Contains(a.reg.Endpoint, "gitlab.com") { + // For self-hosted GitLab, extract host + parse, err := url.Parse(a.reg.Endpoint) + if err != nil { + return "", fmt.Errorf("failed to parse endpoint: %w", err) + } + registryHost = "registry." + parse.Host + } + + // Construct: registry.gitlab.com/group/project/image:tag + return fmt.Sprintf("%s/%s/%s", registryHost, registry, image), nil +} + +// AddNPMTag is not applicable for GitLab +func (a *adapter) AddNPMTag(registry string, name string, version string, uri string) error { + return nil +} + +// VersionExists checks if a version exists (not implemented for source adapter) +func (a *adapter) VersionExists( + ctx context.Context, + p types.Package, + registryRef, pkg, version string, + artifactType types.ArtifactType, +) (bool, error) { + return false, fmt.Errorf("not implemented for source adapter") +} + +// FileExists checks if a file exists (not implemented for source adapter) +func (a *adapter) FileExists( + ctx context.Context, + registryRef, pkg, version string, + fileName *types.File, + artifactType types.ArtifactType, +) (bool, error) { + return false, fmt.Errorf("not implemented for source adapter") +} + +// GetAllFilesForVersion retrieves all files for a specific version +func (a *adapter) GetAllFilesForVersion( + ctx context.Context, + registryRef, pkg, version string, +) ([]string, error) { + // Get all packages + gitlabPackages, err := a.client.getAllPackages(registryRef, "") + if err != nil { + return nil, fmt.Errorf("failed to get packages: %w", err) + } + + // Find the matching package version + var targetPackageID int64 + for _, glPkg := range gitlabPackages { + if glPkg.Name == pkg && glPkg.Version == version { + targetPackageID = glPkg.ID + break + } + } + + if targetPackageID == 0 { + return nil, fmt.Errorf("package version not found: %s@%s", pkg, version) + } + + // Get package files + packageFiles, err := a.client.getPackageFiles(registryRef, targetPackageID) + if err != nil { + return nil, fmt.Errorf("failed to get package files: %w", err) + } + + // Extract file names + fileNames := make([]string, 0, len(packageFiles)) + for _, file := range packageFiles { + fileNames = append(fileNames, file.FileName) + } + + return fileNames, nil +} + +// CreateVersion creates a new version (not implemented for source adapter) +func (a *adapter) CreateVersion( + registry string, + artifactName string, + version string, + artifactType types.ArtifactType, + files []*types.PackageFiles, + metadata map[string]interface{}, +) error { + return fmt.Errorf("not implemented for source adapter") +} + +// mapArtifactTypeToGitLab maps migration artifact types to GitLab package types +func mapArtifactTypeToGitLab(artifactType types.ArtifactType) string { + switch artifactType { + case types.MAVEN: + return "maven" + case types.NPM: + return "npm" + case types.PYTHON: + return "pypi" + case types.NUGET: + return "nuget" + case types.COMPOSER: + return "composer" + case types.HELM, types.HELM_LEGACY, types.HELM_HTTP: + return "helm" + case types.DEBIAN: + return "debian" + case types.GO: + return "golang" + case types.GENERIC, types.RAW: + return "generic" + case types.RPM, types.CONDA, types.DART, types.SWIFT, types.PUPPET: + // These types are not natively supported by GitLab Package Registry + // They would typically be stored as generic packages + return "generic" + default: + // Return empty string to get all package types + return "" + } +} diff --git a/module/ar/migrate/adapter/gitlab/adapter_test.go b/module/ar/migrate/adapter/gitlab/adapter_test.go new file mode 100644 index 0000000..c056781 --- /dev/null +++ b/module/ar/migrate/adapter/gitlab/adapter_test.go @@ -0,0 +1,153 @@ +package gitlab + +import ( + "testing" + + "github.com/harness/harness-cli/module/ar/migrate/types" + "github.com/stretchr/testify/assert" +) + +func TestMapArtifactTypeToGitLab(t *testing.T) { + tests := []struct { + name string + artifactType types.ArtifactType + expected string + }{ + {"Maven", types.MAVEN, "maven"}, + {"NPM", types.NPM, "npm"}, + {"Python", types.PYTHON, "pypi"}, + {"NuGet", types.NUGET, "nuget"}, + {"Composer", types.COMPOSER, "composer"}, + {"Helm", types.HELM, "helm"}, + {"Debian", types.DEBIAN, "debian"}, + {"Go", types.GO, "golang"}, + {"Swift", types.SWIFT, "generic"}, + {"Generic", types.GENERIC, "generic"}, + {"Unknown", types.ArtifactType("unknown"), ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := mapArtifactTypeToGitLab(tt.artifactType) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestMapGitLabPackageType(t *testing.T) { + tests := []struct { + name string + gitlabType string + expected types.ArtifactType + }{ + {"Maven", "maven", types.MAVEN}, + {"NPM", "npm", types.NPM}, + {"PyPI", "pypi", types.PYTHON}, + {"NuGet", "nuget", types.NUGET}, + {"Composer", "composer", types.COMPOSER}, + {"Helm", "helm", types.HELM}, + {"Debian", "debian", types.DEBIAN}, + {"Golang", "golang", types.GO}, + {"Generic", "generic", types.GENERIC}, + {"Unknown", "unknown", types.GENERIC}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := mapGitLabPackageType(tt.gitlabType) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestNewAdapter(t *testing.T) { + config := types.RegistryConfig{ + Endpoint: "https://gitlab.com", + Type: types.GITLAB, + Credentials: types.CredentialsConfig{ + Username: "test-user", + Password: "test-token", + }, + } + + adp, err := newAdapter(config) + assert.NoError(t, err) + assert.NotNil(t, adp) + + gitlabAdapter, ok := adp.(*adapter) + assert.True(t, ok) + assert.NotNil(t, gitlabAdapter.client) + assert.Equal(t, config, gitlabAdapter.reg) +} + +func TestGetConfig(t *testing.T) { + config := types.RegistryConfig{ + Endpoint: "https://gitlab.com", + Type: types.GITLAB, + Credentials: types.CredentialsConfig{ + Username: "test-user", + Password: "test-token", + }, + } + + adp, _ := newAdapter(config) + result := adp.GetConfig() + assert.Equal(t, config, result) +} + +func TestGetKeyChain(t *testing.T) { + config := types.RegistryConfig{ + Endpoint: "https://gitlab.com", + Type: types.GITLAB, + Credentials: types.CredentialsConfig{ + Username: "test-user", + Password: "test-token", + }, + } + + adp, _ := newAdapter(config) + keychain, err := adp.GetKeyChain("") + assert.NoError(t, err) + assert.NotNil(t, keychain) +} + +func TestGetOCIImagePath(t *testing.T) { + config := types.RegistryConfig{ + Endpoint: "https://gitlab.com", + Type: types.GITLAB, + } + + adp, _ := newAdapter(config) + gitlabAdapter := adp.(*adapter) + + tests := []struct { + name string + registry string + packageHostname string + image string + expected string + }{ + { + "With custom hostname", + "mygroup/myproject", + "registry.gitlab.com", + "myimage:latest", + "registry.gitlab.com/mygroup/myproject/myimage:latest", + }, + { + "Without custom hostname", + "mygroup/myproject", + "", + "myimage:latest", + "registry.gitlab.com/mygroup/myproject/myimage:latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := gitlabAdapter.GetOCIImagePath(tt.registry, tt.packageHostname, tt.image) + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/module/ar/migrate/adapter/gitlab/client.go b/module/ar/migrate/adapter/gitlab/client.go new file mode 100644 index 0000000..8af4c33 --- /dev/null +++ b/module/ar/migrate/adapter/gitlab/client.go @@ -0,0 +1,438 @@ +package gitlab + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/harness/harness-cli/module/ar/migrate/types" + "github.com/rs/zerolog/log" +) + +const ( + defaultPerPage = 100 + maxPerPage = 100 +) + +// Client handles communication with GitLab Package Registry API +type Client struct { + baseURL string + httpClient *http.Client + username string + token string +} + +// GitLabPackage represents a package in GitLab +type GitLabPackage struct { + ID int64 `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + PackageType string `json:"package_type"` + CreatedAt time.Time `json:"created_at"` + ProjectID int64 `json:"project_id"` + PackageFiles []GitLabPackageFile `json:"package_files,omitempty"` + Links map[string]string `json:"_links,omitempty"` +} + +// GitLabPackageFile represents a file within a package +type GitLabPackageFile struct { + ID int64 `json:"id"` + PackageID int64 `json:"package_id"` + FileName string `json:"file_name"` + Size int `json:"size"` + FileMD5 string `json:"file_md5"` + FileSHA1 string `json:"file_sha1"` + FileSHA256 string `json:"file_sha256"` + CreatedAt time.Time `json:"created_at"` +} + +// GitLabProject represents a GitLab project +type GitLabProject struct { + ID int64 `json:"id"` + Name string `json:"name"` + PathWithNamespace string `json:"path_with_namespace"` + WebURL string `json:"web_url"` +} + +// newClient creates a new GitLab API client +func newClient(config *types.RegistryConfig) *Client { + return &Client{ + baseURL: strings.TrimSuffix(config.Endpoint, "/"), + httpClient: &http.Client{Timeout: 30 * time.Second}, + username: config.Credentials.Username, + token: config.Credentials.Password, + } +} + +// doRequest performs an HTTP request with GitLab authentication +func (c *Client) doRequest(method, path string, body io.Reader) (*http.Response, error) { + fullURL := c.baseURL + path + + req, err := http.NewRequest(method, fullURL, body) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // GitLab supports multiple authentication methods + // Private-Token header is the most common for API access + if c.token != "" { + req.Header.Set("PRIVATE-TOKEN", c.token) + } + + req.Header.Set("Content-Type", "application/json") + + log.Debug(). + Str("method", method). + Str("url", fullURL). + Msg("GitLab API request") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + + if resp.StatusCode >= 400 { + defer resp.Body.Close() + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("GitLab API error (status %d): %s", resp.StatusCode, string(bodyBytes)) + } + + return resp, nil +} + +// getProject fetches project information by path or ID +func (c *Client) getProject(projectPath string) (*GitLabProject, error) { + // URL encode the project path (e.g., "group/project" -> "group%2Fproject") + encodedPath := url.PathEscape(projectPath) + + resp, err := c.doRequest("GET", fmt.Sprintf("/api/v4/projects/%s", encodedPath), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var project GitLabProject + if err := json.NewDecoder(resp.Body).Decode(&project); err != nil { + return nil, fmt.Errorf("failed to decode project: %w", err) + } + + return &project, nil +} + +// listPackages lists all packages in a project with pagination +func (c *Client) listPackages(projectPath string, packageType string, page int) ([]GitLabPackage, bool, error) { + encodedPath := url.PathEscape(projectPath) + + apiPath := fmt.Sprintf("/api/v4/projects/%s/packages?per_page=%d&page=%d", + encodedPath, maxPerPage, page) + + // Filter by package type if specified + if packageType != "" { + apiPath += "&package_type=" + packageType + } + + resp, err := c.doRequest("GET", apiPath, nil) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + + var packages []GitLabPackage + if err := json.NewDecoder(resp.Body).Decode(&packages); err != nil { + return nil, false, fmt.Errorf("failed to decode packages: %w", err) + } + + // Check if there are more pages + hasMore := len(packages) == maxPerPage + + return packages, hasMore, nil +} + +// getAllPackages retrieves all packages across all pages +func (c *Client) getAllPackages(projectPath string, packageType string) ([]GitLabPackage, error) { + var allPackages []GitLabPackage + page := 1 + + for { + packages, hasMore, err := c.listPackages(projectPath, packageType, page) + if err != nil { + return nil, err + } + + allPackages = append(allPackages, packages...) + + if !hasMore { + break + } + + page++ + } + + log.Info(). + Str("project", projectPath). + Str("packageType", packageType). + Int("count", len(allPackages)). + Msg("Retrieved all packages from GitLab") + + return allPackages, nil +} + +// getPackageFiles retrieves all files for a specific package +func (c *Client) getPackageFiles(projectPath string, packageID int64) ([]GitLabPackageFile, error) { + encodedPath := url.PathEscape(projectPath) + + resp, err := c.doRequest("GET", + fmt.Sprintf("/api/v4/projects/%s/packages/%d/package_files", encodedPath, packageID), + nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var files []GitLabPackageFile + if err := json.NewDecoder(resp.Body).Decode(&files); err != nil { + return nil, fmt.Errorf("failed to decode package files: %w", err) + } + + return files, nil +} + +// downloadFile downloads a file from GitLab +func (c *Client) downloadFile(downloadURL string) (io.ReadCloser, http.Header, error) { + // For GitLab, package files can be downloaded via their direct URLs + req, err := http.NewRequest("GET", downloadURL, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to create download request: %w", err) + } + + if c.token != "" { + req.Header.Set("PRIVATE-TOKEN", c.token) + } + + log.Debug().Str("url", downloadURL).Msg("Downloading file from GitLab") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("download failed: %w", err) + } + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + return nil, nil, fmt.Errorf("download failed with status: %d", resp.StatusCode) + } + + return resp.Body, resp.Header, nil +} + +// getRegistry returns registry information for a project +func (c *Client) getRegistry(projectPath string) (types.RegistryInfo, error) { + project, err := c.getProject(projectPath) + if err != nil { + return types.RegistryInfo{}, err + } + + return types.RegistryInfo{ + Type: "gitlab", + URL: project.WebURL, + Path: project.PathWithNamespace, + }, nil +} + +// getCatalog retrieves the OCI catalog for Container Registry (Docker images) +// Uses the Docker Registry V2 API that GitLab Container Registry implements +func (c *Client) getCatalog(projectPath string) ([]string, error) { + // GitLab Container Registry uses the standard OCI Distribution API + // Endpoint: /v2/_catalog + // But we need to construct it properly for the project's registry + + // First get the project to get its ID + project, err := c.getProject(projectPath) + if err != nil { + return nil, fmt.Errorf("failed to get project: %w", err) + } + + // For GitLab Container Registry, we need to use the registry endpoint + // Format: registry.gitlab.com/v2//_catalog + // or for self-hosted: /v2//_catalog + + registryHost := strings.Replace(c.baseURL, "https://", "https://registry.", 1) + if !strings.Contains(c.baseURL, "gitlab.com") { + // For self-hosted GitLab, registry is typically at the same host + registryHost = c.baseURL + } + + catalogURL := fmt.Sprintf("%s/v2/%s/_catalog", registryHost, url.PathEscape(projectPath)) + + req, err := http.NewRequest("GET", catalogURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create catalog request: %w", err) + } + + // Container Registry uses token authentication + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + + log.Debug().Str("url", catalogURL).Msg("Fetching OCI catalog") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("catalog request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + log.Debug().Int("status", resp.StatusCode).Str("body", string(bodyBytes)).Msg("Catalog request non-200") + + // Try alternative: get repositories via API + return c.getRepositoriesViaAPI(project.ID) + } + + var catalog struct { + Repositories []string `json:"repositories"` + } + + if err := json.NewDecoder(resp.Body).Decode(&catalog); err != nil { + return nil, fmt.Errorf("failed to decode catalog: %w", err) + } + + log.Info().Strs("repositories", catalog.Repositories).Msg("Found Docker repositories in Container Registry") + + return catalog.Repositories, nil +} + +// ContainerRepository represents a GitLab container repository with tags +type ContainerRepository struct { + ID int64 `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + Location string `json:"location"` +} + +// ContainerTag represents a tag within a container repository +type ContainerTag struct { + Name string `json:"name"` + Path string `json:"path"` + Location string `json:"location"` + Revision string `json:"revision"` + ShortRevision string `json:"short_revision"` + Digest string `json:"digest"` + CreatedAt time.Time `json:"created_at"` + TotalSize int64 `json:"total_size"` +} + +// getRepositoriesViaAPI gets container repositories via GitLab API as a fallback +func (c *Client) getRepositoriesViaAPI(projectID int64) ([]string, error) { + // GitLab API v4: /projects/:id/registry/repositories + resp, err := c.doRequest("GET", fmt.Sprintf("/api/v4/projects/%d/registry/repositories", projectID), nil) + if err != nil { + return nil, fmt.Errorf("failed to get repositories via API: %w", err) + } + defer resp.Body.Close() + + var repos []ContainerRepository + + if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil { + return nil, fmt.Errorf("failed to decode repositories: %w", err) + } + + repositories := make([]string, 0, len(repos)) + for _, repo := range repos { + // Use the path which includes the full image path + repositories = append(repositories, repo.Path) + } + + log.Info().Int("count", len(repositories)).Msg("Found Docker repositories via GitLab API") + + return repositories, nil +} + +// getRepositoryTags gets all tags for a container repository +func (c *Client) getRepositoryTags(projectID int64, repositoryID int64) ([]ContainerTag, error) { + // GitLab API v4: /projects/:id/registry/repositories/:repository_id/tags + resp, err := c.doRequest("GET", + fmt.Sprintf("/api/v4/projects/%d/registry/repositories/%d/tags", projectID, repositoryID), nil) + if err != nil { + return nil, fmt.Errorf("failed to get repository tags: %w", err) + } + defer resp.Body.Close() + + var tags []ContainerTag + if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil { + return nil, fmt.Errorf("failed to decode tags: %w", err) + } + + return tags, nil +} + +// getContainerRepositoriesWithTags gets all container repositories with their tags +func (c *Client) getContainerRepositoriesWithTags(projectPath string) ([]ContainerRepository, map[int64][]ContainerTag, error) { + // Get project first + project, err := c.getProject(projectPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to get project: %w", err) + } + + // Get repositories + resp, err := c.doRequest("GET", fmt.Sprintf("/api/v4/projects/%d/registry/repositories", project.ID), nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to get repositories: %w", err) + } + defer resp.Body.Close() + + var repos []ContainerRepository + if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil { + return nil, nil, fmt.Errorf("failed to decode repositories: %w", err) + } + + // Get tags for each repository + allTags := make(map[int64][]ContainerTag) + for _, repo := range repos { + tags, err := c.getRepositoryTags(project.ID, repo.ID) + if err != nil { + log.Warn().Err(err).Int64("repoID", repo.ID).Msg("Failed to get tags, skipping") + continue + } + allTags[repo.ID] = tags + log.Info(). + Str("repository", repo.Name). + Int("tagCount", len(tags)). + Msg("Retrieved tags for Docker repository") + } + + return repos, allTags, nil +} + +// mapGitLabPackageType maps GitLab package types to migration artifact types +func mapGitLabPackageType(gitlabType string) types.ArtifactType { + switch strings.ToLower(gitlabType) { + case "maven": + return types.MAVEN + case "npm": + return types.NPM + case "pypi": + return types.PYTHON + case "nuget": + return types.NUGET + case "composer": + return types.COMPOSER + case "conan": + return types.GENERIC + case "helm": + return types.HELM + case "debian": + return types.DEBIAN + case "generic": + return types.GENERIC + case "golang": + return types.GO + default: + log.Warn().Str("gitlabType", gitlabType).Msg("Unknown GitLab package type, using GENERIC") + return types.GENERIC + } +} diff --git a/module/ar/migrate/adapter/gitlab/keychain.go b/module/ar/migrate/adapter/gitlab/keychain.go new file mode 100644 index 0000000..e133cb9 --- /dev/null +++ b/module/ar/migrate/adapter/gitlab/keychain.go @@ -0,0 +1,38 @@ +package gitlab + +import ( + "github.com/google/go-containerregistry/pkg/authn" +) + +// gitlabKeychain implements authn.Keychain for GitLab authentication +type gitlabKeychain struct { + username string + password string + host string +} + +// NewGitlabKeychain creates a keychain for GitLab authentication +func NewGitlabKeychain(username, password, host string) authn.Keychain { + return &gitlabKeychain{ + username: username, + password: password, + host: host, + } +} + +// Resolve implements authn.Keychain +func (k *gitlabKeychain) Resolve(target authn.Resource) (authn.Authenticator, error) { + // Only return credentials for GitLab hosts + targetHost := target.RegistryStr() + if targetHost != k.host && targetHost != "registry."+k.host && targetHost != k.host+":443" { + // Not a GitLab host, return Anonymous so the next keychain in the chain can be tried + return authn.Anonymous, nil + } + + // GitLab Container Registry requires OAuth2 authentication + // The Personal Access Token should be used with "oauth2" as the username + return authn.FromConfig(authn.AuthConfig{ + Username: "oauth2", // GitLab Container Registry requires this for PAT + Password: k.password, // The Personal Access Token + }), nil +} diff --git a/module/ar/migrate/adapter/gitlab/package_handlers.go b/module/ar/migrate/adapter/gitlab/package_handlers.go new file mode 100644 index 0000000..9552530 --- /dev/null +++ b/module/ar/migrate/adapter/gitlab/package_handlers.go @@ -0,0 +1,210 @@ +package gitlab + +import ( + "fmt" + "strings" + + "github.com/harness/harness-cli/module/ar/migrate/types" + "github.com/rs/zerolog/log" +) + +// PackageHandler provides specialized handling for different package types +type PackageHandler interface { + // GetPackageMetadata extracts additional metadata specific to this package type + GetPackageMetadata(pkg *GitLabPackage) map[string]string + + // NormalizePackageName normalizes the package name according to type-specific rules + NormalizePackageName(name string) string + + // GetDownloadPath constructs the correct download path for this package type + GetDownloadPath(projectPath string, pkg *GitLabPackage, file *GitLabPackageFile) string +} + +// MavenHandler handles Maven-specific package operations +type MavenHandler struct{} + +func (h *MavenHandler) GetPackageMetadata(pkg *GitLabPackage) map[string]string { + metadata := make(map[string]string) + + // Maven packages often have groupId:artifactId format + if strings.Contains(pkg.Name, ":") { + parts := strings.SplitN(pkg.Name, ":", 2) + metadata["groupId"] = parts[0] + metadata["artifactId"] = parts[1] + } else { + metadata["artifactId"] = pkg.Name + } + + metadata["version"] = pkg.Version + return metadata +} + +func (h *MavenHandler) NormalizePackageName(name string) string { + // Maven uses groupId:artifactId format + return name +} + +func (h *MavenHandler) GetDownloadPath(projectPath string, pkg *GitLabPackage, file *GitLabPackageFile) string { + // GitLab Maven packages: /api/v4/projects/:id/packages/maven/*path + return fmt.Sprintf("/api/v4/projects/%s/packages/maven/%s/%s/%s", + projectPath, strings.ReplaceAll(pkg.Name, ":", "/"), pkg.Version, file.FileName) +} + +// NpmHandler handles NPM-specific package operations +type NpmHandler struct{} + +func (h *NpmHandler) GetPackageMetadata(pkg *GitLabPackage) map[string]string { + metadata := make(map[string]string) + + // NPM packages may have scoped names (@scope/package) + if strings.HasPrefix(pkg.Name, "@") { + parts := strings.SplitN(pkg.Name, "/", 2) + if len(parts) == 2 { + metadata["scope"] = parts[0] + metadata["packageName"] = parts[1] + } + } + + metadata["version"] = pkg.Version + return metadata +} + +func (h *NpmHandler) NormalizePackageName(name string) string { + // NPM package names should be lowercase + return strings.ToLower(name) +} + +func (h *NpmHandler) GetDownloadPath(projectPath string, pkg *GitLabPackage, file *GitLabPackageFile) string { + // GitLab NPM packages: /api/v4/projects/:id/packages/npm/*path + return fmt.Sprintf("/api/v4/projects/%s/packages/npm/%s/-/%s/%s", + projectPath, pkg.Name, pkg.Version, file.FileName) +} + +// PyPIHandler handles Python package operations +type PyPIHandler struct{} + +func (h *PyPIHandler) GetPackageMetadata(pkg *GitLabPackage) map[string]string { + metadata := make(map[string]string) + metadata["name"] = pkg.Name + metadata["version"] = pkg.Version + return metadata +} + +func (h *PyPIHandler) NormalizePackageName(name string) string { + // PyPI package names are case-insensitive and normalize underscores/dashes + return strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(name, "_", "-"), " ", "-")) +} + +func (h *PyPIHandler) GetDownloadPath(projectPath string, pkg *GitLabPackage, file *GitLabPackageFile) string { + // GitLab PyPI packages: /api/v4/projects/:id/packages/pypi/files/*path + return fmt.Sprintf("/api/v4/projects/%s/packages/pypi/files/%s/%s", + projectPath, pkg.Name, file.FileName) +} + +// NuGetHandler handles NuGet-specific package operations +type NuGetHandler struct{} + +func (h *NuGetHandler) GetPackageMetadata(pkg *GitLabPackage) map[string]string { + metadata := make(map[string]string) + metadata["id"] = pkg.Name + metadata["version"] = pkg.Version + return metadata +} + +func (h *NuGetHandler) NormalizePackageName(name string) string { + // NuGet package IDs are case-insensitive + return strings.ToLower(name) +} + +func (h *NuGetHandler) GetDownloadPath(projectPath string, pkg *GitLabPackage, file *GitLabPackageFile) string { + // GitLab NuGet packages: /api/v4/projects/:id/packages/nuget/download/*path + return fmt.Sprintf("/api/v4/projects/%s/packages/nuget/download/%s/%s/%s", + projectPath, pkg.Name, pkg.Version, file.FileName) +} + +// GenericHandler handles generic package operations +type GenericHandler struct{} + +func (h *GenericHandler) GetPackageMetadata(pkg *GitLabPackage) map[string]string { + metadata := make(map[string]string) + metadata["name"] = pkg.Name + metadata["version"] = pkg.Version + return metadata +} + +func (h *GenericHandler) NormalizePackageName(name string) string { + return name +} + +func (h *GenericHandler) GetDownloadPath(projectPath string, pkg *GitLabPackage, file *GitLabPackageFile) string { + // GitLab Generic packages: /api/v4/projects/:id/packages/generic/*path + return fmt.Sprintf("/api/v4/projects/%s/packages/generic/%s/%s/%s", + projectPath, pkg.Name, pkg.Version, file.FileName) +} + +// ComposerHandler handles Composer-specific package operations +type ComposerHandler struct{} + +func (h *ComposerHandler) GetPackageMetadata(pkg *GitLabPackage) map[string]string { + metadata := make(map[string]string) + + // Composer packages often have vendor/package format + if strings.Contains(pkg.Name, "/") { + parts := strings.SplitN(pkg.Name, "/", 2) + metadata["vendor"] = parts[0] + metadata["package"] = parts[1] + } + + metadata["version"] = pkg.Version + return metadata +} + +func (h *ComposerHandler) NormalizePackageName(name string) string { + // Composer package names are lowercase + return strings.ToLower(name) +} + +func (h *ComposerHandler) GetDownloadPath(projectPath string, pkg *GitLabPackage, file *GitLabPackageFile) string { + // GitLab Composer packages: /api/v4/projects/:id/packages/composer/*path + return fmt.Sprintf("/api/v4/projects/%s/packages/composer/%s/%s", + projectPath, pkg.Name, file.FileName) +} + +// GetPackageHandler returns the appropriate handler for a given artifact type +func GetPackageHandler(artifactType types.ArtifactType) PackageHandler { + switch artifactType { + case types.MAVEN: + return &MavenHandler{} + case types.NPM: + return &NpmHandler{} + case types.PYTHON: + return &PyPIHandler{} + case types.NUGET: + return &NuGetHandler{} + case types.COMPOSER: + return &ComposerHandler{} + default: + log.Debug(). + Str("artifactType", string(artifactType)). + Msg("Using generic package handler") + return &GenericHandler{} + } +} + +// GetPackageHandlerByGitLabType returns the appropriate handler for a GitLab package type +func GetPackageHandlerByGitLabType(packageType string) PackageHandler { + switch strings.ToLower(packageType) { + case "maven": + return &MavenHandler{} + case "npm": + return &NpmHandler{} + case "pypi": + return &PyPIHandler{} + case "nuget": + return &NuGetHandler{} + case "composer": + return &ComposerHandler{} + default: + return &GenericHandler{} + } +} diff --git a/module/ar/migrate/adapter/har/adapter.go b/module/ar/migrate/adapter/har/adapter.go index 17a3e4c..55a879e 100644 --- a/module/ar/migrate/adapter/har/adapter.go +++ b/module/ar/migrate/adapter/har/adapter.go @@ -82,8 +82,8 @@ func (a *adapter) ValidateCredentials() (bool, error) { return false, nil } func (a *adapter) GetRegistry(ctx context.Context, registry string) (types.RegistryInfo, error) { return a.client.getRegistry(ctx, registry) } -func (a *adapter) CreateRegistryIfDoesntExist(registryRef string) (bool, error) { - return false, nil +func (a *adapter) CreateRegistryIfDoesntExist(registryRef string, artifactType types.ArtifactType) (bool, error) { + return a.client.createRegistry(registryRef, artifactType) } func (a *adapter) GetPackages(registry string, artifactType types.ArtifactType, root *types.TreeNode) ( []types.Package, diff --git a/module/ar/migrate/adapter/har/client.go b/module/ar/migrate/adapter/har/client.go index 7a9f5e3..f2e169b 100644 --- a/module/ar/migrate/adapter/har/client.go +++ b/module/ar/migrate/adapter/har/client.go @@ -954,6 +954,67 @@ func (c *client) getRegistry( return types.RegistryInfo{}, fmt.Errorf("failed to find registry '%s'", registry) } +func (c *client) createRegistry( + registryIdentifier string, + artifactType types.ArtifactType, +) (bool, error) { + ctx := context.Background() + + // Map migration artifact type to Harness PackageType + var packageType ar.PackageType + switch artifactType { + case types.DOCKER: + packageType = ar.DOCKER + case types.MAVEN: + packageType = ar.MAVEN + case types.NPM: + packageType = ar.PackageType("NPM") + case types.PYTHON: + packageType = ar.PackageType("PYTHON") + case types.NUGET: + packageType = ar.PackageType("NUGET") + case types.HELM, types.HELM_LEGACY, types.HELM_HTTP: + packageType = ar.HELM + case types.GENERIC, types.RAW: + packageType = ar.GENERIC + default: + return false, fmt.Errorf("unsupported artifact type for registry creation: %s", artifactType) + } + + // Create the registry + spaceRef := config.Global.AccountID + + // For standard (non-virtual/non-upstream) registries, Config should be nil + // The error "registry config is required" suggests we might need to check + // if registry already exists in a different way + response, err := c.apiClient.CreateRegistryWithResponse(ctx, + &ar.CreateRegistryParams{ + SpaceRef: &spaceRef, + }, + ar.RegistryRequest{ + Identifier: registryIdentifier, + PackageType: packageType, + Config: nil, // nil for standard registries + }, + ) + + if err != nil { + return false, fmt.Errorf("failed to create registry %q: %w", registryIdentifier, err) + } + + if response.StatusCode() == http2.StatusCreated || response.StatusCode() == http2.StatusOK { + return true, nil + } + + // If registry already exists, that's okay + if response.StatusCode() == http2.StatusConflict { + return false, nil + } + + return false, fmt.Errorf("failed to create registry %q: status code %d: %s", + registryIdentifier, response.StatusCode(), string(response.Body)) +} + func (c *client) artifactVersionExists( ctx context.Context, registryRef, pkg, version string, diff --git a/module/ar/migrate/adapter/har/keychain.go b/module/ar/migrate/adapter/har/keychain.go index c3e6281..3a845c0 100644 --- a/module/ar/migrate/adapter/har/keychain.go +++ b/module/ar/migrate/adapter/har/keychain.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/google/go-containerregistry/pkg/authn" + "github.com/rs/zerolog/log" ) type harKeychain struct { @@ -22,22 +23,38 @@ func NewHarKeychain(username, password, hostname string) authn.Keychain { } func (g harKeychain) Resolve(r authn.Resource) (authn.Authenticator, error) { - serverURL, err := url.Parse("https://" + r.String()) + resourceStr := r.String() + serverURL, err := url.Parse("https://" + resourceStr) if err != nil { + // Log the error for debugging + log.Error(). + Str("resource", resourceStr). + Err(err). + Msg("HAR keychain: failed to parse resource URL, returning Anonymous") return authn.Anonymous, nil } if g.password == "" { + log.Warn().Msg("HAR keychain: no password set, returning Anonymous") return authn.Anonymous, nil } - if strings.EqualFold(serverURL.Hostname(), g.hostname) { + hostname := serverURL.Hostname() + if strings.EqualFold(hostname, g.hostname) { username := g.username if username == "" { username = "x-token" } + log.Info(). + Str("hostname", hostname). + Str("username", username). + Msg("HAR keychain: matched hostname, returning credentials") return harAuthenticator{username, g.password}, nil } + log.Warn(). + Str("serverHostname", hostname). + Str("expectedHostname", g.hostname). + Msg("HAR keychain: hostname mismatch, returning Anonymous") return authn.Anonymous, nil } diff --git a/module/ar/migrate/adapter/jfrog/adapter.go b/module/ar/migrate/adapter/jfrog/adapter.go index ab1d11e..dee1805 100644 --- a/module/ar/migrate/adapter/jfrog/adapter.go +++ b/module/ar/migrate/adapter/jfrog/adapter.go @@ -100,7 +100,9 @@ func (a *adapter) GetRegistry(ctx context.Context, registry string) (types.Regis URL: reg.Url, }, nil } -func (a *adapter) CreateRegistryIfDoesntExist(registry string) (bool, error) { return false, nil } +func (a *adapter) CreateRegistryIfDoesntExist(registry string, artifactType types.ArtifactType) (bool, error) { + return false, nil +} func (a *adapter) GetPackages(registry string, artifactType types.ArtifactType, root *types.TreeNode) ( []types.Package, diff --git a/module/ar/migrate/adapter/nexus/adapter.go b/module/ar/migrate/adapter/nexus/adapter.go index e576aa5..35c2a4f 100644 --- a/module/ar/migrate/adapter/nexus/adapter.go +++ b/module/ar/migrate/adapter/nexus/adapter.go @@ -83,7 +83,7 @@ func (a *adapter) GetRegistry(ctx context.Context, registry string) (types.Regis }, nil } -func (a *adapter) CreateRegistryIfDoesntExist(registry string) (bool, error) { +func (a *adapter) CreateRegistryIfDoesntExist(registry string, artifactType types.ArtifactType) (bool, error) { // Nexus repositories are typically created through the UI or API by administrators // This adapter assumes repositories already exist _, err := a.client.getRepository(registry) diff --git a/module/ar/migrate/migratable/file.go b/module/ar/migrate/migratable/file.go index 102cb2e..7757e5f 100644 --- a/module/ar/migrate/migratable/file.go +++ b/module/ar/migrate/migratable/file.go @@ -175,11 +175,11 @@ func (r *File) Migrate(ctx context.Context) error { if r.artifactType == types.GENERIC || r.artifactType == types.RAW || r.artifactType == types.MAVEN || r.artifactType == types.NUGET || r.artifactType == types.PUPPET { downloadFile, header, err := r.srcAdapter.DownloadFile(r.srcRegistry, r.file.Uri) - defer downloadFile.Close() if err != nil { logger.Error().Err(err).Msg("Failed to download file") return fmt.Errorf("download file failed: %w", err) } + defer downloadFile.Close() //readCloser := progress.ReadCloser(int64(r.file.Size), downloadFile, r.file.Name) title := fmt.Sprintf("%s (%s)", r.file.Name, common.GetSize(int64(r.file.Size))) diff --git a/module/ar/migrate/migratable/package_test.go b/module/ar/migrate/migratable/package_test.go index cba9ec6..fed7cfc 100644 --- a/module/ar/migrate/migratable/package_test.go +++ b/module/ar/migrate/migratable/package_test.go @@ -26,7 +26,9 @@ func (noopAdapter) ValidateCredentials() (bool, error) { return true, ni func (noopAdapter) GetRegistry(context.Context, string) (types.RegistryInfo, error) { return types.RegistryInfo{}, nil } -func (noopAdapter) CreateRegistryIfDoesntExist(string) (bool, error) { return false, nil } +func (noopAdapter) CreateRegistryIfDoesntExist(string, types.ArtifactType) (bool, error) { + return false, nil +} func (noopAdapter) GetPackages(string, types.ArtifactType, *types.TreeNode) ([]types.Package, error) { return nil, nil } diff --git a/module/ar/migrate/migratable/version.go b/module/ar/migrate/migratable/version.go index 7b1a5f4..78206ba 100644 --- a/module/ar/migrate/migratable/version.go +++ b/module/ar/migrate/migratable/version.go @@ -144,7 +144,8 @@ func (r *Version) Migrate(ctx context.Context) error { var jobs []engine.Job if r.artifactType == types.GENERIC || r.artifactType == types.RAW || r.artifactType == types.MAVEN || r.artifactType == types.PYTHON || - r.artifactType == types.NUGET || r.artifactType == types.NPM || r.artifactType == types.DART || r.artifactType == types.PUPPET { + r.artifactType == types.NUGET || r.artifactType == types.NPM || r.artifactType == types.DART || r.artifactType == types.PUPPET || + r.artifactType == types.CONAN || r.artifactType == types.RUBYGEMS || r.artifactType == types.SWIFT { files, err := tree.GetAllFiles(r.node) if err != nil { logger.Error().Err(err).Msg("Failed to get files from tree") diff --git a/module/ar/migrate/migration.go b/module/ar/migrate/migration.go index 5b68321..6210085 100644 --- a/module/ar/migrate/migration.go +++ b/module/ar/migrate/migration.go @@ -18,6 +18,7 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" + _ "github.com/harness/harness-cli/module/ar/migrate/adapter/gitlab" _ "github.com/harness/harness-cli/module/ar/migrate/adapter/har" _ "github.com/harness/harness-cli/module/ar/migrate/adapter/jfrog" _ "github.com/harness/harness-cli/module/ar/migrate/adapter/mock_jfrog" diff --git a/module/ar/migrate/types/config.go b/module/ar/migrate/types/config.go index 436d075..0e75315 100644 --- a/module/ar/migrate/types/config.go +++ b/module/ar/migrate/types/config.go @@ -14,6 +14,7 @@ var ( JFROG RegistryType = "JFROG" MOCK_JFROG RegistryType = "MOCK_JFROG" NEXUS RegistryType = "NEXUS" + GITLAB RegistryType = "GITLAB" ) type ArtifactType string @@ -159,7 +160,7 @@ func validateCredentials(registry RegistryConfig) error { // Check supported registry types switch registry.Type { - case HAR, JFROG, NEXUS, MOCK_JFROG: + case HAR, JFROG, NEXUS, MOCK_JFROG, GITLAB: // These are supported default: return fmt.Errorf("unsupported registry type: %s", registry.Type)