diff --git a/README.md b/README.md index 1304623..d0fc93a 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ gh devlake deploy local --dir ./devlake gh devlake configure full ``` -After setup, open Grafana at **http://localhost:3002** (admin / admin). DORA and Copilot dashboards will populate after the first sync completes. +After setup, open the URL bundle printed by `gh devlake deploy local`. Local deploys normally use `8080/4000/3002`, and automatically fall back once to `8085/4004/3004` when the default ports are already in use. DORA and Copilot dashboards will populate after the first sync completes. | Service | URL | |---------|-----| -| Grafana | http://localhost:3002 (admin/admin) | -| Config UI | http://localhost:4000 | -| Backend API | http://localhost:8080 | +| Grafana | http://localhost:3002 or http://localhost:3004 (admin/admin) | +| Config UI | http://localhost:4000 or http://localhost:4004 | +| Backend API | http://localhost:8080 or http://localhost:8085 | --- diff --git a/cmd/deploy_azure.go b/cmd/deploy_azure.go index 14e9a12..36983b8 100644 --- a/cmd/deploy_azure.go +++ b/cmd/deploy_azure.go @@ -144,7 +144,9 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { fmt.Println("\nš Checking Azure CLI login...") acct, err := azure.CheckLogin() if err != nil { - fmt.Println(" Not logged in. Running az login...") + // Bounded recovery: Auto-login (single attempt) + fmt.Println(" ā Not logged in") + fmt.Println("\nš§ Recovery: Running az login...") if loginErr := azure.Login(); loginErr != nil { return fmt.Errorf("az login failed: %w", loginErr) } @@ -152,6 +154,7 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("still not logged in after az login: %w", err) } + fmt.Println(" ā Recovery successful") } fmt.Printf(" Logged in as: %s\n", acct.User.Name) @@ -240,9 +243,12 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { fmt.Println("\nšļø Checking MySQL state...") state, err := azure.MySQLState(mysqlName, azureRG) if err == nil && state == "Stopped" { - fmt.Println(" MySQL is stopped. Starting...") + // Bounded recovery: Start stopped MySQL (single attempt) + fmt.Println(" ā MySQL is stopped") + fmt.Println("\nš§ Recovery: Starting MySQL...") if err := azure.MySQLStart(mysqlName, azureRG); err != nil { fmt.Printf(" ā ļø Could not start MySQL: %v\n", err) + fmt.Println(" Continuing deployment ā MySQL may start later") } else { fmt.Println(" Waiting 30s for MySQL...") time.Sleep(30 * time.Second) @@ -258,11 +264,14 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { kvName := fmt.Sprintf("%skv%s", azureBaseName, suffix) found, _ := azure.CheckSoftDeletedKeyVault(kvName) if found { - fmt.Printf("\nš Key Vault %q found in soft-deleted state, purging...\n", kvName) + // Bounded recovery: Purge soft-deleted Key Vault (single attempt) + fmt.Printf("\nš Key Vault conflict detected\n") + fmt.Printf(" Key Vault %q is in soft-deleted state\n", kvName) + fmt.Println("\nš§ Recovery: Purging soft-deleted Key Vault...") if err := azure.PurgeKeyVault(kvName, azureLocation); err != nil { return fmt.Errorf("failed to purge soft-deleted Key Vault %q: %w\nManual fix: az keyvault purge --name %s --location %s", kvName, err, kvName, azureLocation) } - fmt.Println(" ā Key Vault purged") + fmt.Println(" ā Key Vault purged ā deployment can proceed") } // āā Deploy infrastructure āā diff --git a/cmd/deploy_errors.go b/cmd/deploy_errors.go new file mode 100644 index 0000000..75e47fe --- /dev/null +++ b/cmd/deploy_errors.go @@ -0,0 +1,299 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +// DeployErrorClass represents a known failure class during deployment. +type DeployErrorClass string + +const ( + ErrorClassDockerPortConflict DeployErrorClass = "docker_port_conflict" + ErrorClassDockerBindFailed DeployErrorClass = "docker_bind_failed" + ErrorClassAzureAuth DeployErrorClass = "azure_auth" + ErrorClassAzureMySQLStopped DeployErrorClass = "azure_mysql_stopped" + ErrorClassAzureKeyVault DeployErrorClass = "azure_keyvault_softdelete" + ErrorClassUnknown DeployErrorClass = "unknown" +) + +// DeployError represents a classified deployment error with recovery context. +type DeployError struct { + Class DeployErrorClass + OriginalErr error + Port string // For port conflict errors + Container string // For port conflict errors + ComposeFile string // For port conflict errors + Message string // Human-readable classification +} + +// classifyDockerComposeError inspects a docker compose error and returns +// a classified error with recovery context. This covers: +// - "port is already allocated" +// - "Bind for 0.0.0.0:PORT" +// - "ports are not available" / "Ports are not available" +// - "address already in use" +// - "failed programming external connectivity" +func classifyDockerComposeError(err error) *DeployError { + if err == nil { + return nil + } + + errStr := err.Error() + errStrLower := strings.ToLower(errStr) + + // Port conflict patterns (case-insensitive) + portConflictPatterns := []string{ + "port is already allocated", + "bind for", + "ports are not available", + "address already in use", + "failed programming external connectivity", + } + + isPortConflict := false + for _, pattern := range portConflictPatterns { + if strings.Contains(errStrLower, pattern) { + isPortConflict = true + break + } + } + + if !isPortConflict { + return &DeployError{ + Class: ErrorClassUnknown, + OriginalErr: err, + Message: "Docker Compose failed", + } + } + + // Extract port number from various error formats + port := extractPortFromError(errStr) + + result := &DeployError{ + Class: ErrorClassDockerPortConflict, + OriginalErr: err, + Port: port, + Message: "Docker port conflict detected", + } + + // Try to identify owning container + if port != "" { + container, composeFile := findPortOwner(port) + result.Container = container + result.ComposeFile = composeFile + } + + return result +} + +// extractPortFromError extracts the port number from various Docker error formats: +// - "Bind for 0.0.0.0:8080: failed: port is already allocated" +// - "Error response from daemon: Ports are not available: exposing port TCP 0.0.0.0:8080" +// - "bind: address already in use (listening on [::]:8080)" +// - "failed programming external connectivity on endpoint devlake (8080/tcp)" +func extractPortFromError(errStr string) string { + // Pattern 1: "Bind for 0.0.0.0:PORT" (case-insensitive using regexp) + re := regexp.MustCompile(`(?i)bind for 0\.0\.0\.0:(\d+)`) + if matches := re.FindStringSubmatch(errStr); len(matches) > 1 { + port := matches[1] + if isValidPort(port) { + return port + } + } + + // Pattern 2: "exposing port TCP 0.0.0.0:PORT" + if idx := strings.Index(errStr, "0.0.0.0:"); idx != -1 { + rest := errStr[idx+len("0.0.0.0:"):] + if end := strings.IndexAny(rest, " ->\n"); end > 0 { + port := rest[:end] + if isValidPort(port) { + return port + } + } + } + + // Pattern 3: "listening on [::]:PORT" or "[::]PORT" + if idx := strings.Index(errStr, "[::]"); idx != -1 { + rest := errStr[idx+len("[::]"):] + // Skip potential colon separator + if strings.HasPrefix(rest, ":") { + rest = rest[1:] + } + if end := strings.IndexAny(rest, " )\n"); end > 0 { + port := rest[:end] + if isValidPort(port) { + return port + } + } + // If no delimiter found, but there are digits, use them + if len(rest) > 0 { + for i, ch := range rest { + if ch < '0' || ch > '9' { + if i > 0 { + port := rest[:i] + if isValidPort(port) { + return port + } + } + break + } + } + } + } + + // Pattern 4: "(PORT/tcp)" or "(PORT/udp)" in endpoint errors + if idx := strings.Index(errStr, "("); idx != -1 { + rest := errStr[idx+1:] + if end := strings.Index(rest, "/tcp)"); end > 0 { + port := strings.TrimSpace(rest[:end]) + if isValidPort(port) { + return port + } + } + if end := strings.Index(rest, "/udp)"); end > 0 { + port := strings.TrimSpace(rest[:end]) + if isValidPort(port) { + return port + } + } + } + + // Pattern 5: Generic port number extraction (last resort) + // Look for sequences like ":8080" or " 8080 " in the error + for _, candidate := range strings.Fields(errStr) { + // Try splitting by colons + if strings.Contains(candidate, ":") { + parts := strings.Split(candidate, ":") + for _, part := range parts { + part = strings.Trim(part, "(),[]") + if isValidPort(part) { + return part + } + } + } + // Try the field itself (for cases like "[::] 3002") + cleaned := strings.Trim(candidate, "(),[]") + if isValidPort(cleaned) { + return cleaned + } + } + + return "" +} + +// isValidPort checks if a string looks like a valid port number (all digits, 1-65535). +func isValidPort(s string) bool { + if len(s) < 1 || len(s) > 5 { + return false + } + for _, ch := range s { + if ch < '0' || ch > '9' { + return false + } + } + // Parse to int and validate range 1-65535 + port := 0 + for _, ch := range s { + port = port*10 + int(ch-'0') + } + return port >= 1 && port <= 65535 +} + +// findPortOwner queries Docker to find which container is using the specified port. +// Returns (containerName, composeFilePath). +func findPortOwner(port string) (string, string) { + out, err := exec.Command( + "docker", + "ps", + "--filter", "publish="+port, + "--format", "{{.Names}}\t{{.Label \"com.docker.compose.project.config_files\"}}\t{{.Label \"com.docker.compose.project.working_dir\"}}", + ).Output() + + if err != nil || len(strings.TrimSpace(string(out))) == 0 { + return "", "" + } + + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + parts := strings.SplitN(lines[0], "\t", 3) + + containerName := parts[0] + configFiles := "" + workDir := "" + + if len(parts) >= 2 { + configFiles = strings.TrimSpace(parts[1]) + } + if len(parts) == 3 { + workDir = strings.TrimSpace(parts[2]) + } + + // Prefer the exact compose file path Docker recorded + if configFiles != "" { + configFile := strings.Split(configFiles, ";")[0] + configFile = strings.TrimSpace(configFile) + if configFile != "" { + if _, statErr := os.Stat(configFile); statErr == nil { + return containerName, configFile + } + } + } + + // Fallback: assume docker-compose.yml under working_dir + if workDir != "" { + composePath := filepath.Join(workDir, "docker-compose.yml") + if _, statErr := os.Stat(composePath); statErr == nil { + return containerName, composePath + } + } + + return containerName, "" +} + +// printDockerPortConflictError prints a user-friendly error message for port conflicts +// with actionable remediation steps. +// If customHeader is provided, it replaces the default "Port conflict detected" header. +// If nextSteps is provided, it replaces the default "Then re-run: gh devlake deploy local" text. +func printDockerPortConflictError(de *DeployError, customHeader string, nextSteps string) { + // Print header + if customHeader != "" { + fmt.Println(customHeader) + } else { + if de.Port != "" { + fmt.Printf("\nā Port conflict detected: port %s is already in use.\n", de.Port) + } else { + fmt.Println("\nā Port conflict detected: a required port is already in use.") + } + } + + // Print container info and stop commands + if de.Container != "" { + fmt.Printf(" Container holding the port: %s\n", de.Container) + + if de.ComposeFile != "" { + fmt.Println(" Stop it with:") + fmt.Printf(" docker compose -f \"%s\" down\n", de.ComposeFile) + } else { + fmt.Println(" Stop it with:") + fmt.Printf(" docker stop %s\n", de.Container) + } + } else if de.Port != "" { + fmt.Println(" Find what's using it:") + fmt.Println(" docker ps --format \"table {{.Names}}\\t{{.Ports}}\"") + } + + // Print next steps + if nextSteps != "" { + fmt.Println(nextSteps) + } else { + fmt.Println(" Then re-run:") + fmt.Println(" gh devlake deploy local") + } + + fmt.Println("\nš” To clean up partial artifacts:") + fmt.Println(" gh devlake cleanup --local --force") +} diff --git a/cmd/deploy_errors_test.go b/cmd/deploy_errors_test.go new file mode 100644 index 0000000..7e68864 --- /dev/null +++ b/cmd/deploy_errors_test.go @@ -0,0 +1,200 @@ +package cmd + +import ( + "errors" + "testing" +) + +func TestExtractPortFromError(t *testing.T) { + tests := []struct { + name string + errStr string + wantPort string + }{ + { + name: "bind for pattern", + errStr: "Error response from daemon: driver failed: Bind for 0.0.0.0:8080 failed: port is already allocated", + wantPort: "8080", + }, + { + name: "exposing port pattern", + errStr: "Error response from daemon: Ports are not available: exposing port TCP 0.0.0.0:8080", + wantPort: "8080", + }, + { + name: "ipv6 listening pattern with colon", + errStr: "bind: address already in use (listening on [::]:8080)", + wantPort: "8080", + }, + { + name: "ipv6 listening pattern without colon", + errStr: "bind: address already in use (listening on [::] 3002)", + wantPort: "3002", + }, + { + name: "port in error context", + errStr: "failed programming external connectivity on endpoint devlake (8080/tcp)", + wantPort: "8080", + }, + { + name: "alternate port 8085", + errStr: "Error response from daemon: driver failed: Bind for 0.0.0.0:8085 failed: port is already allocated", + wantPort: "8085", + }, + { + name: "grafana port 3002", + errStr: "Bind for 0.0.0.0:3002: failed: port is already allocated", + wantPort: "3002", + }, + { + name: "config-ui port 4000", + errStr: "Ports are not available: exposing port TCP 0.0.0.0:4000", + wantPort: "4000", + }, + { + name: "no port in error", + errStr: "Error response from daemon: some other error", + wantPort: "", + }, + { + name: "empty error", + errStr: "", + wantPort: "", + }, + { + name: "port 0 should be rejected", + errStr: "Error response from daemon: Bind for 0.0.0.0:0 failed", + wantPort: "", + }, + { + name: "port 00000 should be rejected", + errStr: "Error response from daemon: Bind for 0.0.0.0:00000 failed", + wantPort: "", + }, + { + name: "port 65536 should be rejected", + errStr: "Error response from daemon: Bind for 0.0.0.0:65536 failed", + wantPort: "", + }, + { + name: "case insensitive bind for with uppercase", + errStr: "Error response from daemon: BIND FOR 0.0.0.0:8080 failed", + wantPort: "8080", + }, + { + name: "non-ASCII characters before match", + errStr: "ćØć©ć¼: Bind for 0.0.0.0:8080 failed: port is already allocated", + wantPort: "8080", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractPortFromError(tt.errStr) + if got != tt.wantPort { + t.Errorf("extractPortFromError() = %q, want %q", got, tt.wantPort) + } + }) + } +} + +func TestClassifyDockerComposeError(t *testing.T) { + tests := []struct { + name string + err error + wantClass DeployErrorClass + wantPort string + wantNilErr bool + }{ + { + name: "port already allocated", + err: errors.New("Error response from daemon: driver failed: Bind for 0.0.0.0:8080 failed: port is already allocated"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "8080", + }, + { + name: "ports are not available", + err: errors.New("Error response from daemon: Ports are not available: exposing port TCP 0.0.0.0:8080"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "8080", + }, + { + name: "address already in use", + err: errors.New("bind: address already in use (listening on [::]:8080)"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "8080", + }, + { + name: "failed programming external connectivity", + err: errors.New("failed programming external connectivity on endpoint devlake"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "", + }, + { + name: "unknown docker error", + err: errors.New("Error response from daemon: some other error"), + wantClass: ErrorClassUnknown, + wantPort: "", + }, + { + name: "nil error", + err: nil, + wantNilErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := classifyDockerComposeError(tt.err) + + if tt.wantNilErr { + if got != nil { + t.Errorf("classifyDockerComposeError() = %v, want nil", got) + } + return + } + + if got == nil { + t.Fatal("classifyDockerComposeError() returned nil, want non-nil") + } + + if got.Class != tt.wantClass { + t.Errorf("classifyDockerComposeError().Class = %v, want %v", got.Class, tt.wantClass) + } + + if got.Port != tt.wantPort { + t.Errorf("classifyDockerComposeError().Port = %q, want %q", got.Port, tt.wantPort) + } + + if got.OriginalErr != tt.err { + t.Errorf("classifyDockerComposeError().OriginalErr = %v, want %v", got.OriginalErr, tt.err) + } + }) + } +} + +func TestClassifyDockerComposeError_AllPatterns(t *testing.T) { + // Test that all documented port conflict patterns are recognized + patterns := []string{ + "port is already allocated", + "Bind for 0.0.0.0:8080", + "ports are not available", + "address already in use", + "failed programming external connectivity", + } + + for _, pattern := range patterns { + t.Run(pattern, func(t *testing.T) { + err := errors.New("Error: " + pattern) + got := classifyDockerComposeError(err) + + if got == nil { + t.Fatal("classifyDockerComposeError() returned nil, want non-nil") + } + + if got.Class != ErrorClassDockerPortConflict { + t.Errorf("Pattern %q not classified as port conflict, got %v", pattern, got.Class) + } + }) + } +} diff --git a/cmd/deploy_local.go b/cmd/deploy_local.go index 917282c..31f08a0 100644 --- a/cmd/deploy_local.go +++ b/cmd/deploy_local.go @@ -1,10 +1,13 @@ package cmd import ( + "encoding/json" "fmt" "os" "os/exec" "path/filepath" + "regexp" + "strconv" "strings" "time" @@ -45,6 +48,11 @@ Image source (interactive prompt or flags): fork Clone a DevLake repo and build images from source custom Use your own docker-compose.yml already in the target directory +For official and fork deployments, if the default local port bundle +(8080/3002/4000) is already in use, the CLI automatically retries once with +alternate ports (8085/3004/4004). Custom deployments require manual port +conflict resolution. + Example: gh devlake deploy local gh devlake deploy local --version v1.0.2 --dir ./devlake @@ -190,7 +198,11 @@ func runDeployLocal(cmd *cobra.Command, args []string) error { if deployLocalSource == "fork" { services = []string{"mysql", "devlake", "grafana", "config-ui"} } - backendURL, err := startLocalContainers(absDir, buildImages, services...) + + // Allow alternate port bundle for official/fork (not custom) + allowPortFallback := deployLocalSource != "custom" + + backendURL, err := startLocalContainers(absDir, buildImages, allowPortFallback, services...) if err != nil { return err } @@ -212,8 +224,11 @@ func runDeployLocal(cmd *cobra.Command, args []string) error { if !deployLocalQuiet { printBanner("ā DevLake is running!") fmt.Printf("\n Backend API: %s\n", backendURL) - fmt.Println(" Config UI: http://localhost:4000") - fmt.Println(" Grafana: http://localhost:3002 (admin/admin)") + // Infer companion URLs based on compose file ports + composePath := findComposeFile(absDir) + grafanaURL, configUIURL := inferCompanionURLs(backendURL, composePath) + fmt.Printf(" Config UI: %s\n", configUIURL) + fmt.Printf(" Grafana: %s (admin/admin)\n", grafanaURL) fmt.Println("\nTo stop/remove DevLake:") fmt.Printf(" cd \"%s\" && gh devlake cleanup\n", absDir) } @@ -424,10 +439,12 @@ func copyDir(src, dst string) error { // startLocalContainers runs docker compose up -d and polls until DevLake is healthy. // If build is true, images are rebuilt from local Dockerfiles (fork mode). +// If allowPortFallback is true, the function will retry once with alternate ports (8085/3004/4004) +// when a port conflict is detected on the default bundle (8080/3002/4000). // If services are specified, only those services are started (used by fork mode // to avoid starting unnecessary services like postgres/authproxy). // Returns the backend URL on success. -func startLocalContainers(dir string, build bool, services ...string) (string, error) { +func startLocalContainers(dir string, build, allowPortFallback bool, services ...string) (string, error) { absDir, _ := filepath.Abs(dir) if build { fmt.Printf("\nš³ Building and starting containers in %s...\n", absDir) @@ -435,101 +452,138 @@ func startLocalContainers(dir string, build bool, services ...string) (string, e } else { fmt.Printf("\nš³ Starting containers in %s...\n", absDir) } - if err := dockerpkg.ComposeUp(absDir, build, services...); err != nil { - // Give a friendlier error for port conflicts - errStr := err.Error() - if strings.Contains(errStr, "port is already allocated") || strings.Contains(errStr, "Bind for") { - // Extract the port number from the error - port := "" - if idx := strings.Index(errStr, "Bind for 0.0.0.0:"); idx != -1 { - rest := errStr[idx+len("Bind for 0.0.0.0:"):] - if end := strings.IndexAny(rest, " \n"); end > 0 { - port = rest[:end] - } - } - fmt.Println() - if port != "" { - fmt.Printf("ā Port conflict: %s is already in use.\n", port) - } else { - fmt.Println("ā Port conflict: a required port is already in use.") - } + // Attempt 1: Default ports + err := dockerpkg.ComposeUp(absDir, build, services...) + if err == nil { + fmt.Println(" ā Containers starting") + return waitAndDetectBackendURL(absDir) + } - // Ask Docker which container owns the port - conflictCmd := "" - if port != "" { - out, dockerErr := exec.Command( - "docker", - "ps", - "--filter", - "publish="+port, - "--format", - "{{.Names}}\t{{.Label \"com.docker.compose.project.config_files\"}}\t{{.Label \"com.docker.compose.project.working_dir\"}}", - ).Output() - if dockerErr == nil && len(strings.TrimSpace(string(out))) > 0 { - lines := strings.Split(strings.TrimSpace(string(out)), "\n") - // Use the first match - parts := strings.SplitN(lines[0], "\t", 3) - containerName := parts[0] - configFiles := "" - workDir := "" - if len(parts) >= 2 { - configFiles = strings.TrimSpace(parts[1]) - } - if len(parts) == 3 { - workDir = strings.TrimSpace(parts[2]) - } - fmt.Printf(" Container holding the port: %s\n", containerName) - // Prefer the exact compose file path Docker recorded (most reliable). - if configFiles != "" { - configFile := strings.Split(configFiles, ";")[0] - configFile = strings.TrimSpace(configFile) - if configFile != "" { - if _, statErr := os.Stat(configFile); statErr == nil { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker compose -f \"%s\" down\n", configFile) - conflictCmd = fmt.Sprintf("docker compose -f \"%s\" down", configFile) - } else { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker stop %s\n", containerName) - fmt.Printf("\n ā ļø Compose file not found at: %s\n", configFile) - fmt.Println(" (It may have been moved/deleted since the container was created.)") - conflictCmd = "docker stop " + containerName - } - } - } else if workDir != "" { - // Fallback for older Docker versions: assume docker-compose.yml under working_dir. - composePath := filepath.Join(workDir, "docker-compose.yml") - if _, statErr := os.Stat(composePath); statErr == nil { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker compose -f \"%s\" down\n", composePath) - conflictCmd = fmt.Sprintf("docker compose -f \"%s\" down", composePath) - } - } - if conflictCmd == "" { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker stop %s\n", containerName) - conflictCmd = "docker stop " + containerName - } - } - } - if conflictCmd == "" { - fmt.Println("\n Find what's using it:") - fmt.Println(" docker ps --format \"table {{.Names}}\\t{{.Ports}}\"") - } - fmt.Println("\n Then re-run:") - fmt.Println(" gh devlake init") - fmt.Println("\nš” To clean up partial artifacts:") - fmt.Println(" gh devlake cleanup --local --force") - return "", fmt.Errorf("port conflict ā stop the conflicting container and retry") - } + // Classify the error + deployErr := classifyDockerComposeError(err) + if deployErr == nil || deployErr.Class != ErrorClassDockerPortConflict { + // Not a port conflict or unknown error - print general cleanup and fail fmt.Println("\nš” To clean up partial artifacts:") fmt.Println(" gh devlake cleanup --local --force") return "", err } - fmt.Println(" ā Containers starting") - backendURLCandidates := []string{"http://localhost:8080", "http://localhost:8085"} + // Port conflict detected + if !allowPortFallback { + // Custom deployments don't get auto-fallback - print friendly error + printDockerPortConflictError(deployErr, "", "") + return "", fmt.Errorf("port conflict ā stop the conflicting container and retry: %w", err) + } + + // Bounded recovery: Try alternate port bundle once + // Find compose file + composePath := findComposeFile(absDir) + + // Detect which port bundle the compose file is using + bundle := detectPortBundle(composePath) + + switch bundle { + case portBundleAlternate: + // Compose file is already on alternate ports - can't fallback further + // Build custom header with port/container info + header := "\nā Port conflict detected on alternate ports (8085/3004/4004)" + if deployErr.Port != "" { + header += fmt.Sprintf("\n Port %s is in use", deployErr.Port) + if deployErr.Container != "" { + header += fmt.Sprintf(" by container: %s", deployErr.Container) + } + } + nextSteps := " The alternate port bundle is already in use.\n Free ports 8085/3004/4004, then retry deployment." + printDockerPortConflictError(deployErr, header, nextSteps) + return "", fmt.Errorf("port conflict on alternate ports: %w", err) + + case portBundleCustom: + // Custom ports - don't attempt automatic rewrite + // Build custom header with port/container info + header := "\nā Port conflict detected on custom ports" + if deployErr.Port != "" { + header += fmt.Sprintf("\n Port %s is in use", deployErr.Port) + if deployErr.Container != "" { + header += fmt.Sprintf(" by container: %s", deployErr.Container) + } + } + nextSteps := " Edit your docker-compose.yml to use different host ports, or stop the conflicting container." + printDockerPortConflictError(deployErr, header, nextSteps) + return "", fmt.Errorf("port conflict on custom ports: %w", err) + + case portBundleDefault: + // Compose file has default ports - try rewriting to alternate bundle + fmt.Println("\nš§ Port conflict detected on default ports (8080/3002/4000)") + if deployErr.Port != "" { + fmt.Printf(" Port %s is in use", deployErr.Port) + if deployErr.Container != "" { + fmt.Printf(" by container: %s", deployErr.Container) + } + fmt.Println() + } + fmt.Println("\nš Retrying with alternate ports (8085/3004/4004)...") + + if err := rewriteComposePorts(composePath); err != nil { + fmt.Printf(" ā ļø Could not rewrite ports: %v\n", err) + printDockerPortConflictError(deployErr, "", "") + return "", fmt.Errorf("port conflict and failed to apply alternate ports: %w", err) + } + + fmt.Println(" ā Ports updated in compose file") + + // Attempt 2: Retry with alternate ports + fmt.Println(" Starting containers with alternate ports...") + err = dockerpkg.ComposeUp(absDir, build, services...) + if err != nil { + // Second attempt failed - classify again + retryErr := classifyDockerComposeError(err) + if retryErr != nil && retryErr.Class == ErrorClassDockerPortConflict { + // Build header that indicates both bundles failed + header := "\nā Alternate ports are also in use." + nextSteps := " Both default (8080/3002/4000) and alternate (8085/3004/4004) port bundles are occupied.\n Free at least one bundle, then retry deployment." + printDockerPortConflictError(retryErr, header, nextSteps) + } else { + fmt.Println("\nš” To clean up partial artifacts:") + fmt.Println(" gh devlake cleanup --local --force") + } + return "", fmt.Errorf("deployment failed after port fallback: %w", err) + } + + fmt.Println(" ā Containers starting on alternate ports") + return waitAndDetectBackendURL(absDir) + } + + return "", fmt.Errorf("unexpected port bundle detection result") +} + +// findComposeFile returns the path to the active docker compose file in the given directory. +// Checks for docker-compose.yml first, then docker-compose-dev.yml. +func findComposeFile(dir string) string { + composePath := filepath.Join(dir, "docker-compose.yml") + if _, err := os.Stat(composePath); err == nil { + return composePath + } + return filepath.Join(dir, "docker-compose-dev.yml") +} + +// waitAndDetectBackendURL polls the backend URL extracted from the compose file. +// Falls back to probing both 8080 and 8085 if extraction fails. +func waitAndDetectBackendURL(dir string) (string, error) { + composePath := findComposeFile(dir) + + // Try to extract the actual backend port from the compose file + ports := extractServicePorts(composePath, "devlake") + var backendURLCandidates []string + + if backendPort, ok := ports["devlake"]; ok { + // Use the extracted port + backendURLCandidates = []string{fmt.Sprintf("http://localhost:%d", backendPort)} + } else { + // Fall back to probing both default and alternate ports + backendURLCandidates = []string{"http://localhost:8080", "http://localhost:8085"} + } + fmt.Println("\nā³ Waiting for DevLake to be ready...") fmt.Println(" Giving MySQL time to initialize (this takes ~30s on first run)...") time.Sleep(30 * time.Second) @@ -540,3 +594,190 @@ func startLocalContainers(dir string, build bool, services ...string) (string, e } return backendURL, nil } + +// portBundle represents the detected port configuration in a compose file +type portBundle int + +const ( + portBundleDefault portBundle = iota // 8080/3002/4000 + portBundleAlternate // 8085/3004/4004 + portBundleCustom // Other custom ports +) + +// detectPortBundle analyzes a compose file to determine which port bundle it uses. +// Returns: +// - portBundleDefault if compose file contains any of 8080:8080, 3002:3002, or 4000:4000 +// - portBundleAlternate if compose file contains any of 8085:8080, 3004:3002, or 4004:4000 +// - portBundleCustom if compose file has other custom host ports +func detectPortBundle(composePath string) portBundle { + data, err := os.ReadFile(composePath) + if err != nil { + return portBundleDefault // Assume default if we can't read - let rewriteComposePorts surface the I/O error + } + + content := string(data) + + // Use regex to match port mappings as list items (avoiding substring matches like 18080:8080) + // Pattern: start of line, optional whitespace, dash, optional whitespace, optional quotes, port mapping + defaultPatterns := []*regexp.Regexp{ + regexp.MustCompile(`(?m)^\s*-\s*["']?8080:8080["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?3002:3002["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?4000:4000["']?`), + } + for _, re := range defaultPatterns { + if re.MatchString(content) { + return portBundleDefault + } + } + + // Check for alternate port bundle + alternatePatterns := []*regexp.Regexp{ + regexp.MustCompile(`(?m)^\s*-\s*["']?8085:8080["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?3004:3002["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?4004:4000["']?`), + } + for _, re := range alternatePatterns { + if re.MatchString(content) { + return portBundleAlternate + } + } + + // If neither default nor alternate, it's custom + return portBundleCustom +} + +// composeFileHasDefaultPorts checks if a compose file contains the default port bundle (8080/3002/4000). +// Returns true only for the default bundle, and false for alternate or other custom port mappings. +func composeFileHasDefaultPorts(composePath string) bool { + return detectPortBundle(composePath) == portBundleDefault +} + +// rewriteComposePorts rewrites the port mappings in a docker-compose.yml file +// from the default bundle (8080/3002/4000) to the alternate bundle (8085/3004/4004). +// Uses regex with proper boundaries to avoid rewriting custom ports like 18080:8080. +func rewriteComposePorts(composePath string) error { + data, err := os.ReadFile(composePath) + if err != nil { + return fmt.Errorf("reading compose file: %w", err) + } + + content := string(data) + modified := content + + // Port mapping patterns with regex boundaries + // Match: "- 8080:8080" or "- "8080:8080" or "- '8080:8080'" at start of list item + // Avoid: "- 18080:8080" (custom host port that contains 8080) + portReplacements := []struct { + pattern string + replacement string + }{ + // Backend: 8080:8080 -> 8085:8080 + {`(?m)(^\s*-\s*)["']?8080:8080["']?`, `${1}8085:8080`}, + // Grafana: 3002:3002 -> 3004:3002 + {`(?m)(^\s*-\s*)["']?3002:3002["']?`, `${1}3004:3002`}, + // Config UI: 4000:4000 -> 4004:4000 + {`(?m)(^\s*-\s*)["']?4000:4000["']?`, `${1}4004:4000`}, + } + + for _, repl := range portReplacements { + re := regexp.MustCompile(repl.pattern) + modified = re.ReplaceAllString(modified, repl.replacement) + } + + if modified == content { + return fmt.Errorf("no port mappings found to rewrite (expected 8080/3002/4000)") + } + + if err := os.WriteFile(composePath, []byte(modified), 0644); err != nil { + return fmt.Errorf("writing compose file: %w", err) + } + + return nil +} + +// extractServicePorts parses the docker-compose.yml file and extracts host ports for the specified services. +// Returns a map of service name to host port (e.g., "devlake" -> 8080). +// Returns empty map if parsing fails or service not found. +func extractServicePorts(composePath string, serviceNames ...string) map[string]int { + result := make(map[string]int) + + // Use docker compose config to parse the compose file reliably + cmd := exec.Command("docker", "compose", "-f", composePath, "config", "--format", "json") + output, err := cmd.Output() + if err != nil { + // Silently return empty map - caller will fall back to default behavior + return result + } + + var config struct { + Services map[string]struct { + Ports []struct { + Published string `json:"published,omitempty"` + Target int `json:"target,omitempty"` + } `json:"ports,omitempty"` + } `json:"services"` + } + + if err := json.Unmarshal(output, &config); err != nil { + return result + } + + // Extract host ports for requested services + for _, serviceName := range serviceNames { + service, exists := config.Services[serviceName] + if !exists { + continue + } + + // Find the first published port mapping + for _, port := range service.Ports { + if port.Published != "" { + // Published can be a port number as string + if hostPort, err := strconv.Atoi(port.Published); err == nil { + result[serviceName] = hostPort + break + } + } + } + } + + return result +} + +// inferCompanionURLs extracts and returns the actual Grafana and Config UI URLs from the compose file. +// Falls back to inferring from backend URL if extraction fails. +func inferCompanionURLs(backendURL string, composePath string) (grafanaURL, configUIURL string) { + // Try to extract actual ports from compose file + ports := extractServicePorts(composePath, "grafana", "config-ui") + + if grafanaPort, ok := ports["grafana"]; ok { + grafanaURL = fmt.Sprintf("http://localhost:%d", grafanaPort) + } + if configUIPort, ok := ports["config-ui"]; ok { + configUIURL = fmt.Sprintf("http://localhost:%d", configUIPort) + } + + // If extraction succeeded for both, return + if grafanaURL != "" && configUIURL != "" { + return grafanaURL, configUIURL + } + + // Fall back to inference from backend URL + if strings.Contains(backendURL, ":8085") { + if grafanaURL == "" { + grafanaURL = "http://localhost:3004" + } + if configUIURL == "" { + configUIURL = "http://localhost:4004" + } + } else { + if grafanaURL == "" { + grafanaURL = "http://localhost:3002" + } + if configUIURL == "" { + configUIURL = "http://localhost:4000" + } + } + + return grafanaURL, configUIURL +} diff --git a/cmd/deploy_local_test.go b/cmd/deploy_local_test.go index b575b10..087138e 100644 --- a/cmd/deploy_local_test.go +++ b/cmd/deploy_local_test.go @@ -1,6 +1,11 @@ package cmd -import "testing" +import ( + "os" + "path/filepath" + "strings" + "testing" +) func TestRewritePoetryInstallLine_RewritesInstallerLine(t *testing.T) { input := "FROM python:3.9-slim-bookworm\nRUN curl -sSL https://install.python-poetry.org | python3 -\n" @@ -38,3 +43,360 @@ func TestRewritePoetryInstallLine_NoChangeWhenLineMissing(t *testing.T) { t.Fatalf("content changed unexpectedly") } } + +func TestRewriteComposePorts(t *testing.T) { + tests := []struct { + name string + input string + wantContain []string + wantErr bool + }{ + { + name: "standard docker-compose format", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 + grafana: + ports: + - 3002:3002 + config-ui: + ports: + - 4000:4000 +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "quoted port mappings", + input: `version: '3' +services: + devlake: + ports: + - "8080:8080" + grafana: + ports: + - "3002:3002" + config-ui: + ports: + - "4000:4000" +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "single-quoted port mappings", + input: `version: '3' +services: + devlake: + ports: + - '8080:8080' + grafana: + ports: + - '3002:3002' + config-ui: + ports: + - '4000:4000' +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "mixed port formats", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 + grafana: + ports: + - "3002:3002" + config-ui: + ports: + - '4000:4000' +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "no matching ports", + input: `version: '3' +services: + mysql: + ports: + - 3306:3306 +`, + wantErr: true, + }, + { + name: "already rewritten ports (alternate bundle)", + input: `version: '3' +services: + devlake: + ports: + - 8085:8080 + grafana: + ports: + - 3004:3002 + config-ui: + ports: + - 4004:4000 +`, + wantErr: true, // No changes made + }, + { + name: "custom host port should not be rewritten", + input: `version: '3' +services: + devlake: + ports: + - 18080:8080 + grafana: + ports: + - 13002:3002 + config-ui: + ports: + - 14000:4000 +`, + wantErr: true, // No matching ports to rewrite + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temp file + tmpDir := t.TempDir() + composePath := filepath.Join(tmpDir, "docker-compose.yml") + if err := os.WriteFile(composePath, []byte(tt.input), 0644); err != nil { + t.Fatalf("Failed to write test compose file: %v", err) + } + + // Run rewrite + err := rewriteComposePorts(composePath) + + if tt.wantErr { + if err == nil { + t.Errorf("rewriteComposePorts() expected error, got nil") + } + return + } + + if err != nil { + t.Errorf("rewriteComposePorts() unexpected error: %v", err) + return + } + + // Read result + result, err := os.ReadFile(composePath) + if err != nil { + t.Fatalf("Failed to read result: %v", err) + } + resultStr := string(result) + + // Check expected content + for _, want := range tt.wantContain { + if !strings.Contains(resultStr, want) { + t.Errorf("rewriteComposePorts() result missing %q\nResult:\n%s", want, resultStr) + } + } + + // Ensure old ports are gone + oldPorts := []string{"8080:8080", "3002:3002", "4000:4000"} + for _, old := range oldPorts { + if strings.Contains(resultStr, old) { + t.Errorf("rewriteComposePorts() result still contains old port %q", old) + } + } + }) + } +} + +func TestRewriteComposePorts_FileNotFound(t *testing.T) { + err := rewriteComposePorts("/nonexistent/path/docker-compose.yml") + if err == nil { + t.Error("rewriteComposePorts() expected error for nonexistent file, got nil") + } + if !strings.Contains(err.Error(), "reading compose file") { + t.Errorf("rewriteComposePorts() error = %v, want error about reading compose file", err) + } +} + +func TestDetectPortBundle(t *testing.T) { + tests := []struct { + name string + input string + want portBundle + }{ + { + name: "default port bundle", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 + grafana: + ports: + - 3002:3002 + config-ui: + ports: + - 4000:4000 +`, + want: portBundleDefault, + }, + { + name: "alternate port bundle", + input: `version: '3' +services: + devlake: + ports: + - 8085:8080 + grafana: + ports: + - 3004:3002 + config-ui: + ports: + - 4004:4000 +`, + want: portBundleAlternate, + }, + { + name: "custom port bundle", + input: `version: '3' +services: + devlake: + ports: + - 18080:8080 + grafana: + ports: + - 13002:3002 + config-ui: + ports: + - 14000:4000 +`, + want: portBundleCustom, + }, + { + name: "mixed custom and unrelated ports", + input: `version: '3' +services: + mysql: + ports: + - 3306:3306 + devlake: + ports: + - 9090:8080 +`, + want: portBundleCustom, + }, + { + name: "partial default bundle (has at least one default port)", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 +`, + want: portBundleDefault, + }, + { + name: "partial alternate bundle (has at least one alternate port)", + input: `version: '3' +services: + devlake: + ports: + - 8085:8080 +`, + want: portBundleAlternate, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temp file + tmpDir := t.TempDir() + composePath := filepath.Join(tmpDir, "docker-compose.yml") + if err := os.WriteFile(composePath, []byte(tt.input), 0644); err != nil { + t.Fatalf("Failed to write test compose file: %v", err) + } + + got := detectPortBundle(composePath) + if got != tt.want { + t.Errorf("detectPortBundle() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDetectPortBundle_FileNotFound(t *testing.T) { + got := detectPortBundle("/nonexistent/path/docker-compose.yml") + if got != portBundleDefault { + t.Errorf("detectPortBundle() for nonexistent file = %v, want %v (default)", got, portBundleDefault) + } +} + +func TestExtractServicePorts_MissingFile(t *testing.T) { + // Should return empty map for non-existent file + ports := extractServicePorts("/nonexistent/docker-compose.yml", "devlake") + if len(ports) != 0 { + t.Errorf("extractServicePorts() for nonexistent file returned %v, want empty map", ports) + } +} + +func TestNewDeployLocalCmd(t *testing.T) { + cmd := newDeployLocalCmd() + + if cmd.Use != "local" { + t.Errorf("expected Use 'local', got %q", cmd.Use) + } + if cmd.Short != "Deploy DevLake locally via Docker Compose" { + t.Errorf("unexpected Short: %q", cmd.Short) + } + if !strings.Contains(cmd.Long, "alternate ports (8085/3004/4004)") { + t.Errorf("expected Long help to mention alternate port fallback, got: %q", cmd.Long) + } + + for _, flag := range []string{"dir", "version", "source", "repo-url", "start"} { + if cmd.Flags().Lookup(flag) == nil { + t.Errorf("expected --%s flag to be registered", flag) + } + } + + if got := cmd.Flags().Lookup("start").DefValue; got != "true" { + t.Errorf("expected --start default true, got %q", got) + } +} + +func TestInferCompanionURLs_FallbackFromBackendURL(t *testing.T) { + tests := []struct { + name string + backendURL string + wantGrafana string + wantConfigUI string + }{ + { + name: "default bundle", + backendURL: "http://localhost:8080", + wantGrafana: "http://localhost:3002", + wantConfigUI: "http://localhost:4000", + }, + { + name: "alternate bundle", + backendURL: "http://localhost:8085", + wantGrafana: "http://localhost:3004", + wantConfigUI: "http://localhost:4004", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + grafanaURL, configUIURL := inferCompanionURLs(tt.backendURL, "/nonexistent/docker-compose.yml") + if grafanaURL != tt.wantGrafana { + t.Errorf("grafanaURL = %q, want %q", grafanaURL, tt.wantGrafana) + } + if configUIURL != tt.wantConfigUI { + t.Errorf("configUIURL = %q, want %q", configUIURL, tt.wantConfigUI) + } + }) + } +} diff --git a/docs/deploy.md b/docs/deploy.md index 32e4b81..d761c4d 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -4,7 +4,7 @@ Deploy DevLake locally via Docker Compose or to Azure via Bicep. ## deploy local -Downloads the official Apache DevLake Docker Compose files, generates an `ENCRYPTION_SECRET`, and prepares the directory for `docker compose up`. +Sets up DevLake Docker Compose files and starts containers (by default). Supports official Apache releases, forked repositories, or custom compose configurations. ### Usage @@ -18,31 +18,55 @@ gh devlake deploy local [flags] |------|---------|-------------| | `--dir` | `.` | Target directory for Docker Compose files | | `--version` | `latest` | DevLake release version (e.g., `v1.0.2`) | +| `--source` | *(interactive if omitted)* | Image source: `official`, `fork`, or `custom` | +| `--repo-url` | | Repository URL to clone (for `fork` source) | +| `--start` | `true` | Start containers after setup | ### What It Does +Depending on `--source`: + +**official** (default): 1. Fetches the latest release tag from GitHub (or uses `--version`) 2. Downloads `docker-compose.yml` and `env.example` from the Apache DevLake release 3. Renames `env.example` ā `.env` 4. Generates and injects a cryptographic `ENCRYPTION_SECRET` into `.env` 5. Checks that Docker is available +6. Starts containers (unless `--start=false`) + +**fork**: +1. Clones the repository specified by `--repo-url` +2. Builds DevLake images from source +3. Generates `.env` with `ENCRYPTION_SECRET` +4. Checks that Docker is available +5. Starts containers (unless `--start=false`) + +**custom**: +1. Uses the existing `docker-compose.yml` (or `docker-compose-dev.yml`) in the target directory +2. Generates or updates `.env` with `ENCRYPTION_SECRET` if needed +3. Checks that Docker is available +4. Starts containers (unless `--start=false`) ### After Running +By default (`--start=true`), containers start automatically and you can access the endpoints immediately (wait ~2ā3 minutes for all services to initialize). + +To stage files without starting containers, use `--start=false`, then run: + ```bash cd