diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 03d39c0..78beb71 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -2,7 +2,17 @@ "permissions": { "allow": [ "Bash(docker stop:*)", - "Bash(docker rm:*)" + "Bash(docker rm:*)", + "Bash(docker run:*)", + "Bash(MSYS_NO_PATHCONV=1 docker run:*)", + "Bash(MSYS_NO_PATHCONV=1 docker exec:*)", + "Bash(for node in idp-worker idp-worker2)", + "Bash(do:*)", + "Bash(echo:*)", + "Bash(done)", + "Bash(for node in idp-control-plane idp-worker idp-worker2)", + "Bash(kubectl --context kind-idp get pods:*)", + "Bash(kubectl --context kind-idp delete pod:*)" ] } } diff --git a/cli/src/Vdk/Commands/CreateClusterCommand.cs b/cli/src/Vdk/Commands/CreateClusterCommand.cs index 034cc61..08d185f 100644 --- a/cli/src/Vdk/Commands/CreateClusterCommand.cs +++ b/cli/src/Vdk/Commands/CreateClusterCommand.cs @@ -51,7 +51,7 @@ public CreateClusterCommand( controlNodes.Aliases.Add("-c"); var workers = new Option("--Workers") { DefaultValueFactory = _ => Defaults.WorkerNodes, Description = "The number of worker nodes in the cluster." }; workers.Aliases.Add("-w"); - var kubeVersion = new Option("--KubeVersion") { DefaultValueFactory = _ => "1.29", Description = "The kubernetes api version." }; + var kubeVersion = new Option("--KubeVersion") { DefaultValueFactory = _ => "1.32", Description = "The kubernetes api version." }; kubeVersion.Aliases.Add("-k"); Options.Add(nameOption); @@ -111,6 +111,16 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla return; } + // Write hosts.toml to a temp file for containerd registry config + // This ensures the file is accessible to Docker regardless of working directory + var hostsTomlContent = """ + server = "http://host.docker.internal:5000" + [host."http://host.docker.internal:5000"] + capabilities = ["pull", "resolve"] + """; + var hostsTomlPath = _fileSystem.Path.Combine(_fileSystem.Path.GetTempPath(), $"hosts-{Guid.NewGuid()}.toml"); + await _fileSystem.File.WriteAllTextAsync(hostsTomlPath, hostsTomlContent); + var cluster = new KindCluster(); for (int index = 0; index < controlPlaneNodes; index++) { @@ -127,7 +137,7 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla { new() { - HostPath = _fileSystem.FileInfo.New("ConfigMounts/hosts.toml").FullName, + HostPath = hostsTomlPath, ContainerPath = "/etc/containerd/certs.d/hub.dev-k8s.cloud/hosts.toml" } }; @@ -144,7 +154,7 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla { new() { - HostPath = _fileSystem.FileInfo.New("ConfigMounts/hosts.toml").FullName, + HostPath = hostsTomlPath, ContainerPath = "/etc/containerd/certs.d/hub.dev-k8s.cloud/hosts.toml" } } @@ -194,6 +204,10 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla } _flux.Bootstrap(name.ToLower(), "./clusters/default", branch: "main"); + + // Wait for all Flux kustomizations to reconcile before configuring the reverse proxy + _flux.WaitForKustomizations(name.ToLower()); + try { _reverseProxy.UpsertCluster(name.ToLower(), masterNode.ExtraPortMappings.First().HostPort, diff --git a/cli/src/Vdk/Services/FluxClient.cs b/cli/src/Vdk/Services/FluxClient.cs index 2abf812..c39f486 100644 --- a/cli/src/Vdk/Services/FluxClient.cs +++ b/cli/src/Vdk/Services/FluxClient.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Text.Json; using k8s; using k8s.Autorest; using k8s.Models; @@ -114,6 +115,91 @@ public void Bootstrap(string clusterName, string path, string branch = DefaultBr } _console.WriteLine("Flux bootstrap complete."); - + + } + + public bool WaitForKustomizations(string clusterName, int maxAttempts = 60, int delaySeconds = 5) + { + _console.WriteLine("Waiting for Flux kustomizations to reconcile..."); + + for (int attempt = 0; attempt < maxAttempts; attempt++) + { + try + { + var result = _client(clusterName).ApiClient.CustomObjects + .ListNamespacedCustomObject( + "kustomize.toolkit.fluxcd.io", "v1", + "flux-system", "kustomizations"); + + var json = JsonSerializer.Serialize(result); + using var doc = JsonDocument.Parse(json); + var items = doc.RootElement.GetProperty("items"); + + if (items.GetArrayLength() == 0) + { + if (attempt % 5 == 0) + _console.WriteLine(" No kustomizations found yet. Waiting..."); + Thread.Sleep(delaySeconds * 1000); + continue; + } + + int total = items.GetArrayLength(); + int readyCount = 0; + + foreach (var item in items.EnumerateArray()) + { + var name = item.GetProperty("metadata").GetProperty("name").GetString(); + bool isReady = false; + + if (item.TryGetProperty("status", out var status) && + status.TryGetProperty("conditions", out var conditions)) + { + foreach (var condition in conditions.EnumerateArray()) + { + if (condition.GetProperty("type").GetString() == "Ready") + { + var condStatus = condition.GetProperty("status").GetString(); + if (condStatus == "True") + { + isReady = true; + } + else if (attempt % 5 == 0) + { + var reason = condition.TryGetProperty("reason", out var r) + ? r.GetString() : "Unknown"; + var message = condition.TryGetProperty("message", out var m) + ? m.GetString() : ""; + _console.WriteLine( + $" Kustomization '{name}' not ready: {reason} - {message}"); + } + break; + } + } + } + + if (isReady) readyCount++; + } + + if (attempt % 5 == 0 || readyCount == total) + _console.WriteLine($" Kustomizations ready: {readyCount}/{total}"); + + if (readyCount == total) + { + _console.WriteLine("All Flux kustomizations are ready."); + return true; + } + } + catch (Exception ex) + { + if (attempt % 5 == 0) + _console.WriteLine($" Error checking kustomizations: {ex.Message}. Retrying..."); + } + + Thread.Sleep(delaySeconds * 1000); + } + + _console.WriteWarning( + "Timed out waiting for Flux kustomizations to reconcile. Proceeding anyway..."); + return false; } } \ No newline at end of file diff --git a/cli/src/Vdk/Services/IFluxClient.cs b/cli/src/Vdk/Services/IFluxClient.cs index 107874e..e607d7e 100644 --- a/cli/src/Vdk/Services/IFluxClient.cs +++ b/cli/src/Vdk/Services/IFluxClient.cs @@ -3,4 +3,5 @@ namespace Vdk.Services; public interface IFluxClient { void Bootstrap(string clusterName, string path, string branch = FluxClient.DefaultBranch); + bool WaitForKustomizations(string clusterName, int maxAttempts = 60, int delaySeconds = 5); } \ No newline at end of file