From dda55769743c08dc76a1af592f69350b7d0e0ac5 Mon Sep 17 00:00:00 2001 From: Eddie Wassef Date: Mon, 26 Jan 2026 17:17:57 -0600 Subject: [PATCH 1/2] Update kube version and improve hosts.toml handling Bumps the default Kubernetes API version to 1.32 in CreateClusterCommand. Hosts.toml is now written to a temporary file to ensure accessibility for Docker, replacing the previous static path usage. --- .claude/settings.local.json | 12 +++++++++++- cli/src/Vdk/Commands/CreateClusterCommand.cs | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 03d39c0..78beb71 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -2,7 +2,17 @@ "permissions": { "allow": [ "Bash(docker stop:*)", - "Bash(docker rm:*)" + "Bash(docker rm:*)", + "Bash(docker run:*)", + "Bash(MSYS_NO_PATHCONV=1 docker run:*)", + "Bash(MSYS_NO_PATHCONV=1 docker exec:*)", + "Bash(for node in idp-worker idp-worker2)", + "Bash(do:*)", + "Bash(echo:*)", + "Bash(done)", + "Bash(for node in idp-control-plane idp-worker idp-worker2)", + "Bash(kubectl --context kind-idp get pods:*)", + "Bash(kubectl --context kind-idp delete pod:*)" ] } } diff --git a/cli/src/Vdk/Commands/CreateClusterCommand.cs b/cli/src/Vdk/Commands/CreateClusterCommand.cs index 034cc61..ed62808 100644 --- a/cli/src/Vdk/Commands/CreateClusterCommand.cs +++ b/cli/src/Vdk/Commands/CreateClusterCommand.cs @@ -51,7 +51,7 @@ public CreateClusterCommand( controlNodes.Aliases.Add("-c"); var workers = new Option("--Workers") { DefaultValueFactory = _ => Defaults.WorkerNodes, Description = "The number of worker nodes in the cluster." }; workers.Aliases.Add("-w"); - var kubeVersion = new Option("--KubeVersion") { DefaultValueFactory = _ => "1.29", Description = "The kubernetes api version." }; + var kubeVersion = new Option("--KubeVersion") { DefaultValueFactory = _ => "1.32", Description = "The kubernetes api version." }; kubeVersion.Aliases.Add("-k"); Options.Add(nameOption); @@ -111,6 +111,16 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla return; } + // Write hosts.toml to a temp file for containerd registry config + // This ensures the file is accessible to Docker regardless of working directory + var hostsTomlContent = """ + server = "http://host.docker.internal:5000" + [host."http://host.docker.internal:5000"] + capabilities = ["pull", "resolve"] + """; + var hostsTomlPath = _fileSystem.Path.Combine(_fileSystem.Path.GetTempPath(), $"hosts-{Guid.NewGuid()}.toml"); + await _fileSystem.File.WriteAllTextAsync(hostsTomlPath, hostsTomlContent); + var cluster = new KindCluster(); for (int index = 0; index < controlPlaneNodes; index++) { @@ -127,7 +137,7 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla { new() { - HostPath = _fileSystem.FileInfo.New("ConfigMounts/hosts.toml").FullName, + HostPath = hostsTomlPath, ContainerPath = "/etc/containerd/certs.d/hub.dev-k8s.cloud/hosts.toml" } }; @@ -144,7 +154,7 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla { new() { - HostPath = _fileSystem.FileInfo.New("ConfigMounts/hosts.toml").FullName, + HostPath = hostsTomlPath, ContainerPath = "/etc/containerd/certs.d/hub.dev-k8s.cloud/hosts.toml" } } From 313afbce251bb9038b82e9dfe7b019ff0dbcc94e Mon Sep 17 00:00:00 2001 From: Eddie Wassef Date: Tue, 27 Jan 2026 07:54:56 -0600 Subject: [PATCH 2/2] Add wait for Flux kustomizations after bootstrap Introduces a WaitForKustomizations method to FluxClient and IFluxClient, and calls it in CreateClusterCommand after bootstrapping Flux. This ensures all Flux kustomizations are reconciled before proceeding with reverse proxy configuration, improving cluster setup reliability. --- cli/src/Vdk/Commands/CreateClusterCommand.cs | 4 + cli/src/Vdk/Services/FluxClient.cs | 88 +++++++++++++++++++- cli/src/Vdk/Services/IFluxClient.cs | 1 + 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/cli/src/Vdk/Commands/CreateClusterCommand.cs b/cli/src/Vdk/Commands/CreateClusterCommand.cs index ed62808..08d185f 100644 --- a/cli/src/Vdk/Commands/CreateClusterCommand.cs +++ b/cli/src/Vdk/Commands/CreateClusterCommand.cs @@ -204,6 +204,10 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla } _flux.Bootstrap(name.ToLower(), "./clusters/default", branch: "main"); + + // Wait for all Flux kustomizations to reconcile before configuring the reverse proxy + _flux.WaitForKustomizations(name.ToLower()); + try { _reverseProxy.UpsertCluster(name.ToLower(), masterNode.ExtraPortMappings.First().HostPort, diff --git a/cli/src/Vdk/Services/FluxClient.cs b/cli/src/Vdk/Services/FluxClient.cs index 2abf812..c39f486 100644 --- a/cli/src/Vdk/Services/FluxClient.cs +++ b/cli/src/Vdk/Services/FluxClient.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Text.Json; using k8s; using k8s.Autorest; using k8s.Models; @@ -114,6 +115,91 @@ public void Bootstrap(string clusterName, string path, string branch = DefaultBr } _console.WriteLine("Flux bootstrap complete."); - + + } + + public bool WaitForKustomizations(string clusterName, int maxAttempts = 60, int delaySeconds = 5) + { + _console.WriteLine("Waiting for Flux kustomizations to reconcile..."); + + for (int attempt = 0; attempt < maxAttempts; attempt++) + { + try + { + var result = _client(clusterName).ApiClient.CustomObjects + .ListNamespacedCustomObject( + "kustomize.toolkit.fluxcd.io", "v1", + "flux-system", "kustomizations"); + + var json = JsonSerializer.Serialize(result); + using var doc = JsonDocument.Parse(json); + var items = doc.RootElement.GetProperty("items"); + + if (items.GetArrayLength() == 0) + { + if (attempt % 5 == 0) + _console.WriteLine(" No kustomizations found yet. Waiting..."); + Thread.Sleep(delaySeconds * 1000); + continue; + } + + int total = items.GetArrayLength(); + int readyCount = 0; + + foreach (var item in items.EnumerateArray()) + { + var name = item.GetProperty("metadata").GetProperty("name").GetString(); + bool isReady = false; + + if (item.TryGetProperty("status", out var status) && + status.TryGetProperty("conditions", out var conditions)) + { + foreach (var condition in conditions.EnumerateArray()) + { + if (condition.GetProperty("type").GetString() == "Ready") + { + var condStatus = condition.GetProperty("status").GetString(); + if (condStatus == "True") + { + isReady = true; + } + else if (attempt % 5 == 0) + { + var reason = condition.TryGetProperty("reason", out var r) + ? r.GetString() : "Unknown"; + var message = condition.TryGetProperty("message", out var m) + ? m.GetString() : ""; + _console.WriteLine( + $" Kustomization '{name}' not ready: {reason} - {message}"); + } + break; + } + } + } + + if (isReady) readyCount++; + } + + if (attempt % 5 == 0 || readyCount == total) + _console.WriteLine($" Kustomizations ready: {readyCount}/{total}"); + + if (readyCount == total) + { + _console.WriteLine("All Flux kustomizations are ready."); + return true; + } + } + catch (Exception ex) + { + if (attempt % 5 == 0) + _console.WriteLine($" Error checking kustomizations: {ex.Message}. Retrying..."); + } + + Thread.Sleep(delaySeconds * 1000); + } + + _console.WriteWarning( + "Timed out waiting for Flux kustomizations to reconcile. Proceeding anyway..."); + return false; } } \ No newline at end of file diff --git a/cli/src/Vdk/Services/IFluxClient.cs b/cli/src/Vdk/Services/IFluxClient.cs index 107874e..e607d7e 100644 --- a/cli/src/Vdk/Services/IFluxClient.cs +++ b/cli/src/Vdk/Services/IFluxClient.cs @@ -3,4 +3,5 @@ namespace Vdk.Services; public interface IFluxClient { void Bootstrap(string clusterName, string path, string branch = FluxClient.DefaultBranch); + bool WaitForKustomizations(string clusterName, int maxAttempts = 60, int delaySeconds = 5); } \ No newline at end of file