From f77639c42965fc124ecac1ea298dba4abd67c349 Mon Sep 17 00:00:00 2001 From: Eddie Wassef Date: Sun, 22 Jun 2025 13:47:32 -0500 Subject: [PATCH 1/2] Adding CoreDNS rewrite This will allow the internal services to communicate against the external DNS without leaving the cluster and having issues with the DNS pointing to 127.0.0.1 --- cli/src/Vdk/Services/ReverseProxyClient.cs | 125 ++++++++++++++- .../Vdk.Tests/ReverseProxyClientTests.cs | 148 +++++++++++++++++- 2 files changed, 264 insertions(+), 9 deletions(-) diff --git a/cli/src/Vdk/Services/ReverseProxyClient.cs b/cli/src/Vdk/Services/ReverseProxyClient.cs index 747d36d..c4076d2 100644 --- a/cli/src/Vdk/Services/ReverseProxyClient.cs +++ b/cli/src/Vdk/Services/ReverseProxyClient.cs @@ -13,13 +13,11 @@ internal class ReverseProxyClient : IReverseProxyClient private readonly IDockerEngine _docker; private readonly Func _client; private readonly IConsole _console; - - + private static readonly string NginxConf = Path.Combine("vega.conf"); - + private readonly IKindClient _kind; - // ReverseProxyHostPort is 443 by default, unless REVERSE_PROXY_HOST_PORT is set as an env var private int ReverseProxyHostPort = GetEnvironmentVariableAsInt("REVERSE_PROXY_HOST_PORT", 443); @@ -147,6 +145,15 @@ public void Delete() } public void UpsertCluster(string clusterName, int targetPortHttps, int targetPortHttp, bool reload = true) + { + PatchNginxConfig(clusterName, targetPortHttps); + if (CreateTlsSecret(clusterName)) return; + PatchCoreDns(clusterName); + if (reload) + ReloadConfigs(); + } + + private void PatchNginxConfig(string clusterName, int targetPortHttps) { // create a new server block in the nginx conf pointing to the target port listening on the https://clusterName.dev-k8s.cloud domain // reload the nginx configuration @@ -179,7 +186,112 @@ public void UpsertCluster(string clusterName, int targetPortHttps, int targetPor _console.WriteWarning($"Error clearing cluster configuration ({NginxConf}): {e.Message}"); _console.WriteWarning("Please check the configuration and try again."); } + } + private bool PatchCoreDns(string clusterName) + { + // let's wait for and find the ingress controller service. + V1Service? ingressService = null; + var attempts = 0; + do + { + // check up to 10 times , waiting 5 seconds each time + ingressService = _client(clusterName).Get("ingress-nginx-controller", "ingress-nginx"); + if (ingressService == null) + { + _console.WriteLine("Waiting for ingress-nginx-controller service to be available..."); + Thread.Sleep(5000); + attempts++; + } + else + { + _console.WriteLine("Ingress-nginx-controller service found."); + break; + } + } + while (ingressService == null && attempts < 6); + if (ingressService == null) + { + _console.WriteError("Ingress-nginx-controller service not found. Please check the configuration and try again."); + return false; + } + var rewriteString = $" rewrite name {clusterName}.dev-k8s.cloud {ingressService.Name()}.{ingressService.Namespace()}.svc.cluster.local"; + // now read the CoreDNS configmap and add the clusterName.dev-k8s.cloud entry to it + var corednsConfigMap = _client(clusterName).Get("coredns", "kube-system"); + if (corednsConfigMap == null) + { + _console.WriteError("CoreDNS configmap not found. Please check the configuration and try again."); + return false; + } + _console.WriteLine("Patching CoreDNS configmap with new cluster entry."); + var corednsData = corednsConfigMap.Data; + if (corednsData == null) + { + corednsData = new Dictionary(); + } + // extract the corefile from the configmap data + if (!corednsData.TryGetValue("Corefile", out var corefile)) + { + _console.WriteError("CoreDNS Corefile not found in configmap. Please check the configuration and try again."); + return false; + } + // add the clusterName.dev-k8s.cloud rewrite entry to the Corefile after th kubernetes block + // if the line already exists, do not add it again + if (corefile.Contains(rewriteString)) + { + _console.WriteLine("CoreDNS Corefile already contains the rewrite entry for the cluster. No changes made."); + return true; + } + var lines = corefile.Split([Environment.NewLine], StringSplitOptions.RemoveEmptyEntries).ToList(); + + var kubernetesBlockIndex = lines.FindIndex(line => line.Trim().StartsWith("kubernetes")); + + if (kubernetesBlockIndex == -1) + { + _console.WriteError("CoreDNS Corefile does not contain a kubernetes block. Please check the configuration and try again."); + return false; + } + + // insert the new entry after the kubernetes block by searching for the closing brace then inserting it + + var closingBraceIndex = lines.FindIndex(kubernetesBlockIndex, line => line.Trim() == "}"); + if (closingBraceIndex == -1) + { + _console.WriteError("CoreDNS Corefile does not contain a closing brace for the kubernetes block. Please check the configuration and try again."); + return false; + } + lines.Insert(closingBraceIndex, rewriteString); + + // join the lines back into a single string + var updatedCorefile = string.Join(Environment.NewLine, lines); + + // update the configmap data with the new Corefile + corednsData["Corefile"] = updatedCorefile; + + // update the configmap + corednsConfigMap.Data = corednsData; + _client(clusterName).Update(corednsConfigMap); + _console.WriteLine("CoreDNS configmap updated successfully."); + + // restart the coredns + _console.WriteLine("Restarting CoreDNS pods to apply changes."); + var corednsPods = _client(clusterName).List("kube-system", labelSelector: "k8s-app=kube-dns"); + if (!corednsPods.Any()) + { + _console.WriteError("No CoreDNS pods found. Please check the configuration and try again."); + return false; + } + foreach (var pod in corednsPods) + { + _console.WriteLine($"Deleting CoreDNS pod {pod.Name()} to apply changes."); + _client(clusterName).Delete(pod); + } + _console.WriteLine("CoreDNS pods deleted successfully. They will be recreated automatically."); + return true; + } + + private bool CreateTlsSecret(string clusterName) + { // wait until the namespace vega exists before proceeding with the secrets creation bool nsVegaExists = false; int nTimesWaiting = 0; @@ -204,7 +316,7 @@ public void UpsertCluster(string clusterName, int targetPortHttps, int targetPor if (nTimesWaiting >= maxTimesWaiting) { _console.WriteError("Namespace 'vega-system' does not exist after waiting. Please check the configuration and try again."); - return; + return true; } // write the cert secret to the cluster @@ -229,8 +341,7 @@ public void UpsertCluster(string clusterName, int targetPortHttps, int targetPor } _console.WriteLine("Creating vega-system secret"); _client(clusterName).Create(tls); - if (reload) - ReloadConfigs(); + return false; } private void ReloadConfigs() diff --git a/cli/tests/Vdk.Tests/ReverseProxyClientTests.cs b/cli/tests/Vdk.Tests/ReverseProxyClientTests.cs index 42e4144..bcf4393 100644 --- a/cli/tests/Vdk.Tests/ReverseProxyClientTests.cs +++ b/cli/tests/Vdk.Tests/ReverseProxyClientTests.cs @@ -15,12 +15,12 @@ public class ReverseProxyClientTests private readonly Mock _dockerMock = new(); private readonly Mock _consoleMock = new(); private readonly Mock _kindMock = new(); - private readonly Mock _k8sMock = new(); + private readonly Mock _kubeClientMock = new(); private readonly Func _clientFunc; public ReverseProxyClientTests() { - _clientFunc = _ => _k8sMock.Object; + _clientFunc = _ => _kubeClientMock.Object; } [Fact] @@ -66,5 +66,149 @@ public void InitConfFile_CreatesFileAndWritesConfig() File.ReadAllText(tempFile).Should().Contain("server {"); File.Delete(tempFile); } + + [Fact] + public void PatchCoreDns_ReturnsFalse_WhenIngressServiceNotFound() + { + // Arrange + _kubeClientMock.SetupSequence(x => x.Get("ingress-nginx-controller", "ingress-nginx")) + .Returns((V1Service?)null); + var client = new ReverseProxyClient(_dockerMock.Object, _clientFunc, _consoleMock.Object, _kindMock.Object); + + // Act + var result = InvokePatchCoreDns(client, "test-cluster"); + + // Assert + Assert.False(result); + _consoleMock.Verify(x => x.WriteError(It.Is(s => s.Contains("Ingress-nginx-controller service not found"))), Times.Once); + } + + [Fact] + public void PatchCoreDns_ReturnsFalse_WhenCoreDnsConfigMapNotFound() + { + // Arrange + _kubeClientMock.Setup(x => x.Get("ingress-nginx-controller", "ingress-nginx")) + .Returns(new V1Service { Metadata = new V1ObjectMeta { Name = "svc", NamespaceProperty = "ns" } }); + _kubeClientMock.Setup(x => x.Get("coredns", "kube-system")) + .Returns((V1ConfigMap?)null); + var client = new ReverseProxyClient(_dockerMock.Object, _clientFunc, _consoleMock.Object, _kindMock.Object); + + // Act + var result = InvokePatchCoreDns(client, "test-cluster"); + + // Assert + Assert.False(result); + _consoleMock.Verify(x => x.WriteError(It.Is(s => s.Contains("CoreDNS configmap not found"))), Times.Once); + } + + [Fact] + public void PatchCoreDns_ReturnsFalse_WhenCorefileMissing() + { + // Arrange + _kubeClientMock.Setup(x => x.Get("ingress-nginx-controller", "ingress-nginx")) + .Returns(new V1Service { Metadata = new V1ObjectMeta { Name = "svc", NamespaceProperty = "ns" } }); + _kubeClientMock.Setup(x => x.Get("coredns", "kube-system")) + .Returns(new V1ConfigMap { Data = new Dictionary() }); + var client = new ReverseProxyClient(_dockerMock.Object, _clientFunc, _consoleMock.Object, _kindMock.Object); + + // Act + var result = InvokePatchCoreDns(client, "test-cluster"); + + // Assert + Assert.False(result); + _consoleMock.Verify(x => x.WriteError(It.Is(s => s.Contains("CoreDNS Corefile not found"))), Times.Once); + } + + [Fact] + public void PatchCoreDns_ReturnsTrue_WhenRewriteAlreadyExists() + { + // Arrange + var clusterName = "test-cluster"; + var rewriteString = $" rewrite name {clusterName}.dev-k8s.cloud svc.ns.svc.cluster.local"; + var corefile = $"kubernetes cluster.local in-addr.arpa ip6.arpa {{\n}}\n{rewriteString}\n"; + _kubeClientMock.Setup(x => x.Get("ingress-nginx-controller", "ingress-nginx")) + .Returns(new V1Service { Metadata = new V1ObjectMeta { Name = "svc", NamespaceProperty = "ns" } }); + _kubeClientMock.Setup(x => x.Get("coredns", "kube-system")) + .Returns(new V1ConfigMap { Data = new Dictionary { { "Corefile", corefile } } }); + var client = new ReverseProxyClient(_dockerMock.Object, _clientFunc, _consoleMock.Object, _kindMock.Object); + + // Act + var result = InvokePatchCoreDns(client, clusterName); + + // Assert + Assert.True(result); + _consoleMock.Verify(x => x.WriteLine(It.Is(s => s.Contains("already contains the rewrite entry"))), Times.Once); + } + + [Fact] + public void PatchCoreDns_ReturnsFalse_WhenNoKubernetesBlock() + { + // Arrange + var corefile = "some unrelated config"; + _kubeClientMock.Setup(x => x.Get("ingress-nginx-controller", "ingress-nginx")) + .Returns(new V1Service { Metadata = new V1ObjectMeta { Name = "svc", NamespaceProperty = "ns" } }); + _kubeClientMock.Setup(x => x.Get("coredns", "kube-system")) + .Returns(new V1ConfigMap { Data = new Dictionary { { "Corefile", corefile } } }); + var client = new ReverseProxyClient(_dockerMock.Object, _clientFunc, _consoleMock.Object, _kindMock.Object); + + // Act + var result = InvokePatchCoreDns(client, "test-cluster"); + + // Assert + Assert.False(result); + _consoleMock.Verify(x => x.WriteError(It.Is(s => s.Contains("does not contain a kubernetes block"))), Times.Once); + } + + [Fact] + public void PatchCoreDns_ReturnsFalse_WhenNoClosingBrace() + { + // Arrange + var corefile = "kubernetes cluster.local in-addr.arpa ip6.arpa {"; + _kubeClientMock.Setup(x => x.Get("ingress-nginx-controller", "ingress-nginx")) + .Returns(new V1Service { Metadata = new V1ObjectMeta { Name = "svc", NamespaceProperty = "ns" } }); + _kubeClientMock.Setup(x => x.Get("coredns", "kube-system")) + .Returns(new V1ConfigMap { Data = new Dictionary { { "Corefile", corefile } } }); + var client = new ReverseProxyClient(_dockerMock.Object, _clientFunc, _consoleMock.Object, _kindMock.Object); + + // Act + var result = InvokePatchCoreDns(client, "test-cluster"); + + // Assert + Assert.False(result); + _consoleMock.Verify(x => x.WriteError(It.Is(s => s.Contains("does not contain a closing brace"))), Times.Once); + } + + [Fact] + public void PatchCoreDns_UpdatesConfigMapAndRestartsPods() + { + // Arrange + var clusterName = "test-cluster"; + var corefile = $"kubernetes cluster.local in-addr.arpa ip6.arpa {{{Environment.NewLine}}}{Environment.NewLine}"; + var configMap = new V1ConfigMap { Data = new Dictionary { { "Corefile", corefile } } }; + var pod = new V1Pod { Metadata = new V1ObjectMeta { Name = "coredns-1" } }; + _kubeClientMock.Setup(x => x.Get("ingress-nginx-controller", "ingress-nginx")) + .Returns(new V1Service { Metadata = new V1ObjectMeta { Name = "svc", NamespaceProperty = "ns" } }); + _kubeClientMock.Setup(x => x.Get("coredns", "kube-system")) + .Returns(configMap); + _kubeClientMock.Setup(x => x.List("kube-system", It.IsAny())) + .Returns(new List { pod }); + var client = new ReverseProxyClient(_dockerMock.Object, _clientFunc, _consoleMock.Object, _kindMock.Object); + + // Act + var result = InvokePatchCoreDns(client, clusterName); + + // Assert + Assert.True(result); + _kubeClientMock.Verify(x => x.Update(It.Is(cm => cm.Data["Corefile"].Contains($"rewrite name {clusterName}.dev-k8s.cloud svc.ns.svc.cluster.local"))), Times.Once); + _kubeClientMock.Verify(x => x.Delete(pod), Times.Once); + _consoleMock.Verify(x => x.WriteLine(It.Is(s => s.Contains("CoreDNS configmap updated successfully."))), Times.Once); + } + + // Helper to invoke private PatchCoreDns via reflection + private static bool InvokePatchCoreDns(ReverseProxyClient client, string clusterName) + { + var method = typeof(ReverseProxyClient).GetMethod("PatchCoreDns", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return (bool)method.Invoke(client, new object[] { clusterName }); + } } } \ No newline at end of file From ed3c19cec1957259f65bde25d58368820ff2ecdc Mon Sep 17 00:00:00 2001 From: Eddie Date: Sun, 22 Jun 2025 14:11:59 -0500 Subject: [PATCH 2/2] Update cli/src/Vdk/Services/ReverseProxyClient.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Eddie --- cli/src/Vdk/Services/ReverseProxyClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/Vdk/Services/ReverseProxyClient.cs b/cli/src/Vdk/Services/ReverseProxyClient.cs index c4076d2..dff19bd 100644 --- a/cli/src/Vdk/Services/ReverseProxyClient.cs +++ b/cli/src/Vdk/Services/ReverseProxyClient.cs @@ -235,7 +235,7 @@ private bool PatchCoreDns(string clusterName) _console.WriteError("CoreDNS Corefile not found in configmap. Please check the configuration and try again."); return false; } - // add the clusterName.dev-k8s.cloud rewrite entry to the Corefile after th kubernetes block + // add the clusterName.dev-k8s.cloud rewrite entry to the Corefile after the kubernetes block // if the line already exists, do not add it again if (corefile.Contains(rewriteString)) {