diff --git a/KubeOps.slnx b/KubeOps.slnx index 91dd83b7c..52a82defc 100644 --- a/KubeOps.slnx +++ b/KubeOps.slnx @@ -1,5 +1,7 @@  + + @@ -29,6 +31,8 @@ + + @@ -41,6 +45,8 @@ + + @@ -49,4 +55,4 @@ - \ No newline at end of file + diff --git a/README.md b/README.md index 4f1969734..96904e518 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,100 @@ The template approach (`dotnet new operator`) scaffolds a basic operator structu For detailed tutorials and guides, visit the [KubeOps Documentation Site](https://dotnet.github.io/dotnet-operator-sdk/). +## Aspire Quickstart + +KubeOps can be used as a first-class resource in a [.NET Aspire](https://learn.microsoft.com/dotnet/aspire/) AppHost. Add `KubeOps.Aspire.Hosting` to the AppHost and `KubeOps.Aspire` to the operator project. + +The repository keeps this in a dedicated `examples/AspireOperator` project so the plain `examples/Operator` sample remains a non-Aspire KubeOps operator. + +In the operator project, register the service defaults after the operator: + +```csharp +using KubeOps.Aspire; +using KubeOps.Operator; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services + .AddKubernetesOperator() + .RegisterComponents(); + +builder.AddKubeOpsServiceDefaults(); + +await builder.Build().RunAsync(); +``` + +In the AppHost, add the operator and choose whether it should run locally, publish to Kubernetes, or both: + +```csharp +var builder = DistributedApplication.CreateBuilder(args); + +var k8s = builder.AddKubernetesEnvironment("k8s") + .WithHelm(helm => + { + helm.WithChartName("my-operator"); + helm.WithReleaseName("my-operator"); + helm.WithNamespace("operator-system"); + }); + +builder.AddKubeOps("operator") + .RunWithKubernetes(k8s) + .PublishAsKubernetesOperator(k8s); + +builder.Build().Run(); +``` + +Run and publish with the Aspire CLI: + +```bash +aspire run --project src/MyApp.AppHost/MyApp.AppHost.csproj +aspire publish --project src/MyApp.AppHost/MyApp.AppHost.csproj --output-path ./artifacts/k8s +``` + +Common AppHost shapes: + +Local development only: + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev, run => run.WithPersistentCrds()); +``` + +Azure publish/deploy only: + +```csharp +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(aks, publish => publish.WithServiceAccount("operator")); +``` + +Local run and Azure deploy: + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev) + .PublishAsKubernetesOperator(aks); +``` + +Publish only without an Aspire Kubernetes environment: + +```csharp +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +Without `RunWithKubernetes(...)`, `AddKubeOps(...)` keeps the operator in explicit-start mode for local Aspire runs. Standalone manifest publish does not require `AddKubernetesEnvironment(...)`, Helm, or a live cluster; publishing with a Kubernetes environment generates an Aspire Helm chart, while `aspire deploy` installs that chart into the selected environment. + ## Packages The runtime libraries target [.NET 8.0](https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-8/overview), [.NET 9.0](https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-9/overview), and [.NET 10.0](https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-10/overview), leveraging modern C# features. The build-time packages (`KubeOps.Generator` and `KubeOps.Templates`) target [.NET Standard 2.0](https://learn.microsoft.com/en-us/dotnet/standard/net-standard) for broad tooling compatibility. The underlying Kubernetes client library (`KubernetesClient`) is referenced for interacting with the Kubernetes API. @@ -61,6 +155,8 @@ The SDK is designed to be modular. You can include only the packages you need: | Package | Description | | -------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [KubeOps.Abstractions](./src/KubeOps.Abstractions/README.md) | Defines core interfaces, attributes (like `[KubernetesEntity]`), and base classes used across the SDK. Essential for defining your custom resources and controllers. | +| [KubeOps.Aspire](./src/KubeOps.Aspire/README.md) | [.NET Aspire](https://learn.microsoft.com/dotnet/aspire/) service defaults for an operator: a single `AddKubeOpsServiceDefaults()` call wiring up OpenTelemetry, service discovery, HTTP resilience, and health checks. | +| [KubeOps.Aspire.Hosting](./src/KubeOps.Aspire.Hosting/README.md) | [.NET Aspire](https://learn.microsoft.com/dotnet/aspire/) hosting integration. Adds `AddKubeOps(...)` so a KubeOps operator can be orchestrated as a resource inside an Aspire AppHost. | | [KubeOps.Cli](./src/KubeOps.Cli/README.md) | A [.NET Tool](https://docs.microsoft.com/en-us/dotnet/core/tools/global-tools) providing commands for scaffolding projects, generating [Custom Resource Definitions (CRDs)](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/), and more. | | [KubeOps.Generator](./src/KubeOps.Generator/README.md) | Contains [Roslyn Source Generators](https://docs.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/source-generators-overview) to automate boilerplate code generation for CRDs and controllers based on your definitions. | | [KubeOps.KubernetesClient](./src/KubeOps.KubernetesClient/README.md) | Provides an enhanced client for interacting with the [Kubernetes API](https://kubernetes.io/docs/reference/kubernetes-api/), built on top of the official `KubernetesClient` library. Offers convenience methods for common operator tasks. | @@ -71,11 +167,7 @@ The SDK is designed to be modular. You can include only the packages you need: ## Examples -You can find various example operators demonstrating different features in the [`examples/`](https://github.com/dotnet/dotnet-operator-sdk/tree/main/examples/) directory of this repository: - -- [`Operator`](./examples/Operator) - A minimal operator with an entity, controller, and finalizer. -- [`WebhookOperator`](./examples/WebhookOperator) - Demonstrates validating and mutating admission webhooks. -- [`ConversionWebhookOperator`](./examples/ConversionWebhookOperator) - Demonstrates converting between entity versions with a conversion webhook. +You can find various example operators demonstrating different features in the [`examples/`](https://github.com/dotnet/dotnet-operator-sdk/tree/main/examples/) directory of this repository. ## License diff --git a/docs/docs/operator/aspire-kubernetes-model.mdx b/docs/docs/operator/aspire-kubernetes-model.mdx new file mode 100644 index 000000000..30e76f1f6 --- /dev/null +++ b/docs/docs/operator/aspire-kubernetes-model.mdx @@ -0,0 +1,243 @@ +--- +title: Aspire Kubernetes Operator Model +description: Run, publish, and deploy KubeOps operators with Aspire Kubernetes environments. +sidebar_position: 12 +--- + +# Aspire Kubernetes Operator Model + +This document describes the implemented Aspire model for running, publishing, and deploying KubeOps operators. + +Aspire has two relevant execution modes: + +- **Run mode** starts the AppHost locally for development and debugging. +- **Publish/deploy mode** generates and applies deployment artifacts for a target environment. + +KubeOps operators need both modes, but they need different behavior in each mode. + +## Design + +- Keep `aspire run` as local process orchestration. +- Keep `aspire publish` and `aspire deploy` as deployment artifact and cluster deployment flows. +- Reuse Aspire Kubernetes and AKS compute environments instead of inventing a separate KubeOps target model. +- Make local CRD handling explicit, safe, and predictable. +- Avoid automatic destructive changes to a developer's current cluster. + +## AppHost Shape + +The AppHost can express a local run target and a publish target separately: + +```csharp +var builder = DistributedApplication.CreateBuilder(args); + +var dev = builder.AddKubernetesEnvironment("dev") + .WithHelm(helm => helm.WithNamespace("operator-dev")); + +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithHelm(helm => helm.WithNamespace("operator-system")); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev, run => + { + run.WithPersistentCrds(); + }) + .PublishAsKubernetesOperator(aks, publish => + { + publish.WithServiceAccount("operator"); + }); + +builder.Build().Run(); +``` + +`RunWithKubernetes(...)` configures the local project process. The operator still runs on the developer machine under Aspire, but its Kubernetes client is configured for the selected Kubernetes target. + +`PublishAsKubernetesOperator(...)` configures the Kubernetes deployment. The operator becomes a Kubernetes workload, and KubeOps-generated CRDs, RBAC, and service-account resources are incorporated into the generated Aspire chart. + +If `RunWithKubernetes(...)` is not configured, `AddKubeOps(...)` keeps the operator resource in Aspire's explicit-start mode during `aspire run`. The project remains in the AppHost model and publish graph, but the local operator process does not start by default. A KubeOps operator without a Kubernetes run target would otherwise fail against an accidental kube context or mutate the wrong cluster. + +## Run Mode + +Run mode defaults to a productive inner loop once you opt in: + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev); +``` + +Default run behavior: + +- If run mode is not configured, do not start the operator automatically. +- Create missing CRDs before starting the local operator process. +- Track which CRDs were created by this Aspire run. +- Remove only the CRDs that this run created when the AppHost shuts down. +- Leave pre-existing CRDs alone. +- Configure the operator process with the target namespace and Kubernetes client hints. + +Run mode exposes explicit CRD lifecycle choices: + +```csharp +run.WithEphemeralCrds(); // default: create missing CRDs, remove only those created by this run +run.WithPersistentCrds(); // create or update CRDs, leave them after shutdown +run.RequireExistingCrds(); // fail before operator start if CRDs are missing +run.SkipCrds(); // do not check or manage CRDs +``` + +RBAC and service accounts are not required for the default local process path because the process normally authenticates with the developer's kubeconfig credentials. They can be added later for parity scenarios, for example a `RunAsServiceAccount(...)` option that creates a service account, binds RBAC, obtains a token, and configures the local process to use it. + +## Publish And Deploy Mode + +Publish mode defaults to production-ready Kubernetes output: + +```csharp +var k8s = builder.AddKubernetesEnvironment("k8s") + .WithHelm(helm => + { + helm.WithChartName("my-operator"); + helm.WithReleaseName("my-operator"); + helm.WithNamespace("operator-system"); + }); + +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(k8s); +``` + +Default publish behavior: + +- Generate CRDs. +- Generate RBAC. +- Generate a service account using the operator resource name. +- Bind generated RBAC to that service account. +- Let Aspire own the operator `Deployment` so image publishing, Helm values, service discovery, configuration, and telemetry wiring remain part of the full Aspire model. +- Merge KubeOps-generated deployment settings into the Aspire-owned deployment instead of emitting a duplicate deployment. + +Publish options make each part explicit: + +```csharp +publish.GenerateCrds(); // default +publish.GenerateRbac(); // default +publish.WithServiceAccount("operator"); // default name: resource name +publish.SkipCrds(); +publish.SkipRbac(); +``` + +With a Kubernetes environment, `aspire publish` writes the generated Helm chart. `aspire deploy` uses the same Kubernetes environment and deployment engine to install it. The KubeOps integration hooks into Aspire's Kubernetes resource customization path (`PublishAsKubernetesService`) rather than running a separate KubeOps deployment. + +The integration filters KubeOps' generated `Deployment` out of the additional resource set because Aspire owns the workload. KubeOps deployment settings such as replicas, termination grace period, environment variables, resource requirements, `POD_NAMESPACE`, and the service account are merged into the Aspire-owned deployment instead. The chart therefore deploys one operator deployment, not a service-only placeholder and not a duplicate operator. + +## Standalone Publish + +You can also generate KubeOps manifests without an Aspire Kubernetes environment: + +```csharp +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +This path is useful when the AppHost should participate in Aspire publish, but the Kubernetes installation is handled by another workflow. It does not call Aspire's Kubernetes publishing integration and does not require `AddKubernetesEnvironment(...)`, Helm, or a live cluster. `aspire publish` writes the raw KubeOps-generated YAML under the operator resource's output directory. + +## AKS And Existing Resources + +AKS follows Aspire's Azure resource conventions. + +When the AppHost declares: + +```csharp +var aks = builder.AddAzureKubernetesEnvironment("aks"); +``` + +then `aspire deploy` provisions the AKS cluster, ACR, identity, and dependent Azure resources according to Aspire's AKS integration, then builds images, pushes them to ACR, generates Helm charts, and installs them into AKS. + +For local run mode, Azure integrations may provision Azure resources unless configured as existing resources. Users who want local run mode to attach to an existing AKS cluster should use Aspire's existing-resource APIs: + +```csharp +var aks = builder.AddAzureKubernetesEnvironment("aks") + .RunAsExisting(aksName, aksResourceGroup); + +builder.AddKubeOps("operator") + .RunWithKubernetes(aks); +``` + +This keeps KubeOps aligned with Aspire instead of inventing a KubeOps-specific `UseExistingAks` switch. + +## Scenario Examples + +### Local Only + +Run the operator as a local process against the selected Kubernetes environment. The operator is not automatically published to Kubernetes unless you also call `PublishAsKubernetesOperator(...)`. + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev, run => run.WithPersistentCrds()); +``` + +### Azure Only + +Deploy the operator into AKS without running it as a local process during `aspire run`. + +```csharp +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(aks, publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +### Local Run And Azure Deploy + +Use a local Kubernetes environment for the development loop and AKS for publish/deploy. + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev) + .PublishAsKubernetesOperator(aks, publish => publish.WithServiceAccount("operator")); +``` + +### Publish Only Without A Kubernetes Environment + +Generate standalone KubeOps manifests without registering an Aspire Kubernetes environment. + +```csharp +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +## Mock And Local Cluster Targets + +KubeOps runtime watchers require a Kubernetes API server. Unit tests can replace `IKubernetesClient`, but the normal operator runtime is not an offline simulator. + +Additional targets can be modeled as Kubernetes environments if they provide Kubernetes API semantics: + +```csharp +var k3s = builder.AddK3sCluster("k3s"); +var mock = builder.AddKubeOpsMockKubernetes("mock-kube"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(k3s); +``` + +A mock target would require operator-side runtime support to replace the Kubernetes client and watcher behavior. It should not be represented as a normal Kubernetes environment unless it provides Kubernetes API semantics. + +## Non-Goals + +- Do not hide cluster mutations behind `#if DEBUG`. +- Do not deploy a second KubeOps-generated operator deployment beside Aspire's deployment. +- Do not require users to use AKS; plain Kubernetes environments, kind, k3s, and existing kubeconfig targets should work. +- Do not make local run mode require RBAC/service-account generation unless the user explicitly asks to run as a service account. diff --git a/docs/docs/operator/aspire.mdx b/docs/docs/operator/aspire.mdx new file mode 100644 index 000000000..c1d51cce0 --- /dev/null +++ b/docs/docs/operator/aspire.mdx @@ -0,0 +1,236 @@ +--- +title: .NET Aspire +description: Orchestrate and observe a KubeOps operator with .NET Aspire +sidebar_position: 9.5 +--- + +# .NET Aspire Integration + +[.NET Aspire](https://learn.microsoft.com/dotnet/aspire/) is an opinionated stack for building observable, production-ready distributed applications. KubeOps ships two packages that let you treat an operator as a first-class Aspire resource: + +| Package | Side | Purpose | +| --- | --- | --- | +| [`KubeOps.Aspire.Hosting`](../packages/aspire-hosting) | AppHost | Adds `AddKubeOps(...)` so the operator is orchestrated as a resource. | +| [`KubeOps.Aspire`](../packages/aspire) | Operator | Adds `AddKubeOpsServiceDefaults()` for OpenTelemetry, service discovery, resilience and health checks. | + +Together they give you a local dashboard with the operator's logs, traces and metrics, automatic service discovery to other resources, and a single entry point to run, publish, or deploy the whole stack. + +## The two halves of an Aspire integration + +Aspire integrations always come in two parts, and KubeOps follows that convention: + +1. **Hosting integration** — code that runs in the *AppHost* project and describes the application model. +2. **Service defaults** — a small amount of wiring inside the *operator* project itself. + +The service-defaults call is the one piece you add to the operator. This is the idiomatic Aspire pattern (`AddServiceDefaults()`); the hosting side then injects the configuration (telemetry endpoint, service discovery variables) automatically. + +## Operator project + +Install `KubeOps.Aspire` and call `AddKubeOpsServiceDefaults()` on the host builder, **after** `AddKubernetesOperator()`: + +```csharp +using KubeOps.Aspire; +using KubeOps.Operator; + +using Microsoft.Extensions.Hosting; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services + .AddKubernetesOperator() + .RegisterComponents(); + +builder.AddKubeOpsServiceDefaults(); + +using var host = builder.Build(); +await host.RunAsync(); +``` + +`AddKubeOpsServiceDefaults()` configures: + +- **OpenTelemetry** logging (with scopes and formatted messages), metrics (runtime + `HttpClient`) and tracing. It subscribes to the operator's `ActivitySource`, which KubeOps registers under the operator name (see [Logging, Tracing, and OpenTelemetry](./logging)). +- **OTLP export** — enabled automatically when the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable is present. Aspire sets this for you, so traces and metrics show up in the dashboard with no extra code. +- **Service discovery** — `HttpClient` instances resolve logical Aspire resource names (e.g. `https://apiservice`). +- **HTTP resilience** — the standard resilience handler (retries, circuit breaker, timeouts) is applied to all `HttpClient` instances. +- **Health checks** — a default `self` liveness check tagged `live`. + +:::tip Operator name +The OpenTelemetry service name and the tracing source name must match `OperatorSettings.Name` — otherwise the operator's reconciliation traces are never captured. Call `AddKubeOpsServiceDefaults()` **after** `AddKubernetesOperator()` so KubeOps can resolve the configured name automatically. If you must call it earlier, pass the name explicitly (and keep it in sync with `OperatorSettings.Name`): + +```csharp +builder.AddKubeOpsServiceDefaults("my-operator"); +``` +::: + +## AppHost project + +Create an [Aspire AppHost](https://learn.microsoft.com/dotnet/aspire/fundamentals/app-host-overview) project, reference `KubeOps.Aspire.Hosting`, and add the operator with `AddKubeOps`: + +The repository sample uses `examples/AspireOperator` for this so `examples/Operator` stays a plain KubeOps operator sample. + +```csharp +var builder = DistributedApplication.CreateBuilder(args); + +builder.AddKubernetesEnvironment("k8s") + .WithHelm(helm => + { + helm.WithChartName("my-operator"); + helm.WithReleaseName("my-operator"); + helm.WithNamespace("operator-system"); + }); + +var apiService = builder.AddProject("apiservice"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(k8s) + .PublishAsKubernetesOperator(k8s) + .WithReference(apiService); + +builder.Build().Run(); +``` + +`AddKubeOps` is a thin, KubeOps-flavoured wrapper around the built-in `AddProject`. It returns the standard `IResourceBuilder`, so every Aspire extension works as usual: + +- `WithReference(apiService)` injects the service-discovery configuration so the operator can call `apiservice` by name. +- `WithEnvironment(...)`, `WaitFor(...)`, `WithReplicas(...)` and friends all apply unchanged. +- `RunWithKubernetes(k8s)` opts the operator into local execution against a Kubernetes target. +- `PublishAsKubernetesOperator(k8s)` includes the operator, CRDs, RBAC, and service account in the Aspire Kubernetes publish/deploy output. + +:::note Project reference +When referencing `KubeOps.Aspire.Hosting` from an AppHost, mark it as a normal code reference so the AppHost SDK does not treat it as a resource: + +```xml + +``` +::: + +## Running locally + +Run the AppHost; the Aspire dashboard opens and starts resources that are configured for local run: + +```bash +dotnet run --project examples/AspireAppHost +``` + +A KubeOps operator is explicit-start by default. It does not run in local mode unless you call `RunWithKubernetes(...)`. This prevents an operator from accidentally using the developer's current kube context. + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev); +``` + +By default, `RunWithKubernetes(...)` creates missing CRDs before the local operator process starts, tracks the CRDs created by this AppHost run, and removes only those CRDs when the AppHost shuts down. Existing CRDs are left alone. + +Use the run options when the development cluster should keep CRDs after shutdown or when CRDs are managed separately: + +```csharp +builder.AddKubeOps("operator") + .RunWithKubernetes(dev, run => + { + run.WithPersistentCrds(); // create or update CRDs and leave them in the cluster + // run.RequireExistingCrds(); + // run.SkipCrds(); + }); +``` + +The operator process runs locally and authenticates through the selected Kubernetes environment. The dashboard shows structured logs, the reconciliation traces emitted from the operator's `ActivitySource`, and runtime/HTTP metrics. + +## Health endpoints + +`KubeOps.Aspire` is usable from a plain console operator and therefore does **not** force an ASP.NET Core dependency. The default health checks are registered in the service collection, but exposing them over HTTP requires an HTTP server. + +If your operator already hosts webhooks via [`KubeOps.Operator.Web`](../packages/operator-web), map the endpoints on the `WebApplication`: + +```csharp +app.MapHealthChecks("/health"); +app.MapHealthChecks("/alive", new HealthCheckOptions +{ + Predicate = registration => registration.Tags.Contains("live"), +}); +``` + +These line up with the standard Kubernetes liveness/readiness probe conventions and the Aspire dashboard's health reporting. + +## Deployment + +The Aspire path for Kubernetes publishing is the official [`Aspire.Hosting.Kubernetes`](https://learn.microsoft.com/dotnet/aspire/deployment/kubernetes-integration) hosting integration. Add the package to the AppHost, call `AddKubernetesEnvironment(...)`, then publish the AppHost with the [Aspire CLI](https://learn.microsoft.com/dotnet/aspire/cli/overview): + +```bash +aspire publish --project examples/AspireAppHost/AspireAppHost.csproj --output-path ./k8s-artifacts +``` + +When you call `PublishAsKubernetesOperator(k8s)`, Aspire writes a Helm chart for the resources in the AppHost. `AddKubeOps` also contributes the Kubernetes API extension pieces that KubeOps generates: + +- CRDs are appended to the generated chart. +- RBAC resources are appended to the generated chart. +- The Aspire-generated operator `Deployment` is patched with the KubeOps-generated pod/deployment settings, `POD_NAMESPACE`, and the configured service account. +- KubeOps' generated `Deployment` is merged into the Aspire workload instead of emitted as a second workload, so the chart does not deploy the operator twice. + +`aspire publish` generates the chart and does not require the Helm CLI. `aspire deploy` installs the chart into the target environment and requires Helm plus the registry and cluster configuration expected by Aspire's Kubernetes deployment flow. + +By default, the hosting integration invokes `kubeops generate operator`. If the CLI is not on `PATH`, configure the invocation: + +```csharp +builder.AddKubeOps( + "operator", + manifests => + { + manifests.Namespace = "operator-system"; + manifests.UseKubeOpsCli("dotnet", "tool", "run", "kubeops", "--"); + }); +``` + +### Standalone manifest publish + +If the AppHost does not define a Kubernetes environment, the operator can still publish KubeOps manifests: + +```csharp +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +This `aspire publish` path writes raw KubeOps YAML under the operator resource output directory and does not require `AddKubernetesEnvironment(...)`, Helm, or a live cluster. + +### Common scenarios + +Local development against a local cluster: + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev, run => run.WithPersistentCrds()); +``` + +Azure deployment into AKS: + +```csharp +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(aks, publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +Local run plus Azure deployment: + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev) + .PublishAsKubernetesOperator(aks, publish => publish.WithServiceAccount("operator")); +``` + +For the deeper run/publish model, see [Aspire Kubernetes Operator Model](./aspire-kubernetes-model). diff --git a/docs/docs/packages/aspire-hosting.mdx b/docs/docs/packages/aspire-hosting.mdx new file mode 100644 index 000000000..6a4c28769 --- /dev/null +++ b/docs/docs/packages/aspire-hosting.mdx @@ -0,0 +1,17 @@ +--- +title: KubeOps.Aspire.Hosting +description: .NET Aspire hosting integration for KubeOps operators +sidebar_position: 4.4 +hide_title: true +--- + +import AspireHosting from "../../../src/KubeOps.Aspire.Hosting/README.md"; + + + +## Related Packages + +- [KubeOps.Aspire](./aspire) - Service defaults for the operator project +- [KubeOps.Operator](./operator) - Main operator engine + +See the [.NET Aspire guide](../operator/aspire) for the full walkthrough. diff --git a/docs/docs/packages/aspire.mdx b/docs/docs/packages/aspire.mdx new file mode 100644 index 000000000..84ad74a66 --- /dev/null +++ b/docs/docs/packages/aspire.mdx @@ -0,0 +1,17 @@ +--- +title: KubeOps.Aspire +description: .NET Aspire service defaults for KubeOps operators +sidebar_position: 4.3 +hide_title: true +--- + +import Aspire from "../../../src/KubeOps.Aspire/README.md"; + + + +## Related Packages + +- [KubeOps.Aspire.Hosting](./aspire-hosting) - AppHost integration for orchestrating the operator +- [KubeOps.Operator](./operator) - Main operator engine + +See the [.NET Aspire guide](../operator/aspire) for the full walkthrough. diff --git a/docs/docs/packages/index.mdx b/docs/docs/packages/index.mdx index dd1e20d17..7f52b92e9 100644 --- a/docs/docs/packages/index.mdx +++ b/docs/docs/packages/index.mdx @@ -14,6 +14,11 @@ KubeOps is designed to be modular, allowing you to include only the packages you - [KubeOps.Operator](./operator) - Main operator engine - [KubeOps.Operator.Web](./operator-web) - ASP.NET Core integration +## Hosting & Orchestration + +- [KubeOps.Aspire.Hosting](./aspire-hosting) - .NET Aspire AppHost integration +- [KubeOps.Aspire](./aspire) - .NET Aspire service defaults for the operator + ## Development Tools - [KubeOps.Cli](./cli) - Command-line tools diff --git a/examples/AspireAppHost/AspireAppHost.csproj b/examples/AspireAppHost/AspireAppHost.csproj new file mode 100644 index 000000000..6ccd7c253 --- /dev/null +++ b/examples/AspireAppHost/AspireAppHost.csproj @@ -0,0 +1,27 @@ + + + + + + Exe + net9.0 + enable + enable + true + false + false + kubeops-aspire-apphost + + + + + + + + + + + + + diff --git a/examples/AspireAppHost/Program.cs b/examples/AspireAppHost/Program.cs new file mode 100644 index 000000000..b0517b5f9 --- /dev/null +++ b/examples/AspireAppHost/Program.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +var builder = DistributedApplication.CreateBuilder(args); + +var k8s = builder.AddKubernetesEnvironment("k8s") + .WithHelm(helm => + { + helm.WithChartName("kubeops-aspire"); + helm.WithReleaseName("kubeops-aspire"); + helm.WithNamespace("operator-system"); + }); + +builder.AddKubeOps("operator") + .RunWithKubernetes(k8s, run => run.WithPersistentCrds()) + .PublishAsKubernetesOperator( + k8s, + manifests => + { + manifests.Namespace = "operator-system"; + manifests.UseKubeOpsCli( + "dotnet", + "run", + "--project", + "..\\..\\src\\KubeOps.Cli\\KubeOps.Cli.csproj", + "--framework", + "net10.0", + "--"); + }); + +builder.Build().Run(); diff --git a/examples/AspireAppHost/Properties/launchSettings.json b/examples/AspireAppHost/Properties/launchSettings.json new file mode 100644 index 000000000..cae0b1d3c --- /dev/null +++ b/examples/AspireAppHost/Properties/launchSettings.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17231;http://localhost:15231", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21231", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22231" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15231", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19231", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20231", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + } + } +} diff --git a/examples/AspireOperator/AspireOperator.csproj b/examples/AspireOperator/AspireOperator.csproj new file mode 100644 index 000000000..78f008731 --- /dev/null +++ b/examples/AspireOperator/AspireOperator.csproj @@ -0,0 +1,20 @@ + + + + Exe + net9.0 + enable + enable + false + false + + + + + + + + + diff --git a/examples/AspireOperator/Controller/V1TestEntityController.cs b/examples/AspireOperator/Controller/V1TestEntityController.cs new file mode 100644 index 000000000..a90212795 --- /dev/null +++ b/examples/AspireOperator/Controller/V1TestEntityController.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using AspireOperator.Entities; + +using k8s.Models; + +using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; + +using Microsoft.Extensions.Logging; + +namespace AspireOperator.Controller; + +[EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] +public sealed class V1TestEntityController(ILogger logger) + : IEntityController +{ + public Task> ReconcileAsync( + V1TestEntity entity, CancellationToken cancellationToken) + { + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); + } + + public Task> DeletedAsync( + V1TestEntity entity, CancellationToken cancellationToken) + { + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); + } +} diff --git a/examples/AspireOperator/Entities/V1TestEntity.cs b/examples/AspireOperator/Entities/V1TestEntity.cs new file mode 100644 index 000000000..a516ec656 --- /dev/null +++ b/examples/AspireOperator/Entities/V1TestEntity.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s.Models; + +using KubeOps.Abstractions.Entities; + +namespace AspireOperator.Entities; + +[KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] +public partial class V1TestEntity : CustomKubernetesEntity +{ + public override string ToString() => $"Test Entity ({Metadata.Name}): {Spec.Username} ({Spec.Email})"; + + public class EntitySpec + { + public string Username { get; set; } = string.Empty; + + public string Email { get; set; } = string.Empty; + } +} diff --git a/examples/AspireOperator/Finalizer/FinalizerOne.cs b/examples/AspireOperator/Finalizer/FinalizerOne.cs new file mode 100644 index 000000000..521914181 --- /dev/null +++ b/examples/AspireOperator/Finalizer/FinalizerOne.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using AspireOperator.Entities; + +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; + +namespace AspireOperator.Finalizer; + +public sealed class FinalizerOne : IEntityFinalizer +{ + public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); +} diff --git a/examples/AspireOperator/Program.cs b/examples/AspireOperator/Program.cs new file mode 100644 index 000000000..c30bfced1 --- /dev/null +++ b/examples/AspireOperator/Program.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using KubeOps.Aspire; +using KubeOps.Operator; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Logging.SetMinimumLevel(LogLevel.Trace); + +builder.Services + .AddKubernetesOperator() + .RegisterComponents(); + +builder.AddKubeOpsServiceDefaults(); + +using var host = builder.Build(); +await host.RunAsync(); diff --git a/examples/AspireOperator/Properties/launchSettings.json b/examples/AspireOperator/Properties/launchSettings.json new file mode 100644 index 000000000..bc8fb6249 --- /dev/null +++ b/examples/AspireOperator/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "AspireOperator": { + "commandName": "Project" + } + } +} diff --git a/examples/Operator/Program.cs b/examples/Operator/Program.cs index 17b2a7699..8615a6a55 100644 --- a/examples/Operator/Program.cs +++ b/examples/Operator/Program.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +#if DEBUG using KubeOps.Abstractions.Crds; +#endif using KubeOps.Operator; using Microsoft.Extensions.Hosting; diff --git a/src/KubeOps.Aspire.Hosting/KubeOps.Aspire.Hosting.csproj b/src/KubeOps.Aspire.Hosting/KubeOps.Aspire.Hosting.csproj new file mode 100644 index 000000000..952363a8a --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOps.Aspire.Hosting.csproj @@ -0,0 +1,24 @@ + + + + net8.0;net9.0;net10.0 + + + + KubeOps.Aspire.Hosting + Kubernetes Operator SDK Aspire Hosting AppHost + + .NET Aspire hosting integration for KubeOps operators. Adds an + AddKubeOps extension to the distributed application builder so a KubeOps + operator project can be orchestrated as a resource in a .NET Aspire + AppHost, wired up with references, environment variables and telemetry. + + + + + + + + + + diff --git a/src/KubeOps.Aspire.Hosting/KubeOpsGeneratedKubernetesResource.cs b/src/KubeOps.Aspire.Hosting/KubeOpsGeneratedKubernetesResource.cs new file mode 100644 index 000000000..5d8d9dbc5 --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOpsGeneratedKubernetesResource.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using Aspire.Hosting.Kubernetes.Resources; + +using YamlDotNet.Serialization; + +namespace Aspire.Hosting; + +internal sealed class KubeOpsGeneratedKubernetesResource(string apiVersion, string kind) + : BaseKubernetesResource(apiVersion, kind) +{ + [YamlMember(Alias = "data")] + public object? Data { get; set; } + + [YamlMember(Alias = "stringData")] + public object? StringData { get; set; } + + [YamlMember(Alias = "type")] + public object? Type { get; set; } + + [YamlMember(Alias = "spec")] + public object? Spec { get; set; } + + [YamlMember(Alias = "rules")] + public object? Rules { get; set; } + + [YamlMember(Alias = "roleRef")] + public object? RoleRef { get; set; } + + [YamlMember(Alias = "subjects")] + public object? Subjects { get; set; } + + [YamlMember(Alias = "webhooks")] + public object? Webhooks { get; set; } + + [YamlMember(Alias = "secrets")] + public object? Secrets { get; set; } + + [YamlMember(Alias = "imagePullSecrets")] + public object? ImagePullSecrets { get; set; } + + public bool ShouldSerializeData() => Data is not null; + + public bool ShouldSerializeStringData() => StringData is not null; + + public bool ShouldSerializeType() => Type is not null; + + public bool ShouldSerializeSpec() => Spec is not null; + + public bool ShouldSerializeRules() => Rules is not null; + + public bool ShouldSerializeRoleRef() => RoleRef is not null; + + public bool ShouldSerializeSubjects() => Subjects is not null; + + public bool ShouldSerializeWebhooks() => Webhooks is not null; + + public bool ShouldSerializeSecrets() => Secrets is not null; + + public bool ShouldSerializeImagePullSecrets() => ImagePullSecrets is not null; +} diff --git a/src/KubeOps.Aspire.Hosting/KubeOpsHostingExtensions.cs b/src/KubeOps.Aspire.Hosting/KubeOpsHostingExtensions.cs new file mode 100644 index 000000000..bd77d502e --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOpsHostingExtensions.cs @@ -0,0 +1,821 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; +using System.Diagnostics; +using System.Text.Json.Nodes; + +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Kubernetes.Resources; +using Aspire.Hosting.Pipelines; + +using Microsoft.Extensions.DependencyInjection; + +namespace Aspire.Hosting; + +/// +/// Extension methods for adding KubeOps operator projects to a +/// . +/// +public static class KubeOpsHostingExtensions +{ + /// + /// Adds a KubeOps operator project to the distributed application model. + /// + /// + /// This is a KubeOps-flavoured wrapper around + /// . + /// It returns the standard builder, so all the usual + /// Aspire extensions (WithReference, WithEnvironment, WaitFor, ...) + /// apply unchanged. Pair it with AddKubeOpsServiceDefaults() from the + /// KubeOps.Aspire package in the operator project to enable telemetry and + /// service discovery. + /// + /// + /// The operator project metadata type generated by the Aspire AppHost SDK + /// (e.g. Projects.MyOperator). + /// + /// The distributed application builder. + /// The resource name of the operator. + /// An for further configuration. + public static IResourceBuilder AddKubeOps( + this IDistributedApplicationBuilder builder, + [ResourceName] string name) + where TProject : IProjectMetadata, new() + { + ArgumentNullException.ThrowIfNull(builder); + + var project = builder.AddProject(name) + .WithExplicitStart(); + var projectPath = new TProject().ProjectPath; + var options = CreateDefaultManifestOptions(name); + + project.Resource.Annotations.Add(new KubeOpsPublishAnnotation(projectPath, options)); + ConfigureStandaloneKubeOpsPublishing(project, projectPath, options); + + return project; + } + + /// + /// Adds a KubeOps operator project to the distributed application model and configures its Kubernetes manifest output. + /// + /// + /// The operator project metadata type generated by the Aspire AppHost SDK + /// (e.g. Projects.MyOperator). + /// + /// The distributed application builder. + /// The resource name of the operator. + /// Configures KubeOps Kubernetes manifest generation. + /// An for further configuration. + public static IResourceBuilder AddKubeOps( + this IDistributedApplicationBuilder builder, + [ResourceName] string name, + Action configureManifests) + where TProject : IProjectMetadata, new() + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(configureManifests); + + return builder.AddKubeOps(name) + .PublishAsKubernetesOperator(configureManifests); + } + + /// + /// Configures a KubeOps operator project as a Kubernetes operator in Aspire's publish output. + /// + /// The operator project resource builder. + /// Configures KubeOps Kubernetes manifest generation. + /// The same project builder so additional Aspire configuration can be chained. + public static IResourceBuilder PublishAsKubernetesOperator( + this IResourceBuilder builder, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + var annotation = builder.Resource.Annotations.OfType().SingleOrDefault() + ?? throw new InvalidOperationException( + $"{nameof(PublishAsKubernetesOperator)} can only be used with project resources added via " + + $"{nameof(AddKubeOps)}."); + + configure?.Invoke(annotation.Options); + return builder; + } + + /// + /// Configures a KubeOps operator project as a Kubernetes operator for the selected compute environment. + /// + /// The operator project resource builder. + /// The Kubernetes environment used for publish and deploy. + /// Configures KubeOps Kubernetes manifest generation. + /// The same project builder after deployment target configuration is attached. + public static IResourceBuilder PublishAsKubernetesOperator( + this IResourceBuilder builder, + IResourceBuilder target, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(target); + + var annotation = builder.Resource.Annotations.OfType().SingleOrDefault() + ?? throw new InvalidOperationException( + $"{nameof(PublishAsKubernetesOperator)} can only be used with project resources added via " + + $"{nameof(AddKubeOps)}."); + + builder.WithComputeEnvironment(target); + ConfigureKubeOpsPublishing(builder, annotation.ProjectPath, annotation.Options); + return builder.PublishAsKubernetesOperator(configure); + } + + /// + /// Configures a KubeOps operator project to run locally with a Kubernetes environment target. + /// + /// The operator project resource builder. + /// The Kubernetes environment used by the local operator process. + /// Configures local run behavior. + /// The same project builder after local run configuration is attached. + public static IResourceBuilder RunWithKubernetes( + this IResourceBuilder builder, + IResourceBuilder target, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(target); + + RemoveExplicitStart(builder.Resource); + + var publishAnnotation = builder.Resource.Annotations.OfType().SingleOrDefault() + ?? throw new InvalidOperationException( + $"{nameof(RunWithKubernetes)} can only be used with project resources added via " + + $"{nameof(AddKubeOps)}."); + + var options = new KubeOpsRunOptions(target.Resource); + configure?.Invoke(options); + var runAnnotation = new KubeOpsRunAnnotation(options); + builder.Resource.Annotations.Add(runAnnotation); + ConfigureKubeOpsRun(builder, publishAnnotation, runAnnotation); + return builder; + } + + private static KubeOpsKubernetesManifestOptions CreateDefaultManifestOptions(string name) + => new() + { + Namespace = $"{name}-system", + OperatorName = name, + ServiceAccountName = name, + }; + + private static void ConfigureKubeOpsPublishing( + IResourceBuilder project, + string projectPath, + KubeOpsKubernetesManifestOptions options) + => project.PublishAsKubernetesService(kubernetes => + { + ConfigureOperatorPod(kubernetes, project.Resource.Name, options); + + foreach (var resource in GenerateKubeOpsResources(projectPath, kubernetes, options)) + { + kubernetes.AdditionalResources.Add(resource); + } + }); + + private static void ConfigureStandaloneKubeOpsPublishing( + IResourceBuilder project, + string projectPath, + KubeOpsKubernetesManifestOptions options) +#pragma warning disable ASPIREPIPELINES001, ASPIREPIPELINES004 + => project.Resource.Annotations.Add( + new PipelineStepAnnotation(context => + { + var resource = context.Resource + ?? throw new InvalidOperationException("KubeOps publish step requires a resource context."); + if (resource.GetComputeEnvironment() is KubernetesEnvironmentResource) + { + return []; + } + + return new[] + { + new PipelineStep + { + Name = $"publish-{resource.Name}-kubeops", + Description = $"Generate KubeOps Kubernetes manifests for {resource.Name}.", + Resource = resource, + RequiredBySteps = [WellKnownPipelineSteps.Publish], + Action = async stepContext => + { + var output = stepContext.Services + .GetRequiredService() + .GetOutputDirectory(resource); + + Directory.CreateDirectory(output); + RunKubeOpsGenerator(projectPath, output, options, "yaml"); + + await stepContext.ReportingStep.SucceedAsync( + $"Generated KubeOps Kubernetes manifests in '{output}'", + stepContext.CancellationToken); + }, + }, + }; + })); +#pragma warning restore ASPIREPIPELINES001, ASPIREPIPELINES004 + + private static void ConfigureKubeOpsRun( + IResourceBuilder project, + KubeOpsPublishAnnotation publish, + KubeOpsRunAnnotation run) + { + project.OnBeforeResourceStarted( + async (_, _, cancellationToken) => await PrepareRunCrdsAsync(publish, run, cancellationToken)); + project.OnResourceStopped( + async (_, _, cancellationToken) => await CleanupRunCrdsAsync(run, cancellationToken)); + } + + private static async Task PrepareRunCrdsAsync( + KubeOpsPublishAnnotation publish, + KubeOpsRunAnnotation run, + CancellationToken cancellationToken) + { + if (run.Options.CrdMode is KubeOpsRunCrdMode.Skip) + { + return; + } + + foreach (var crd in GenerateKubeOpsJsonResources(publish.ProjectPath, publish.Options) + .Where(resource => IsKind(resource, "CustomResourceDefinition"))) + { + var name = crd["metadata"]?["name"]?.GetValue(); + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + var exists = await KubernetesResourceExistsAsync(run.Options.Target, "crd", name, cancellationToken); + if (run.Options.CrdMode is KubeOpsRunCrdMode.RequireExisting) + { + if (!exists) + { + throw new InvalidOperationException( + $"Kubernetes CRD '{name}' is required before running operator " + + $"'{publish.Options.OperatorName ?? "operator"}'."); + } + + continue; + } + + if (!exists || run.Options.CrdMode is KubeOpsRunCrdMode.Persistent) + { + await ApplyKubernetesJsonAsync(run.Options.Target, crd, cancellationToken); + } + + if (!exists && run.Options.CrdMode is KubeOpsRunCrdMode.Ephemeral) + { + run.CreatedCrds.Add(name); + } + } + } + + private static async Task CleanupRunCrdsAsync( + KubeOpsRunAnnotation run, + CancellationToken cancellationToken) + { + if (run.Options.CrdMode is not KubeOpsRunCrdMode.Ephemeral) + { + return; + } + + foreach (var crd in run.CreatedCrds) + { + await RunKubectlAsync( + run.Options.Target, + ["delete", "crd", crd, "--ignore-not-found"], + cancellationToken, + throwOnError: false); + } + } + + private static async Task KubernetesResourceExistsAsync( + KubernetesEnvironmentResource target, + string kind, + string name, + CancellationToken cancellationToken) + { + var exitCode = await RunKubectlAsync( + target, + ["get", kind, name, "-o", "name"], + cancellationToken, + throwOnError: false); + + return exitCode == 0; + } + + private static async Task ApplyKubernetesJsonAsync( + KubernetesEnvironmentResource target, + JsonObject resource, + CancellationToken cancellationToken) + { + var file = Path.Combine(Path.GetTempPath(), "kubeops-aspire", $"{Guid.NewGuid():N}.json"); + Directory.CreateDirectory(Path.GetDirectoryName(file)!); + await File.WriteAllTextAsync(file, resource.ToJsonString(), cancellationToken); + + try + { + await RunKubectlAsync(target, ["apply", "-f", file], cancellationToken); + } + finally + { + File.Delete(file); + } + } + + private static async Task RunKubectlAsync( + KubernetesEnvironmentResource target, + IReadOnlyList arguments, + CancellationToken cancellationToken, + bool throwOnError = true) + { + var startInfo = new ProcessStartInfo + { + FileName = "kubectl", + RedirectStandardError = true, + RedirectStandardOutput = true, + UseShellExecute = false, + }; + + if (!string.IsNullOrWhiteSpace(target.KubeConfigPath)) + { + startInfo.ArgumentList.Add("--kubeconfig"); + startInfo.ArgumentList.Add(target.KubeConfigPath); + } + + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + using var process = StartProcess( + startInfo, + "kubectl", + "Install kubectl or ensure it is available on PATH before using RunWithKubernetes."); + var stdoutTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + var stderrTask = process.StandardError.ReadToEndAsync(cancellationToken); + await process.WaitForExitAsync(cancellationToken); + + if (throwOnError && process.ExitCode != 0) + { + throw new InvalidOperationException( + $"kubectl failed with exit code {process.ExitCode}.{Environment.NewLine}" + + $"{await stdoutTask}{Environment.NewLine}{await stderrTask}"); + } + + return process.ExitCode; + } + + private static void ConfigureOperatorPod( + KubernetesResource kubernetes, + string name, + KubeOpsKubernetesManifestOptions options) + { + var podSpec = kubernetes.Workload?.PodTemplate?.Spec; + if (podSpec is null) + { + return; + } + + podSpec.ServiceAccountName = options.ServiceAccountName; + + var container = podSpec.Containers.FirstOrDefault(container => container.Name == name) + ?? podSpec.Containers.FirstOrDefault(); + if (container is null || container.Env.Any(env => env.Name == "POD_NAMESPACE")) + { + return; + } + + container.Env.Add( + new EnvVarV1 + { + Name = "POD_NAMESPACE", + ValueFrom = new EnvVarSourceV1 + { + FieldRef = new ObjectFieldSelectorV1 { FieldPath = "metadata.namespace" }, + }, + }); + } + + private static void RemoveExplicitStart(ProjectResource resource) + { + var explicitStartAnnotations = resource.Annotations + .OfType() + .Cast() + .ToList(); + + foreach (var annotation in explicitStartAnnotations) + { + resource.Annotations.Remove(annotation); + } + } + + private static IEnumerable GenerateKubeOpsResources( + string projectPath, + KubernetesResource kubernetes, + KubeOpsKubernetesManifestOptions options) + { + foreach (var json in GenerateKubeOpsJsonResources(projectPath, options)) + { + if (IsKind(json, "Deployment")) + { + MergeKubeOpsDeployment(kubernetes, json); + continue; + } + + if (!TryCreateKubernetesResource(json, options, out var resource)) + { + continue; + } + + yield return resource; + } + + if (options.IncludeServiceAccount) + { + yield return CreateServiceAccount(options); + } + } + + private static IEnumerable GenerateKubeOpsJsonResources( + string projectPath, + KubeOpsKubernetesManifestOptions options) + { + var outputPath = Path.Combine( + Path.GetTempPath(), + "kubeops-aspire", + Guid.NewGuid().ToString("N")); + + try + { + Directory.CreateDirectory(outputPath); + RunKubeOpsGenerator(projectPath, outputPath, options, "json"); + + foreach (var file in Directory.EnumerateFiles(outputPath, "*.json")) + { + var json = JsonNode.Parse(File.ReadAllText(file))?.AsObject(); + if (json is null) + { + continue; + } + + yield return json; + } + } + finally + { + try + { + Directory.Delete(outputPath, true); + } + catch (DirectoryNotFoundException) + { + // The generator did not create an output directory. + } + } + } + + private static void RunKubeOpsGenerator( + string projectPath, + string outputPath, + KubeOpsKubernetesManifestOptions options, + string format) + { + var arguments = new List(options.KubeOpsCliArguments) + { + "generate", + "operator", + "--format", + format, + "--out", + outputPath, + "--clear-out", + "--no-ansi", + "--namespace", + options.Namespace ?? $"{options.OperatorName}-system", + "--docker-image", + options.DockerImage, + "--docker-image-tag", + options.DockerImageTag, + }; + + if (!string.IsNullOrWhiteSpace(options.TargetFramework)) + { + arguments.Add("--target-framework"); + arguments.Add(options.TargetFramework); + } + + arguments.Add(options.OperatorName ?? "operator"); + arguments.Add(projectPath); + + var startInfo = new ProcessStartInfo + { + FileName = options.KubeOpsCliExecutable, + WorkingDirectory = Path.GetDirectoryName(projectPath) ?? Environment.CurrentDirectory, + RedirectStandardError = true, + RedirectStandardOutput = true, + UseShellExecute = false, + }; + + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + using var process = StartProcess( + startInfo, + options.KubeOpsCliExecutable, + $"Install the KubeOps CLI or configure {nameof(KubeOpsKubernetesManifestOptions)}." + + $"{nameof(KubeOpsKubernetesManifestOptions.KubeOpsCliExecutable)}."); + + var stdout = process.StandardOutput.ReadToEnd(); + var stderr = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"KubeOps CLI failed with exit code {process.ExitCode}.{Environment.NewLine}" + + $"{stdout}{Environment.NewLine}{stderr}"); + } + } + + private static Process StartProcess(ProcessStartInfo startInfo, string executable, string failureHint) + { + try + { + return Process.Start(startInfo) + ?? throw new InvalidOperationException($"Failed to start '{executable}'."); + } + catch (Win32Exception exception) + { + throw new InvalidOperationException( + $"Failed to start '{executable}'. {failureHint}", + exception); + } + } + + private static bool TryCreateKubernetesResource( + JsonObject json, + KubeOpsKubernetesManifestOptions options, + out BaseKubernetesResource resource) + { + resource = null!; + + var apiVersion = json["apiVersion"]?.GetValue(); + var kind = json["kind"]?.GetValue(); + if (string.IsNullOrWhiteSpace(apiVersion) || string.IsNullOrWhiteSpace(kind) || ShouldSkip(kind, options)) + { + return false; + } + + RewriteServiceAccountSubjects(json, options); + + var generated = new KubeOpsGeneratedKubernetesResource(apiVersion, kind) + { + Metadata = CreateMetadata(json["metadata"] as JsonObject), + Data = ConvertJson(json["data"]), + StringData = ConvertJson(json["stringData"]), + Type = ConvertJson(json["type"]), + Spec = ConvertJson(json["spec"]), + Rules = ConvertJson(json["rules"]), + RoleRef = ConvertJson(json["roleRef"]), + Subjects = ConvertJson(json["subjects"]), + Webhooks = ConvertJson(json["webhooks"]), + Secrets = ConvertJson(json["secrets"]), + ImagePullSecrets = ConvertJson(json["imagePullSecrets"]), + }; + + resource = generated; + return true; + } + + private static bool ShouldSkip(string kind, KubeOpsKubernetesManifestOptions options) + => kind is "Namespace" or "Kustomization" + || (!options.IncludeCrds && kind is "CustomResourceDefinition") + || (!options.IncludeRbac && kind is "ClusterRole" or "ClusterRoleBinding" or "Role" or "RoleBinding"); + + private static bool IsKind(JsonObject json, string kind) + => string.Equals(json["kind"]?.GetValue(), kind, StringComparison.Ordinal); + + private static void MergeKubeOpsDeployment(KubernetesResource kubernetes, JsonObject deployment) + { + var generatedSpec = deployment["spec"] as JsonObject; + if (generatedSpec is null) + { + return; + } + + if (kubernetes.Workload is Deployment workload) + { + if (TryGetInt32(generatedSpec["replicas"], out var replicas)) + { + workload.Spec.Replicas = replicas; + } + + if (TryGetInt32(generatedSpec["revisionHistoryLimit"], out var revisionHistoryLimit)) + { + workload.Spec.RevisionHistoryLimit = revisionHistoryLimit; + } + } + + var generatedPodSpec = generatedSpec["template"]?["spec"] as JsonObject; + var podSpec = kubernetes.Workload?.PodTemplate?.Spec; + if (generatedPodSpec is null || podSpec is null) + { + return; + } + + if (TryGetInt64(generatedPodSpec["terminationGracePeriodSeconds"], out var terminationGracePeriodSeconds)) + { + podSpec.TerminationGracePeriodSeconds = terminationGracePeriodSeconds; + } + + var generatedContainer = (generatedPodSpec["containers"] as JsonArray)? + .OfType() + .FirstOrDefault(); + var container = podSpec.Containers.FirstOrDefault(); + if (generatedContainer is null || container is null) + { + return; + } + + MergeEnvironmentVariables(container, generatedContainer["env"] as JsonArray); + MergeResourceRequirements(container, generatedContainer["resources"] as JsonObject); + } + + private static void MergeEnvironmentVariables(ContainerV1 container, JsonArray? generatedEnvironment) + { + if (generatedEnvironment is null) + { + return; + } + + foreach (var generatedVariable in generatedEnvironment.OfType()) + { + var name = generatedVariable["name"]?.GetValue(); + if (string.IsNullOrWhiteSpace(name) || container.Env.Any(env => env.Name == name)) + { + continue; + } + + var environmentVariable = new EnvVarV1 { Name = name }; + if (generatedVariable["value"]?.GetValue() is { } value) + { + environmentVariable.Value = value; + } + else if (generatedVariable["valueFrom"]?["fieldRef"] is JsonObject fieldRef + && fieldRef["fieldPath"]?.GetValue() is { } fieldPath) + { + environmentVariable.ValueFrom = new EnvVarSourceV1 + { + FieldRef = new ObjectFieldSelectorV1 + { + FieldPath = fieldPath, + }, + }; + } + + container.Env.Add(environmentVariable); + } + } + + private static void MergeResourceRequirements(ContainerV1 container, JsonObject? generatedResources) + { + if (generatedResources is null) + { + return; + } + + container.Resources ??= new ResourceRequirementsV1(); + CopyStringDictionary( + generatedResources["limits"] as JsonObject, + container.Resources.Limits); + CopyStringDictionary( + generatedResources["requests"] as JsonObject, + container.Resources.Requests); + } + + private static void CopyStringDictionary(JsonObject? source, IDictionary target) + { + if (source is null) + { + return; + } + + foreach (var item in ConvertStringDictionary(source) ?? []) + { + target[item.Key] = item.Value; + } + } + + private static bool TryGetInt32(JsonNode? node, out int value) + { + if (node is JsonValue jsonValue && jsonValue.TryGetValue(out value)) + { + return true; + } + + value = default; + return false; + } + + private static bool TryGetInt64(JsonNode? node, out long value) + { + if (node is JsonValue jsonValue && jsonValue.TryGetValue(out value)) + { + return true; + } + + value = default; + return false; + } + + private static void RewriteServiceAccountSubjects(JsonObject json, KubeOpsKubernetesManifestOptions options) + { + if (json["subjects"] is not JsonArray subjects) + { + return; + } + + foreach (var subject in subjects.OfType()) + { + if (subject["kind"]?.GetValue() is not "ServiceAccount") + { + continue; + } + + subject["name"] = options.ServiceAccountName; + subject["namespace"] = options.Namespace; + } + } + + private static BaseKubernetesResource CreateServiceAccount(KubeOpsKubernetesManifestOptions options) + { + var metadata = new ObjectMetaV1 { Name = options.ServiceAccountName }; + if (options.Namespace is { } namespaceName) + { + metadata.Namespace = namespaceName; + } + + return new KubeOpsGeneratedKubernetesResource("v1", "ServiceAccount") + { + Metadata = metadata, + }; + } + + private static ObjectMetaV1 CreateMetadata(JsonObject? metadata) + { + var result = new ObjectMetaV1(); + if (metadata?["name"]?.GetValue() is { } name) + { + result.Name = name; + } + + if (metadata?["namespace"]?.GetValue() is { } namespaceName) + { + result.Namespace = namespaceName; + } + + if (ConvertStringDictionary(metadata?["labels"] as JsonObject) is { } labels) + { + result.Labels = labels; + } + + if (ConvertStringDictionary(metadata?["annotations"] as JsonObject) is { } annotations) + { + foreach (var annotation in annotations) + { + result.Annotations[annotation.Key] = annotation.Value; + } + } + + return result; + } + + private static Dictionary? ConvertStringDictionary(JsonObject? json) + => json?.ToDictionary( + property => property.Key, + property => property.Value?.GetValue() ?? string.Empty); + + private static object? ConvertJson(JsonNode? node) + => node switch + { + null => null, + JsonObject jsonObject => jsonObject.ToDictionary( + property => property.Key, + property => ConvertJson(property.Value)), + JsonArray jsonArray => jsonArray.Select(ConvertJson).ToList(), + JsonValue jsonValue when jsonValue.TryGetValue(out var value) => value, + JsonValue jsonValue when jsonValue.TryGetValue(out var value) => value, + JsonValue jsonValue when jsonValue.TryGetValue(out var value) => value, + JsonValue jsonValue when jsonValue.TryGetValue(out var value) => value, + _ => node.ToJsonString(), + }; +} diff --git a/src/KubeOps.Aspire.Hosting/KubeOpsKubernetesManifestOptions.cs b/src/KubeOps.Aspire.Hosting/KubeOpsKubernetesManifestOptions.cs new file mode 100644 index 000000000..075624d67 --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOpsKubernetesManifestOptions.cs @@ -0,0 +1,145 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace Aspire.Hosting; + +/// +/// Options for adding KubeOps generated Kubernetes resources to Aspire's +/// Kubernetes publish output. +/// +public sealed class KubeOpsKubernetesManifestOptions +{ + /// + /// Gets or sets the executable used to invoke the KubeOps CLI. + /// + public string KubeOpsCliExecutable { get; set; } = "kubeops"; + + /// + /// Gets the arguments placed before the KubeOps command. + /// + /// + /// This is useful when invoking a local tool or project, for example + /// dotnet tool run kubeops -- or + /// dotnet run --project ./src/KubeOps.Cli/KubeOps.Cli.csproj --. + /// + public IList KubeOpsCliArguments { get; } = []; + + /// + /// Gets or sets the Kubernetes namespace used in generated RBAC subjects. + /// + public string? Namespace { get; set; } + + /// + /// Gets or sets the Kubernetes service account used by the Aspire generated workload. + /// + public string ServiceAccountName { get; set; } = "default"; + + /// + /// Gets a value indicating whether generated CRDs are included in the Kubernetes output. + /// + public bool IncludeCrds { get; private set; } = true; + + /// + /// Gets a value indicating whether generated RBAC resources are included in the Kubernetes output. + /// + public bool IncludeRbac { get; private set; } = true; + + /// + /// Gets a value indicating whether the configured service account is included in the Kubernetes output. + /// + public bool IncludeServiceAccount { get; private set; } = true; + + /// + /// Gets or sets the operator name passed to the KubeOps generator. + /// + public string? OperatorName { get; set; } + + /// + /// Gets or sets the target framework passed to the KubeOps generator. + /// + public string? TargetFramework { get; set; } + + /// + /// Gets or sets the docker image name passed to the KubeOps generator. + /// + public string DockerImage { get; set; } = "operator"; + + /// + /// Gets or sets the docker image tag passed to the KubeOps generator. + /// + public string DockerImageTag { get; set; } = "latest"; + + /// + /// Configures the command used to invoke the KubeOps CLI. + /// + /// The executable to run. + /// Optional arguments to place before the KubeOps command. + /// The configured options. + public KubeOpsKubernetesManifestOptions UseKubeOpsCli(string executable, params string[] arguments) + { + KubeOpsCliExecutable = executable; + KubeOpsCliArguments.Clear(); + + foreach (var argument in arguments) + { + KubeOpsCliArguments.Add(argument); + } + + return this; + } + + /// + /// Includes generated CRDs in the Kubernetes output. + /// + /// The configured options. + public KubeOpsKubernetesManifestOptions GenerateCrds() + { + IncludeCrds = true; + return this; + } + + /// + /// Excludes generated CRDs from the Kubernetes output. + /// + /// The configured options. + public KubeOpsKubernetesManifestOptions SkipCrds() + { + IncludeCrds = false; + return this; + } + + /// + /// Includes generated RBAC resources in the Kubernetes output. + /// + /// The configured options. + public KubeOpsKubernetesManifestOptions GenerateRbac() + { + IncludeRbac = true; + return this; + } + + /// + /// Excludes generated RBAC resources from the Kubernetes output. + /// + /// The configured options. + public KubeOpsKubernetesManifestOptions SkipRbac() + { + IncludeRbac = false; + return this; + } + + /// + /// Configures the service account used by the Aspire generated workload. + /// + /// The service account name. + /// The configured options. + public KubeOpsKubernetesManifestOptions WithServiceAccount(string name) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + ServiceAccountName = name; + IncludeServiceAccount = true; + return this; + } +} diff --git a/src/KubeOps.Aspire.Hosting/KubeOpsPublishAnnotation.cs b/src/KubeOps.Aspire.Hosting/KubeOpsPublishAnnotation.cs new file mode 100644 index 000000000..85d14c5cf --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOpsPublishAnnotation.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting; + +internal sealed class KubeOpsPublishAnnotation( + string projectPath, + KubeOpsKubernetesManifestOptions options) : IResourceAnnotation +{ + public string ProjectPath { get; } = projectPath; + + public KubeOpsKubernetesManifestOptions Options { get; } = options; +} diff --git a/src/KubeOps.Aspire.Hosting/KubeOpsRunAnnotation.cs b/src/KubeOps.Aspire.Hosting/KubeOpsRunAnnotation.cs new file mode 100644 index 000000000..11b9d67d1 --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOpsRunAnnotation.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting; + +internal sealed class KubeOpsRunAnnotation(KubeOpsRunOptions options) : IResourceAnnotation +{ + public KubeOpsRunOptions Options { get; } = options; + + public ISet CreatedCrds { get; } = new HashSet(StringComparer.OrdinalIgnoreCase); +} diff --git a/src/KubeOps.Aspire.Hosting/KubeOpsRunCrdMode.cs b/src/KubeOps.Aspire.Hosting/KubeOpsRunCrdMode.cs new file mode 100644 index 000000000..f71f017e5 --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOpsRunCrdMode.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace Aspire.Hosting; + +/// +/// Defines how CRDs are handled for a local Aspire run. +/// +public enum KubeOpsRunCrdMode +{ + /// + /// Create missing CRDs before run and remove only CRDs created by this run on shutdown. + /// + Ephemeral, + + /// + /// Create or update CRDs before run and leave them in the cluster after shutdown. + /// + Persistent, + + /// + /// Require CRDs to exist before run and fail otherwise. + /// + RequireExisting, + + /// + /// Do not check or manage CRDs during run. + /// + Skip, +} diff --git a/src/KubeOps.Aspire.Hosting/KubeOpsRunOptions.cs b/src/KubeOps.Aspire.Hosting/KubeOpsRunOptions.cs new file mode 100644 index 000000000..34d2452ea --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/KubeOpsRunOptions.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using Aspire.Hosting.Kubernetes; + +namespace Aspire.Hosting; + +/// +/// Options for running a KubeOps operator as a local Aspire project process with a Kubernetes target. +/// +public sealed class KubeOpsRunOptions +{ + internal KubeOpsRunOptions(KubernetesEnvironmentResource target) + { + Target = target; + } + + /// + /// Gets the Kubernetes environment used by the local operator process. + /// + public KubernetesEnvironmentResource Target { get; } + + /// + /// Gets the CRD lifecycle mode used for local run. + /// + public KubeOpsRunCrdMode CrdMode { get; private set; } = KubeOpsRunCrdMode.Ephemeral; + + /// + /// Creates missing CRDs before run and removes only CRDs created by this run on shutdown. + /// + /// The configured options. + public KubeOpsRunOptions WithEphemeralCrds() + { + CrdMode = KubeOpsRunCrdMode.Ephemeral; + return this; + } + + /// + /// Creates or updates CRDs before run and leaves them in the cluster after shutdown. + /// + /// The configured options. + public KubeOpsRunOptions WithPersistentCrds() + { + CrdMode = KubeOpsRunCrdMode.Persistent; + return this; + } + + /// + /// Requires CRDs to exist before run and fails otherwise. + /// + /// The configured options. + public KubeOpsRunOptions RequireExistingCrds() + { + CrdMode = KubeOpsRunCrdMode.RequireExisting; + return this; + } + + /// + /// Skips CRD checks and management during run. + /// + /// The configured options. + public KubeOpsRunOptions SkipCrds() + { + CrdMode = KubeOpsRunCrdMode.Skip; + return this; + } +} diff --git a/src/KubeOps.Aspire.Hosting/README.md b/src/KubeOps.Aspire.Hosting/README.md new file mode 100644 index 000000000..95f9d069a --- /dev/null +++ b/src/KubeOps.Aspire.Hosting/README.md @@ -0,0 +1,78 @@ +# KubeOps.Aspire.Hosting + +`KubeOps.Aspire.Hosting` is the [.NET Aspire](https://learn.microsoft.com/dotnet/aspire/) **hosting** integration for KubeOps operators. It lets you orchestrate a KubeOps operator project as a resource inside a .NET Aspire AppHost. + +It is the AppHost-side counterpart to the [`KubeOps.Aspire`](https://www.nuget.org/packages/KubeOps.Aspire) service-defaults integration. + +## Usage + +The repository sample uses a dedicated `examples/AspireOperator` project so the plain `examples/Operator` sample remains independent of Aspire. + +In your Aspire AppHost project: + +```csharp +var k8s = builder.AddKubernetesEnvironment("k8s") + .WithHelm(helm => + { + helm.WithChartName("my-operator"); + helm.WithReleaseName("my-operator"); + helm.WithNamespace("operator-system"); + }); + +var apiService = builder.AddProject("apiservice"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(k8s) + .PublishAsKubernetesOperator(k8s) + .WithReference(apiService); +``` + +`AddKubeOps` behaves like a KubeOps-flavoured wrapper around the built-in `AddProject` and returns the standard `IResourceBuilder`, so every Aspire extension (`WithReference`, `WithEnvironment`, `WaitFor`, ...) works as usual. + +To complete the loop, add `AddKubeOpsServiceDefaults()` from the `KubeOps.Aspire` package in the operator project. This enables OpenTelemetry export to the Aspire dashboard and service discovery for the references wired up above. + +## Run and publish behavior + +`AddKubeOps` does not start the operator during local Aspire runs unless you opt in with `RunWithKubernetes(...)`. This prevents an operator from accidentally using the developer's current kube context. With the default run options, missing CRDs are created before startup and only CRDs created by that AppHost run are removed on shutdown. + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev, run => run.WithPersistentCrds()); +``` + +When the AppHost uses Aspire's Kubernetes publishing support, `PublishAsKubernetesOperator(k8s)` invokes `kubeops generate operator` and appends the generated CRDs, RBAC resources, and service account to the Aspire-generated Helm chart. The operator deployment itself remains Aspire-owned, but KubeOps' generated deployment settings are merged into that workload so the chart deploys one correctly wired operator deployment. + +```csharp +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(k8s, publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +If there is no Aspire Kubernetes environment, `PublishAsKubernetesOperator(...)` still participates in `aspire publish` by writing raw KubeOps YAML for the operator resource. This standalone path does not require Helm or a live Kubernetes cluster. + +```csharp +builder.AddKubeOps("operator") + .PublishAsKubernetesOperator(publish => + { + publish.Namespace = "operator-system"; + publish.WithServiceAccount("operator"); + }); +``` + +Use Azure and local targets independently when the development loop and deployment target are different: + +```csharp +var dev = builder.AddKubernetesEnvironment("dev"); +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +builder.AddKubeOps("operator") + .RunWithKubernetes(dev) + .PublishAsKubernetesOperator(aks, publish => publish.WithServiceAccount("operator")); +``` + +See the [.NET Aspire guide](https://dotnet.github.io/dotnet-operator-sdk/docs/operator/aspire) for the full picture. diff --git a/src/KubeOps.Aspire/KubeOps.Aspire.csproj b/src/KubeOps.Aspire/KubeOps.Aspire.csproj new file mode 100644 index 000000000..454afd7ab --- /dev/null +++ b/src/KubeOps.Aspire/KubeOps.Aspire.csproj @@ -0,0 +1,34 @@ + + + + net8.0;net9.0;net10.0 + + + + KubeOps.Aspire + Kubernetes Operator SDK Aspire OpenTelemetry ServiceDiscovery + + .NET Aspire service defaults for KubeOps operators. Provides a single + AddKubeOpsServiceDefaults extension that wires up OpenTelemetry (logging, + metrics and tracing with OTLP export), service discovery, HTTP resilience + and default health checks so an operator integrates cleanly with a + .NET Aspire AppHost. + + + + + + + + + + + + + + + + + + + diff --git a/src/KubeOps.Aspire/KubeOpsServiceDefaultsExtensions.cs b/src/KubeOps.Aspire/KubeOpsServiceDefaultsExtensions.cs new file mode 100644 index 000000000..d69d6739f --- /dev/null +++ b/src/KubeOps.Aspire/KubeOpsServiceDefaultsExtensions.cs @@ -0,0 +1,127 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using KubeOps.Abstractions.Builder; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace KubeOps.Aspire; + +/// +/// Adds .NET Aspire "service defaults" to a KubeOps operator: OpenTelemetry +/// (logging, metrics and tracing), OTLP export, service discovery, HTTP +/// resilience and default health checks. +/// +public static class KubeOpsServiceDefaultsExtensions +{ + /// + /// Wires up the standard Aspire service defaults for a KubeOps operator. + /// Call this once on the host builder, after AddKubernetesOperator(), + /// so the configured can be resolved and + /// the tracing source matches the operator's . + /// If you call it earlier, pass explicitly with + /// the same value as . + /// + /// The host application builder type. + /// The host application builder. + /// + /// Optional name used as the OpenTelemetry service and tracing + /// name. When null, the name is + /// taken from the registered (if + /// AddKubernetesOperator() ran first) and otherwise falls back to + /// . + /// + /// The same for chaining. + public static TBuilder AddKubeOpsServiceDefaults(this TBuilder builder, string? operatorName = null) + where TBuilder : IHostApplicationBuilder + { + var serviceName = ResolveOperatorName(builder, operatorName); + + builder.ConfigureKubeOpsOpenTelemetry(serviceName); + + builder.Services.AddHealthChecks() + + // Liveness check: the operator process is up and the host has started. + .AddCheck("self", () => HealthCheckResult.Healthy(), tags: ["live"]); + + builder.Services.AddServiceDiscovery(); + + builder.Services.ConfigureHttpClientDefaults(http => + { + // Turn on resilience by default, so calls to referenced Aspire services are retried. + http.AddStandardResilienceHandler(); + + // Resolve logical Aspire service names (e.g. "https+http://apiservice") via service discovery. + http.AddServiceDiscovery(); + }); + + return builder; + } + + /// + /// Configures OpenTelemetry logging, metrics and tracing for the operator. + /// When the OTEL_EXPORTER_OTLP_ENDPOINT environment variable (or + /// configuration key) is present, the OTLP exporter is enabled for all signals. + /// + /// The host application builder type. + /// The host application builder. + /// + /// The OpenTelemetry service name and the tracing source name to subscribe to. + /// This must match the operator name (), since + /// KubeOps registers its under that name. + /// + /// The same for chaining. + public static TBuilder ConfigureKubeOpsOpenTelemetry(this TBuilder builder, string serviceName) + where TBuilder : IHostApplicationBuilder + { + builder.Logging.AddOpenTelemetry(logging => + { + logging.IncludeFormattedMessage = true; + logging.IncludeScopes = true; + }); + + builder.Services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService(serviceName)) + .WithMetrics(metrics => metrics + .AddRuntimeInstrumentation() + .AddHttpClientInstrumentation()) + .WithTracing(tracing => tracing + .AddHttpClientInstrumentation() + + // KubeOps registers an ActivitySource named after the operator (OperatorSettings.Name). + .AddSource(serviceName)); + + if (!string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"])) + { + builder.Services.AddOpenTelemetry().UseOtlpExporter(); + } + + return builder; + } + + private static string ResolveOperatorName(IHostApplicationBuilder builder, string? operatorName) + { + if (!string.IsNullOrWhiteSpace(operatorName)) + { + return operatorName; + } + + var settings = builder.Services + .FirstOrDefault(descriptor => descriptor.ServiceType == typeof(OperatorSettings))? + .ImplementationInstance as OperatorSettings; + + return settings?.Name + ?? builder.Configuration["KubeOps:OperatorName"] + ?? builder.Environment.ApplicationName; + } +} diff --git a/src/KubeOps.Aspire/README.md b/src/KubeOps.Aspire/README.md new file mode 100644 index 000000000..2bcf28946 --- /dev/null +++ b/src/KubeOps.Aspire/README.md @@ -0,0 +1,36 @@ +# KubeOps.Aspire + +`KubeOps.Aspire` is the [.NET Aspire](https://learn.microsoft.com/dotnet/aspire/) **service defaults** integration for KubeOps operators. It is the operator-side counterpart to the [`KubeOps.Aspire.Hosting`](https://www.nuget.org/packages/KubeOps.Aspire.Hosting) AppHost integration. + +A single call wires up the cross-cutting concerns that Aspire expects from a well-behaved resource: + +- **OpenTelemetry** — logging, metrics and tracing, including the operator's `ActivitySource`. +- **OTLP export** — enabled automatically when `OTEL_EXPORTER_OTLP_ENDPOINT` is set (Aspire injects this). +- **Service discovery** — so the operator can call other Aspire resources by their logical name. +- **HTTP resilience** — a standard resilience handler on all `HttpClient` instances. +- **Health checks** — a default `self` liveness check. + +## Usage + +```csharp +using KubeOps.Aspire; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services + .AddKubernetesOperator() + .RegisterComponents(); + +builder.AddKubeOpsServiceDefaults(); + +using var host = builder.Build(); +await host.RunAsync(); +``` + +Call `AddKubeOpsServiceDefaults()` **after** `AddKubernetesOperator()` so the OpenTelemetry service and tracing source names match `OperatorSettings.Name` and the operator's reconciliation traces are captured. If you must call it earlier, pass the name explicitly (and keep it in sync with `OperatorSettings.Name`): + +```csharp +builder.AddKubeOpsServiceDefaults("my-operator"); +``` + +See the [.NET Aspire guide](https://dotnet.github.io/dotnet-operator-sdk/docs/operator/aspire) for the full picture. diff --git a/test/KubeOps.Aspire.Hosting.Test/KubeOps.Aspire.Hosting.Test.csproj b/test/KubeOps.Aspire.Hosting.Test/KubeOps.Aspire.Hosting.Test.csproj new file mode 100644 index 000000000..0e7ec3f2f --- /dev/null +++ b/test/KubeOps.Aspire.Hosting.Test/KubeOps.Aspire.Hosting.Test.csproj @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/KubeOps.Aspire.Hosting.Test/KubeOpsHosting.Test.cs b/test/KubeOps.Aspire.Hosting.Test/KubeOpsHosting.Test.cs new file mode 100644 index 000000000..9646b75bc --- /dev/null +++ b/test/KubeOps.Aspire.Hosting.Test/KubeOpsHosting.Test.cs @@ -0,0 +1,137 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Testing; + +using FluentAssertions; + +namespace KubeOps.Aspire.Hosting.Test; + +public class KubeOpsHostingTest +{ + [Fact] + public void AddKubeOps_Adds_Named_Project_Resource() + { + var builder = CreateBuilder(); + + builder.AddKubeOps("operator-test"); + + builder.Resources.OfType() + .Should().ContainSingle(resource => resource.Name == "operator-test"); + } + + [Fact] + public void AddKubeOps_Returns_Builder_For_Chaining() + { + var builder = CreateBuilder(); + + var resourceBuilder = builder.AddKubeOps("operator-test"); + + resourceBuilder.Should().NotBeNull(); + resourceBuilder.Resource.Name.Should().Be("operator-test"); + } + + [Fact] + public void AddKubeOps_Uses_Explicit_Start_Without_Run_Target() + { + var builder = CreateBuilder(); + + var resourceBuilder = builder.AddKubeOps("operator-test"); + + resourceBuilder.Resource.Annotations + .Should().Contain(annotation => annotation is ExplicitStartupAnnotation); + } + + [Fact] + public void AddKubeOps_Adds_Standalone_Publish_Step_Without_Kubernetes_Target() + { + var builder = CreateBuilder(); + + var resourceBuilder = builder.AddKubeOps("operator-test"); + + resourceBuilder.Resource.Annotations + .Should().Contain(annotation => annotation.GetType().FullName == "Aspire.Hosting.Pipelines.PipelineStepAnnotation"); + } + + [Fact] + public void RunWithKubernetes_Enables_Local_Run_With_Ephemeral_Crds_By_Default() + { + var builder = CreateBuilder(); + var kubernetes = builder.AddKubernetesEnvironment("test-k8s"); + + var resourceBuilder = builder.AddKubeOps("operator-test") + .RunWithKubernetes(kubernetes); + + resourceBuilder.Resource.Annotations + .Should().NotContain(annotation => annotation is ExplicitStartupAnnotation); + resourceBuilder.Resource.Annotations.OfType() + .Should().ContainSingle() + .Which.Options.CrdMode.Should().Be(KubeOpsRunCrdMode.Ephemeral); + } + + [Fact] + public void RunWithKubernetes_Can_Use_Persistent_Crds() + { + var builder = CreateBuilder(); + var kubernetes = builder.AddKubernetesEnvironment("test-k8s"); + + var resourceBuilder = builder.AddKubeOps("operator-test") + .RunWithKubernetes(kubernetes, run => run.WithPersistentCrds()); + + resourceBuilder.Resource.Annotations.OfType() + .Should().ContainSingle() + .Which.Options.CrdMode.Should().Be(KubeOpsRunCrdMode.Persistent); + } + + [Fact] + public void PublishAsKubernetesOperator_Configures_Service_Account() + { + var builder = CreateBuilder(); + + var resourceBuilder = builder.AddKubeOps("operator-test") + .PublishAsKubernetesOperator(publish => publish.WithServiceAccount("operator-sa")); + + resourceBuilder.Resource.Annotations.OfType() + .Should().ContainSingle() + .Which.Options.ServiceAccountName.Should().Be("operator-sa"); + } + + [Fact] + public void AddKubeOps_Defaults_Published_Service_Account_To_Resource_Name() + { + var builder = CreateBuilder(); + + var resourceBuilder = builder.AddKubeOps("operator-test"); + + resourceBuilder.Resource.Annotations.OfType() + .Should().ContainSingle() + .Which.Options.ServiceAccountName.Should().Be("operator-test"); + } + + [Fact] + public void PublishAsKubernetesOperator_Binds_To_Kubernetes_Environment() + { + var builder = CreateBuilder(); + var kubernetes = builder.AddKubernetesEnvironment("publish-k8s"); + + var resourceBuilder = builder.AddKubeOps("operator-test") + .PublishAsKubernetesOperator(kubernetes); + + resourceBuilder.Resource.GetComputeEnvironment() + .Should().BeSameAs(kubernetes.Resource); + } + + private static IDistributedApplicationTestingBuilder CreateBuilder() + => global::Aspire.Hosting.Testing.DistributedApplicationTestingBuilder + .CreateAsync() + .GetAwaiter() + .GetResult(); + + private sealed class TestProjectMetadata : IProjectMetadata + { + public string ProjectPath => typeof(TestProjectMetadata).Assembly.Location; + } +} diff --git a/test/KubeOps.Aspire.Test/KubeOps.Aspire.Test.csproj b/test/KubeOps.Aspire.Test/KubeOps.Aspire.Test.csproj new file mode 100644 index 000000000..5aee074fb --- /dev/null +++ b/test/KubeOps.Aspire.Test/KubeOps.Aspire.Test.csproj @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/test/KubeOps.Aspire.Test/KubeOpsServiceDefaults.Test.cs b/test/KubeOps.Aspire.Test/KubeOpsServiceDefaults.Test.cs new file mode 100644 index 000000000..e173bc739 --- /dev/null +++ b/test/KubeOps.Aspire.Test/KubeOpsServiceDefaults.Test.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using KubeOps.Aspire; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; + +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace KubeOps.Aspire.Test; + +public class KubeOpsServiceDefaultsTest +{ + [Fact] + public void Should_Register_OpenTelemetry_Providers() + { + var builder = Host.CreateApplicationBuilder(); + builder.AddKubeOpsServiceDefaults("test-operator"); + + using var provider = builder.Services.BuildServiceProvider(); + + provider.GetService().Should().NotBeNull(); + provider.GetService().Should().NotBeNull(); + } + + [Fact] + public void Should_Register_Self_Liveness_Health_Check() + { + var builder = Host.CreateApplicationBuilder(); + builder.AddKubeOpsServiceDefaults("test-operator"); + + using var provider = builder.Services.BuildServiceProvider(); + + var options = provider.GetRequiredService>(); + options.Value.Registrations.Should().Contain(r => r.Name == "self" && r.Tags.Contains("live")); + } + + [Fact] + public void Should_Register_Service_Discovery() + { + var builder = Host.CreateApplicationBuilder(); + builder.AddKubeOpsServiceDefaults("test-operator"); + + builder.Services.Should().Contain(descriptor => + descriptor.ServiceType.Namespace != null && + descriptor.ServiceType.Namespace.StartsWith("Microsoft.Extensions.ServiceDiscovery")); + } +}