diff --git a/deploy/charts/nebulous/Chart.yaml b/deploy/charts/nebulous/Chart.yaml index 7620500..ed3b6fe 100644 --- a/deploy/charts/nebulous/Chart.yaml +++ b/deploy/charts/nebulous/Chart.yaml @@ -7,10 +7,10 @@ home: https://github.com/agentsea/nebulous # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.2.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.1.61" +appVersion: "0.1.86" diff --git a/deploy/charts/nebulous/README.md b/deploy/charts/nebulous/README.md index 0b68418..a7e8437 100644 --- a/deploy/charts/nebulous/README.md +++ b/deploy/charts/nebulous/README.md @@ -1,6 +1,6 @@ # nebulous -![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.61](https://img.shields.io/badge/AppVersion-0.1.61-informational?style=flat-square) +![Version: 0.2.1](https://img.shields.io/badge/Version-0.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.75](https://img.shields.io/badge/AppVersion-0.1.75-informational?style=flat-square) A cross-cloud container orchestrator for AI workloads @@ -19,6 +19,27 @@ encryptionKey: encodedValue: "" ``` +Add the Tailscale API key and auth key: +```yaml +tailscale: + apiKey: + authKey: +``` + +The integrated Redis database requires an auth key for Tailscale as well: +```yaml +redis: + create: true + tailscale: + authKey: +``` + +Finally, enable the creation of the integrated Postgres database: +```yaml +postgres: + create: true +``` + Add the nebulous chart repository and install the chart into a dedicated namespace: ```bash @@ -27,24 +48,65 @@ helm install nebulous nebulous/nebulous -f values.yaml \ --namespace nebulous --create-namespace ``` +## Credential secrets + +In production, the encryption key and Tailscale keys should be provided as Kubernetes secrets +and not as Helm chart values. + +You can use the following template to create them. +This template assumes installation in the `nebulous` namespace +and the secret names and keys as defined in the Helm chart's default [values.yaml](./values.yaml). + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: nebulous-secret + namespace: nebulous +data: + ENCRYPTION_KEY: +--- +apiVersion: v1 +kind: Secret +metadata: + name: tailscale-secret + namespace: nebulous +stringData: + API_KEY: "" + AUTH_KEY: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: tailscale-redis-secret + namespace: nebulous +data: + AUTH_KEY: "" +``` + ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| bucket.name | string | `"nebulous-rs"` | The name of the bucket to use for Nebulous. | +| bucket.region | string | `"us-east-1"` | The region of the bucket to use for Nebulous. | | encryptionKey.encodedValue | string | `""` | The 32 byte encryption key encoded in base64. Not recommended for production. | | encryptionKey.secret.keys.encryptionKey | string | `"ENCRYPTION_KEY"` | The key in the secret containing the encryption key. | | encryptionKey.secret.name | string | `"nebulous-secret"` | The name of the secret containing the 32 byte encryption key. | +| extraEnv | list | `[]` | Additional environment variables to pass to the Nebulous server container. | | headscale.create | bool | `false` | If true, create a Headscale deployment and service. Overrides tailscale configuration. Not recommended for production. | | headscale.derp | object | `{"configMap":{"key":"servers.yaml","name":""},"externalMaps":[]}` | The Headscale DERP configuration. Either 'externalMapUrls' or 'configMap' must be set. | | headscale.derp.configMap.key | string | `"servers.yaml"` | The key in the ConfigMap containing the DERP server configuration YAML file. | | headscale.derp.configMap.name | string | `""` | The name of the ConfigMap containing the DERP server configuration. | | headscale.derp.externalMaps | list | `[]` | URLs of externally available DERP maps encoded in JSON. | -| headscale.dns.base_domain | string | `""` | The base domain for MagicDNS hostnames. Cannot be the same as the Headscale server's domain. Refer to https://github.com/juanfont/headscale/blob/main/config-example.yaml for details. | -| headscale.domain | string | `""` | The domain under which the Headscale server is exposed. | +| headscale.dns.baseDomain | string | `""` | The base domain for MagicDNS hostnames. Cannot be the same as the Headscale server's domain. Refer to https://github.com/juanfont/headscale/blob/main/config-example.yaml for details. | +| headscale.domain | string | `""` | The domain under which the Headscale server is exposed. Required if create is true. The headscale server must be reachable at https://${domain}:443. | | headscale.imageTag | string | `"latest"` | The Headscale image tag. | | headscale.ingress.annotations | object | `{}` | Annotations to add to the Ingress resource. | | headscale.ingress.enabled | bool | `false` | If enabled, create an Ingress resource. Ignored unless 'enabled' is true. | | headscale.ingress.ingressClassName | string | `""` | The ingress class. | +| headscale.log.format | string | `"text"` | The log format of the Headscale server. Options are "text" or "json". | +| headscale.log.level | string | `"info"` | The log level of the Headscale server. Options are "off", "trace", "debug", "info", "warn", "error". | | headscale.namespaceOverride | string | `""` | Namespace override for the Headscale deployment. | | headscale.prefixes | object | `{"v4":"100.64.0.0/10","v6":"fd7a:115c:a1e0::/48"}` | Prefixes to allocate tailaddresses from. Must be within the IP ranges supported by the Tailscale client. Refer to https://github.com/juanfont/headscale/blob/main/config-example.yaml for details. | | headscale.privateKeys.claimName | string | `"headscale-keys-pvc"` | The name of the PersistentVolumeClaim for the Headscale private keys. | @@ -54,21 +116,32 @@ helm install nebulous nebulous/nebulous -f values.yaml \ | headscale.service.annotations | object | `{}` | The annotations to add to the Kubernetes service. | | headscale.service.nameOverride | string | `""` | Override the name of the Kubernetes service. | | headscale.service.port | int | `80` | The port of the Kubernetes service. | +| headscale.service.type | string | `"ClusterIP"` | The type of the Kubernetes service. Options are "ClusterIP", "NodePort", and "LoadBalancer". | | headscale.sqlite.claimName | string | `"headscale-sqlite-pvc"` | The name of the PersistentVolumeClaim for the Headscale sqlite database. | | headscale.sqlite.createPersistentVolumeClaim | bool | `true` | If true, create a PersistentVolumeClaim for the Headscale sqlite database. | | headscale.sqlite.size | string | `"10Gi"` | The size of the PersistentVolumeClaim created for the Headscale sqlite database. | | headscale.sqlite.storageClassName | string | `""` | The storage class of the PersistentVolumeClaim created for the Headscale sqlite database. | +| headscale.tls.letsencrypt.claimName | string | `"headscale-tls-pvc"` | The name of the PersistentVolumeClaim for the Headscale Let's Encrypt cache. | +| headscale.tls.letsencrypt.createPersistentVolumeClaim | bool | `true` | If true, create a PersistentVolumeClaim for the Headscale Let's Encrypt cache. | +| headscale.tls.letsencrypt.email | string | `""` | The email address for the Let's Encrypt certificate. | +| headscale.tls.letsencrypt.hostname | string | `""` | The hostname for the Let's Encrypt certificate. Has to match the domain of the Headscale server. | +| headscale.tls.letsencrypt.size | string | `"16Mi"` | The size of the PersistentVolumeClaim created for the Headscale Let's Encrypt cache. | +| headscale.tls.letsencrypt.storageClassName | string | `""` | The storage class of the PersistentVolumeClaim created for the Headscale Let's Encrypt cache. | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"us-docker.pkg.dev/agentsea-dev/nebulous/server"` | The repository to pull the server image from. | | image.tag | string | `""` | The nebulous image tag. Defaults to the Helm chart's appVersion. | | ingress.annotations | object | `{}` | Annotations to add to the Ingress resource. | | ingress.enabled | bool | `false` | If enabled, create an Ingress resource. | -| ingress.host | string | `""` | The host field of the Ingress rule. | | ingress.ingressClassName | string | `""` | The ingress class. | | local.enabled | bool | `false` | If enabled, nebulous can run Pods in the local cluster. | | logLevel | string | `"info"` | The log level of the Nebulous server. Options are "off", "trace", "debug", "info", "warn", "error". | | messageQueue.type | string | `"redis"` | The message queue type. The currently only supported value is "redis". | | namespaceOverride | string | `""` | Override the namespace. By default, Nebulous is deployed to the Helm release's namespace. | +| openmeter.enabled | bool | `false` | Enable usage monitoring with OpenMeter. | +| openmeter.secret.keys.token | string | `"TOKEN"` | The key in the eecret containing the OpenMeter API token. | +| openmeter.secret.name | string | `"openmeter-secret"` | The name of the secrets containing the OpenMeter API token. | +| openmeter.token | string | `""` | The OpenMeter API token. Not recommended for production. | +| openmeter.url | string | `"https://openmeter.cloud"` | The URL to report OpenMeter data to. | | postgres.auth | object | `{"database":"nebulous","host":"","password":"nebulous","port":5432,"user":"nebulous"}` | Manual configuration of the Postgres connection. Except for 'host', this information is also used if 'create' is true. | | postgres.create | bool | `false` | If enabled, create a Postgres deployment and service. Not recommended for production. | | postgres.imageTag | string | `"latest"` | The postgres image tag. Ignored unless 'create' is true. | @@ -77,29 +150,31 @@ helm install nebulous nebulous/nebulous -f values.yaml \ | postgres.persistence.enabled | bool | `false` | If enabled, use a PersistentVolumeClaim for the Postgres data. Ignored unless 'create' is true. | | postgres.persistence.size | string | `"100Gi"` | The size of the PersistentVolumeClaim for the Postgres data. | | postgres.persistence.storageClassName | string | `""` | The storage class of the PersistentVolumeClaim for the Postgres data. | -| postgres.secret.keys.connection_string | string | `"CONNECTION_STRING"` | The key in the secret containing the Postgres connection string. | +| postgres.secret.keys.connectionString | string | `"CONNECTION_STRING"` | The key in the secret containing the Postgres connection string. | | postgres.secret.name | string | `"postgres-secret"` | Name of the secret with the Postgres connection string. | | providers.aws.auth | object | `{"accessKeyId":"","secretAccessKey":""}` | Manual configuration of the AWS credentials. Not recommended for production. | | providers.aws.enabled | bool | `false` | Enable access to AWS. | | providers.aws.secret.keys.accessKeyId | string | `"AWS_ACCESS_KEY_ID"` | The key in the secret containing the access key ID. | | providers.aws.secret.keys.secretAccessKey | string | `"AWS_SECRET_ACCESS_KEY"` | The key in the secret containing the secret access key. | | providers.aws.secret.name | string | `"aws-secret"` | The name of the secret containing the AWS credentials. | -| providers.runpod.auth | object | `{"apiKey":""}` | Manual configuration of the Runpod API key. Not recommended for production. | +| providers.runpod.auth | object | `{"apiKey":"","containerRegistryAuthId":""}` | Manual configuration of the Runpod credentials. Not recommended for production. | | providers.runpod.enabled | bool | `false` | Enable access to Runpod. | | providers.runpod.secret.keys.apiKey | string | `"RUNPOD_API_KEY"` | The key in the secret containing the API key. | -| providers.runpod.secret.name | string | `"runpod-secret"` | The name of the secret containing the API key. | +| providers.runpod.secret.keys.containerRegistryAuthId | string | `"RUNPOD_CONTAINER_REGISTRY_AUTH_ID"` | The key in the secret containing the container registry auth ID. | +| providers.runpod.secret.name | string | `"runpod-secret"` | The name of the secret containing the Runpod credentials. | | redis.auth | object | `{"database":0,"host":"","password":"nebulous","port":6379}` | Manual configuration of the Redis connection. Except for 'host', this information is also used if 'create' is true. | | redis.create | bool | `false` | If enabled, create a Redis deployment and service. Not recommended for production. | | redis.imageTag | string | `"latest"` | The redis image tag. Ignored unless 'create' is true. | -| redis.ingress.annotations | object | `{}` | Annotations to add to the Ingress resource. | -| redis.ingress.enabled | bool | `false` | If enabled, create an Ingress resource. Ignored unless 'create' is true. | -| redis.ingress.host | string | `""` | The host field of the Ingress rule. | -| redis.ingress.ingressClassName | string | `""` | The ingress class. | -| redis.secret.keys.connection_string | string | `"CONNECTION_STRING"` | The key in the secret containing the Redis connection string. | +| redis.secret.keys.connectionString | string | `"CONNECTION_STRING"` | The key in the secret containing the Redis connection string. | | redis.secret.keys.password | string | `"PASSWORD"` | The key in the secret containing the Redis password. | | redis.secret.name | string | `"redis-secret"` | Name of the secret with the Redis connection string and password. | | redis.service.annotations | object | `{}` | The annotations to add to the Kubernetes service. | | redis.service.nameOverride | string | `""` | Override the name of the Kubernetes service. | +| redis.serviceAccountName | string | `"redis"` | The name of the Kubernetes service account for the Redis Pod. | +| redis.tailscale.authKey | string | `""` | The Tailscale auth key for Redis. If headscale.enabled is true, this is ignored. | +| redis.tailscale.secret.keys.authKey | string | `"AUTH_KEY"` | The key in the secret containing the Tailscale auth key. | +| redis.tailscale.secret.name | string | `"tailscale-redis-secret"` | Name of the secret with the Tailscale auth key for Redis. | +| rootOwner | string | `"agentsea"` | The owner of the Nebulous root. | | service.annotations | object | `{}` | Annotations to add to the Kubernetes service. | | service.nameOverride | string | `""` | Override the name of the Kubernetes service. | | service.port | int | `3000` | The port of the Kubernetes service. | @@ -123,8 +198,7 @@ helm install nebulous nebulous/nebulous -f values.yaml \ | tailscale.apiKey | string | `""` | The Tailscale API key. If headscale.enabled is true, this is ignored. | | tailscale.authKey | string | `""` | The Tailscale auth key. If headscale.enabled is true, this is ignored. | | tailscale.loginServer | string | `"https://login.tailscale.com"` | The Tailscale host to connect to. If headscale.enabled is true, this is ignored. | -| tailscale.secret.keys.apiKey | string | `"API_KEY"` | The key in the secret containing the Tailscale API key | -| tailscale.secret.keys.authKey | string | `"AUTH_KEY"` | The key in the secret containing the Tailscale auth key | -| tailscale.secret.keys.loginServer | string | `"LOGIN_SERVER"` | The key in the secret containing the Tailscale host. | -| tailscale.secret.name | string | `"tailscale-secret"` | Name of the secret with the Redis connection string and password. | +| tailscale.secret.keys.apiKey | string | `"API_KEY"` | The key in the secret containing the Tailscale API key. | +| tailscale.secret.keys.authKey | string | `"AUTH_KEY"` | The key in the secret containing the Tailscale auth key. | +| tailscale.secret.name | string | `"tailscale-secret"` | Name of the secret with the Tailscale auth key and API key. | diff --git a/deploy/charts/nebulous/README.md.gotmpl b/deploy/charts/nebulous/README.md.gotmpl index a686ec0..79199c3 100644 --- a/deploy/charts/nebulous/README.md.gotmpl +++ b/deploy/charts/nebulous/README.md.gotmpl @@ -21,6 +21,27 @@ encryptionKey: encodedValue: "" ``` +Add the Tailscale API key and auth key: +```yaml +tailscale: + apiKey: + authKey: +``` + +The integrated Redis database requires an auth key for Tailscale as well: +```yaml +redis: + create: true + tailscale: + authKey: +``` + +Finally, enable the creation of the integrated Postgres database: +```yaml +postgres: + create: true +``` + Add the nebulous chart repository and install the chart into a dedicated namespace: ```bash @@ -29,6 +50,42 @@ helm install nebulous {{ $appName }}/{{ template "chart.name" . }} -f values.yam --namespace nebulous --create-namespace ``` +## Credential secrets + +In production, the encryption key and Tailscale keys should be provided as Kubernetes secrets +and not as Helm chart values. + +You can use the following template to create them. +This template assumes installation in the `nebulous` namespace +and the secret names and keys as defined in the Helm chart's default [values.yaml](./values.yaml). + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: nebulous-secret + namespace: nebulous +data: + ENCRYPTION_KEY: +--- +apiVersion: v1 +kind: Secret +metadata: + name: tailscale-secret + namespace: nebulous +stringData: + API_KEY: "" + AUTH_KEY: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: tailscale-redis-secret + namespace: nebulous +data: + AUTH_KEY: "" +``` + {{ template "chart.valuesSection" . }} {{ template "helm-docs.versionFooter" . }} \ No newline at end of file diff --git a/deploy/charts/nebulous/templates/NOTES.txt b/deploy/charts/nebulous/templates/NOTES.txt index cd9b263..fde040f 100644 --- a/deploy/charts/nebulous/templates/NOTES.txt +++ b/deploy/charts/nebulous/templates/NOTES.txt @@ -16,15 +16,6 @@ WARNING: Using the integrated Postgres database. This is not recommended for pro {{- end }} {{ if .Values.redis.create }} Internal Redis endpoint: {{ printf "redis://%s:%d" (include "redis.host" . ) (int .Values.redis.auth.port) }} -{{- if .Values.redis.ingress.enabled }} -{{- if .Values.redis.ingress.host }} -External Redis endpoint: {{ printf "redis://%s" .Values.redis.ingress.host }} -{{- else }} -External Redis endpoint: .Values.redis.ingress.host not specified, refer to Ingress configuration provided through values.yaml. -{{- end }} -{{- else }} -External Redis endpoint: Set .Values.redis.ingress.enabled to true to expose Redis externally. -{{- end }} {{- end }} {{ if .Values.encryptionKey.encodedValue }} WARNING: Encryption key is not provided through a user-managed secret. This is not recommended for production. diff --git a/deploy/charts/nebulous/templates/_helpers.tpl b/deploy/charts/nebulous/templates/_helpers.tpl index 3deb881..647e801 100644 --- a/deploy/charts/nebulous/templates/_helpers.tpl +++ b/deploy/charts/nebulous/templates/_helpers.tpl @@ -33,6 +33,10 @@ app.kubernetes.io/instance: {{ .Release.Name | trunc 63 | trimSuffix "-" }} {{- printf "%s-local-role" .Release.Name }} {{- end }} +{{- define "nebulous.tailscaleStateSecretName" -}} +tailscale-{{- include "nebulous.serviceAccountName" . }}-state-secret +{{- end }} + {{- define "headscale.name" -}} headscale {{- end }} @@ -46,7 +50,15 @@ headscale {{- end }} {{- define "headscale.host" -}} -{{- include "headscale.serviceName" . }}.{{- include "headscale.namespace" . }}.svc.cluster.local +https://{{- required ".Values.headscale.domain is required" .Values.headscale.domain }} +{{- end }} + +{{- define "tailscale.loginServer" }} +{{- if .Values.headscale.create }} +{{- include "headscale.host" . }} +{{- else }} +{{- required ".Values.tailscale.loginServer is required" .Values.tailscale.loginServer }} +{{- end }} {{- end }} {{- define "postgres.name" -}} @@ -76,3 +88,7 @@ redis {{- required ".Values.redis.auth.host is required" .Values.redis.auth.host }} {{- end }} {{- end }} + +{{- define "redis.tailscaleStateSecretName" -}} +tailscale-{{- include "redis.name" . }}-state-secret +{{- end }} diff --git a/deploy/charts/nebulous/templates/deployment.yaml b/deploy/charts/nebulous/templates/deployment.yaml index f21a147..86484e9 100644 --- a/deploy/charts/nebulous/templates/deployment.yaml +++ b/deploy/charts/nebulous/templates/deployment.yaml @@ -28,6 +28,10 @@ spec: {{- end }} ports: - containerPort: 3000 + command: + - "/bin/sh" + - "-c" + - "exec nebu serve --host 0.0.0.0 --port 3000" env: - name: NEBU_ENCRYPTION_KEY valueFrom: @@ -38,7 +42,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.postgres.secret.name }} - key: {{ .Values.postgres.secret.keys.connection_string }} + key: {{ .Values.postgres.secret.keys.connectionString }} - name: MESSAGE_QUEUE_TYPE value: {{ .Values.messageQueue.type }} {{- if eq .Values.messageQueue.type "redis" }} @@ -46,7 +50,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.redis.secret.name }} - key: {{ .Values.redis.secret.keys.connection_string }} + key: {{ .Values.redis.secret.keys.connectionString }} - name: REDIS_PASSWORD valueFrom: secretKeyRef: @@ -62,31 +66,46 @@ spec: - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: {{ .Values.providers.aws.secret.name }} + name: {{ .Values.providers.aws.secret.name }} key: {{ .Values.providers.aws.secret.keys.secretAccessKey }} {{- end }} {{- if .Values.providers.runpod.enabled }} - name: RUNPOD_API_KEY valueFrom: secretKeyRef: - name: {{ .Values.providers.runpod.secret.name }} + name: {{ .Values.providers.runpod.secret.name }} key: {{ .Values.providers.runpod.secret.keys.apiKey }} + - name: RUNPOD_CONTAINER_REGISTRY_AUTH_ID + valueFrom: + secretKeyRef: + name: {{ .Values.providers.runpod.secret.name }} + key: {{ .Values.providers.runpod.secret.keys.containerRegistryAuthId }} {{- end }} - - name: TS_AUTHKEY + - name: RUST_LOG + value: {{ .Values.logLevel | lower }} + - name: NEBU_BUCKET_NAME + value: {{ .Values.bucket.name }} + - name: NEBU_BUCKET_REGION + value: {{ .Values.bucket.region }} + - name: NEBU_ROOT_OWNER + value: {{ .Values.rootOwner }} + - name: TS_API_KEY valueFrom: secretKeyRef: name: {{ .Values.tailscale.secret.name }} - key: {{ .Values.tailscale.secret.keys.authKey }} - - name: TS_LOGINSERVER + key: {{ .Values.tailscale.secret.keys.apiKey }} + {{- if .Values.openmeter.enabled }} + - name: OPENMETER_URL + value: {{ .Values.openmeter.url }} + - name: OPENMETER_TOKEN valueFrom: - secretKeyRef: - name: {{ .Values.tailscale.secret.name }} - key: {{ .Values.tailscale.secret.keys.loginServer }} - - name: RUST_LOG - value: {{ .Values.logLevel | lower }} - envFrom: - - secretRef: - name: {{ .Values.tailscale.secret.name }} + secretKeyRef: + name: {{ .Values.openmeter.secret.name }} + key: {{ .Values.openmeter.secret.keys.token }} + {{- end }} +{{- with .Values.extraEnv }} +{{ toYaml . | indent 12 }} +{{- end }} volumeMounts: - name: huggingface-pvc mountPath: /huggingface @@ -96,6 +115,48 @@ spec: mountPath: /datasets - name: model-pvc mountPath: /models + # Share /var/run/tailscale and /tmp to provide nebulous access to the Tailscale daemon + - name: var-run-tailscale + mountPath: /var/run/tailscale + - name: tmp + mountPath: /tmp + # Reference: https://github.com/tailscale/tailscale/blob/main/docs/k8s/sidecar.yaml + # Docs: https://tailscale.com/kb/1185/kubernetes + - name: tailscale + image: tailscale/tailscale:latest + imagePullPolicy: Always + env: + - name: TS_USERSPACE + value: "false" + - name: TS_DEBUG_FIREWALL_MODE + value: auto + - name: TS_AUTH_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.tailscale.secret.name }} + key: {{ .Values.tailscale.secret.keys.authKey }} + - name: TS_HOSTNAME + value: "nebu" + - name: TS_EXTRA_ARGS + value: --login-server {{ include "tailscale.loginServer" . }} + - name: TS_KUBE_SECRET + value: {{ include "nebulous.tailscaleStateSecretName" . }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + securityContext: + privileged: true + volumeMounts: + # Share /var/run/tailscale and /tmp to provide nebulous access to the Tailscale daemon + - name: var-run-tailscale + mountPath: /var/run/tailscale + - name: tmp + mountPath: /tmp volumes: - name: huggingface-pvc persistentVolumeClaim: @@ -109,3 +170,7 @@ spec: - name: model-pvc persistentVolumeClaim: claimName: {{ .Values.storage.model.claimName }} + - name: var-run-tailscale + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/deploy/charts/nebulous/templates/headscale.yaml b/deploy/charts/nebulous/templates/headscale.yaml index 43382ec..e220217 100644 --- a/deploy/charts/nebulous/templates/headscale.yaml +++ b/deploy/charts/nebulous/templates/headscale.yaml @@ -11,15 +11,20 @@ data: server_url: {{ printf "https://%s:443" .Values.headscale.domain }} listen_addr: 0.0.0.0:8080 metrics_listen_addr: 0.0.0.0:9090 - # TLS termination happens at the Ingress level - # (see https://headscale.net/stable/ref/integration/reverse-proxy/#tls) - tls_cert_path: "" - tls_key_path: "" noise: private_key_path: /mnt/keys/noise_private.key prefixes: v4: 100.64.0.0/10 v6: fd7a:115c:a1e0::/48 + {{- if .Values.headscale.tls.letsencrypt.hostname }} + tls_letsencrypt_hostname: {{ .Values.headscale.tls.letsencrypt.hostname }} + acme_email: {{ .Values.headscale.tls.letsencrypt.email }} + tls_letsencrypt_listen: ":http" + tls_letsencrypt_cache_dir: /mnt/letsencrypt + tls_letsencrypt_challenge_type: HTTP-01 + {{- end }} + tls_cert_path: "" + tls_key_path: "" database: type: sqlite derp: @@ -33,7 +38,10 @@ data: - /mnt/derp/{{ .Values.headscale.derp.configMap.key }} {{- end }} dns: - base_domain: {{ .Values.headscale.dns.base_domain }} + base_domain: {{ .Values.headscale.dns.baseDomain }} + log: + format: {{ .Values.headscale.log.format }} + level: {{ .Values.headscale.log.level }} --- {{- if .Values.headscale.sqlite.createPersistentVolumeClaim }} apiVersion: v1 @@ -73,6 +81,25 @@ spec: storage: {{ .Values.headscale.privateKeys.size }} --- {{- end }} +{{- if and .Values.headscale.tls.letsencrypt.hostname .Values.headscale.tls.letsencrypt.createPersistentVolumeClaim }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.headscale.tls.letsencrypt.claimName }} + namespace: {{ include "headscale.namespace" . }} + labels: + {{- include "common.labels" . | nindent 4 }} +spec: + {{- with .Values.headscale.tls.letsencrypt.storageClassName }} + storageClassName: {{.}} + {{- end }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.headscale.tls.letsencrypt.size }} +--- +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -106,6 +133,10 @@ spec: name: sqlite - mountPath: /mnt/keys name: private-keys + {{- if .Values.headscale.tls.letsencrypt.hostname }} + - mountPath: /mnt/letsencrypt + name: tls-letsencrypt + {{- end }} {{- if .Values.headscale.derp.configMap.name }} - mountPath: /mnt/derp name: derp-config @@ -120,6 +151,11 @@ spec: - name: private-keys persistentVolumeClaim: claimName: {{ .Values.headscale.privateKeys.claimName }} + {{- if .Values.headscale.tls.letsencrypt.hostname }} + - name: tls-letsencrypt + persistentVolumeClaim: + claimName: {{ .Values.headscale.tls.letsencrypt.claimName }} + {{- end }} {{- if .Values.headscale.derp.configMap.name }} - name: derp-config configMap: @@ -235,10 +271,10 @@ spec: done; POD_NAME=$(kubectl get pod -l app={{ include "headscale.name" . }} -n {{ include "headscale.namespace" . }} -o jsonpath='{.items[0].metadata.name}') - kubectl exec $POD_NAME -- headscale users create nebu || echo 'User nebu already exists. That is OK.'; + kubectl exec $POD_NAME -n {{ include "headscale.namespace" . }} -- headscale users create nebu || echo 'User nebu already exists. That is OK.'; - API_KEY=$(kubectl exec $POD_NAME -- headscale apikeys create --expiration 99y); - AUTH_KEY=$(kubectl exec $POD_NAME -- headscale preauthkeys create --user nebu --reusable); + API_KEY=$(kubectl exec $POD_NAME -n {{ include "headscale.namespace" . }} -- headscale apikeys create --expiration 99y); + AUTH_KEY=$(kubectl exec $POD_NAME -n {{ include "headscale.namespace" . }} -- headscale preauthkeys create --user nebu --reusable); kubectl create secret generic {{ .Values.tailscale.secret.name }} -n {{ include "nebulous.namespace" . }} \ --from-literal={{ .Values.tailscale.secret.keys.apiKey }}=$API_KEY \ @@ -254,9 +290,9 @@ metadata: labels: {{- include "common.labels" . | nindent 4 }} {{- with .Values.headscale.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: selector: app: {{ include "headscale.name" . }} @@ -264,7 +300,7 @@ spec: - protocol: TCP port: {{ .Values.headscale.service.port }} targetPort: 8080 - type: ClusterIP + type: {{ .Values.headscale.service.type }} --- apiVersion: v1 kind: Service @@ -298,18 +334,10 @@ spec: {{- with .Values.headscale.ingress.ingressClassName }} ingressClassName: {{.}} {{- end }} - rules: - - http: - paths: - - backend: - service: - name: {{ include "headscale.name" . }} - port: - number: {{ .Values.headscale.service.port }} - path: / - pathType: Prefix - {{- with .Values.headscale.domain }} - host: {{.}} - {{- end }} + defaultBackend: + service: + name: {{ include "headscale.serviceName" . }} + port: + number: {{ .Values.headscale.service.port }} {{- end }} {{- end }} diff --git a/deploy/charts/nebulous/templates/ingress.yaml b/deploy/charts/nebulous/templates/ingress.yaml index 31edfb6..bdd3de1 100644 --- a/deploy/charts/nebulous/templates/ingress.yaml +++ b/deploy/charts/nebulous/templates/ingress.yaml @@ -11,20 +11,12 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- with .Values.ingress.className }} + {{- with .Values.ingress.ingressClassName }} ingressClassName: {{.}} {{- end }} - rules: - - http: - paths: - - backend: - service: - name: {{ include "nebulous.serviceName" . }} - port: - number: {{ .Values.service.port }} - path: / - pathType: Prefix - {{- with .Values.ingress.host }} - host: {{.}} - {{- end }} + defaultBackend: + service: + name: {{ include "nebulous.serviceName" . }} + port: + number: {{ .Values.service.port }} {{- end }} diff --git a/deploy/charts/nebulous/templates/openmeter.yaml b/deploy/charts/nebulous/templates/openmeter.yaml new file mode 100644 index 0000000..1b6db58 --- /dev/null +++ b/deploy/charts/nebulous/templates/openmeter.yaml @@ -0,0 +1,12 @@ +{{- if .Values.openmeter.token }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.openmeter.secret.name }} + namespace: {{ include "nebulous.namespace" . }} + labels: + {{- include "common.labels" . | nindent 4 }} +type: Opaque +data: + {{ .Values.openmeter.secret.keys.token }}: {{ .Values.openmeter.token | b64enc }} +{{- end }} diff --git a/deploy/charts/nebulous/templates/postgres.yaml b/deploy/charts/nebulous/templates/postgres.yaml index fc5dfa1..13956bb 100644 --- a/deploy/charts/nebulous/templates/postgres.yaml +++ b/deploy/charts/nebulous/templates/postgres.yaml @@ -8,7 +8,7 @@ metadata: {{- include "common.labels" . | nindent 4 }} type: Opaque data: - {{ .Values.postgres.secret.keys.connection_string }}: {{ printf "postgresql://%s:%s@%s:%d/%s" .Values.postgres.auth.user .Values.postgres.auth.password (include "postgres.host") (int .Values.postgres.auth.port) .Values.postgres.auth.database | b64enc }} + {{ .Values.postgres.secret.keys.connectionString }}: {{ printf "postgresql://%s:%s@%s:%d/%s" .Values.postgres.auth.user .Values.postgres.auth.password (include "postgres.host" . ) (int .Values.postgres.auth.port) .Values.postgres.auth.database | b64enc }} --- {{- end }} {{- if .Values.postgres.create }} diff --git a/deploy/charts/nebulous/templates/provider_runpod.yaml b/deploy/charts/nebulous/templates/provider_runpod.yaml index 979b186..058cb9a 100644 --- a/deploy/charts/nebulous/templates/provider_runpod.yaml +++ b/deploy/charts/nebulous/templates/provider_runpod.yaml @@ -1,5 +1,5 @@ {{- if .Values.providers.runpod.enabled }} -{{- if .Values.providers.runpod.auth.apiKey }} +{{- if or .Values.providers.runpod.auth.apiKey .Values.providers.runpod.auth.containerRegistryAuthId }} apiVersion: v1 kind: Secret metadata: @@ -9,6 +9,7 @@ metadata: {{- include "common.labels" . | nindent 4 }} type: Opaque data: - {{ .Values.providers.runpod.secret.keys.apiKey }}: {{ .Values.providers.runpod.auth.apiKey | b64enc }} + {{ .Values.providers.runpod.secret.keys.apiKey }}: {{ required ".Values.providers.runpod.auth.apiKey is required" .Values.providers.runpod.auth.apiKey | b64enc }} + {{ .Values.providers.runpod.secret.keys.containerRegistryAuthId }}: {{ required ".Values.providers.runpod.auth.containerRegistryAuthId is required" .Values.providers.runpod.auth.containerRegistryAuthId | b64enc }} {{- end }} {{- end }} diff --git a/deploy/charts/nebulous/templates/redis.yaml b/deploy/charts/nebulous/templates/redis.yaml index 557e8c7..368737f 100644 --- a/deploy/charts/nebulous/templates/redis.yaml +++ b/deploy/charts/nebulous/templates/redis.yaml @@ -8,11 +8,58 @@ metadata: {{- include "common.labels" . | nindent 4 }} type: Opaque data: - {{ .Values.redis.secret.keys.connection_string }}: {{ printf "redis://%s:%d/%d" (include "redis.host" .) (int .Values.redis.auth.port) (int .Values.redis.auth.database) | b64enc }} + {{ .Values.redis.secret.keys.connectionString }}: {{ printf "redis://:%s@%s:%d/%d" .Values.redis.auth.password (include "redis.host" .) (int .Values.redis.auth.port) (int .Values.redis.auth.database) | b64enc }} {{ .Values.redis.secret.keys.password}}: {{ .Values.redis.auth.password | b64enc }} --- {{- end }} +{{- if and .Values.redis.tailscale.authKey (not .Values.headscale.create) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.redis.tailscale.secret.name }} + namespace: {{ include "nebulous.namespace" . }} + labels: + {{- include "common.labels" . | nindent 4 }} +type: Opaque +data: + {{ .Values.redis.tailscale.secret.keys.authKey }}: {{ required ".Values.redis.tailscale.authKey is required" .Values.tailscale.authKey | b64enc }} +--- +{{- end }} {{- if .Values.redis.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.redis.serviceAccountName }} +--- +# Reference: https://github.com/tailscale/tailscale/blob/main/docs/k8s/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tailscale-{{ .Values.redis.serviceAccountName }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: [""] + resourceNames: ["{{ include "redis.tailscaleStateSecretName" . }}"] + resources: ["secrets"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tailscale-{{ .Values.redis.serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ .Values.redis.serviceAccountName }} +roleRef: + kind: Role + name: tailscale-redis + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -30,6 +77,7 @@ spec: labels: app: {{ include "redis.name" . }} spec: + serviceAccountName: {{ .Values.redis.serviceAccountName }} containers: - name: redis image: "redis:{{ .Values.redis.imageTag }}" @@ -47,6 +95,36 @@ spec: key: {{ .Values.redis.secret.keys.password }} ports: - containerPort: 6379 + # Reference: https://github.com/tailscale/tailscale/blob/main/docs/k8s/userspace-sidecar.yaml + # Docs: https://tailscale.com/kb/1185/kubernetes + - name: tailscale + image: tailscale/tailscale:latest + imagePullPolicy: Always + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + env: + - name: TS_USERSPACE + value: "true" + - name: TS_AUTH_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.redis.tailscale.secret.name }} + key: {{ .Values.redis.tailscale.secret.keys.authKey }} + - name: TS_HOSTNAME + value: "redis" + - name: TS_EXTRA_ARGS + value: --login-server {{ include "tailscale.loginServer" . }} + - name: TS_KUBE_SECRET + value: {{ include "redis.tailscaleStateSecretName" . }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid --- apiVersion: v1 kind: Service @@ -67,35 +145,4 @@ spec: port: {{ .Values.redis.auth.port }} targetPort: 6379 type: ClusterIP ---- -{{- if .Values.redis.ingress.enabled }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ include "redis.serviceName" . }} - namespace: {{ include "nebulous.namespace" . }} - labels: - {{- include "common.labels" . | nindent 4 }} - {{- with .Values.redis.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- with .Values.redis.ingress.ingressClassName }} - ingressClassName: {{.}} - {{- end }} - rules: - - http: - paths: - - backend: - service: - name: {{ include "redis.name" . }} - port: - number: {{ .Values.redis.auth.port }} - path: / - pathType: Prefix - {{- with .Values.redis.ingress.host }} - host: {{.}} - {{- end }} -{{- end }} {{- end }} diff --git a/deploy/charts/nebulous/templates/serviceaccount.yaml b/deploy/charts/nebulous/templates/serviceaccount.yaml index 145eecd..c227ada 100644 --- a/deploy/charts/nebulous/templates/serviceaccount.yaml +++ b/deploy/charts/nebulous/templates/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if .Values.serviceAccount.name }} +{{- if not .Values.serviceAccount.name }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/deploy/charts/nebulous/templates/tailscale.yaml b/deploy/charts/nebulous/templates/tailscale.yaml index a539829..2f04cb6 100644 --- a/deploy/charts/nebulous/templates/tailscale.yaml +++ b/deploy/charts/nebulous/templates/tailscale.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.tailscale.loginServer (not .Values.headscale.create) }} +{{- if and (or .Values.tailscale.apiKey .Values.tailscale.authKey) (not .Values.headscale.create) }} apiVersion: v1 kind: Secret metadata: @@ -8,7 +8,35 @@ metadata: {{- include "common.labels" . | nindent 4 }} type: Opaque data: - {{ .Values.tailscale.secret.keys.loginServer }}: {{ .Values.tailscale.loginServer | b64enc }} {{ .Values.tailscale.secret.keys.apiKey }}: {{ required ".Values.tailscale.apiKey is required" .Values.tailscale.apiKey | b64enc }} {{ .Values.tailscale.secret.keys.authKey }}: {{ required ".Values.tailscale.authKey is required" .Values.tailscale.authKey | b64enc }} +--- {{- end }} +# Reference: https://github.com/tailscale/tailscale/blob/main/docs/k8s/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tailscale-{{ include "nebulous.serviceAccountName" . }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: [""] + resourceNames: ["{{ include "nebulous.tailscaleStateSecretName" . }}"] + resources: ["secrets"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tailscale-nebu +subjects: + - kind: ServiceAccount + name: {{ include "nebulous.serviceAccountName" . }} +roleRef: + kind: Role + name: tailscale-{{ include "nebulous.serviceAccountName" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/charts/nebulous/values.yaml b/deploy/charts/nebulous/values.yaml index 55c04b0..945ba7f 100644 --- a/deploy/charts/nebulous/values.yaml +++ b/deploy/charts/nebulous/values.yaml @@ -1,6 +1,9 @@ # -- Override the namespace. By default, Nebulous is deployed to the Helm release's namespace. namespaceOverride: "" +# -- Additional environment variables to pass to the Nebulous server container. +extraEnv: [] + ## Nebulous configuration image: pullPolicy: "IfNotPresent" @@ -33,13 +36,20 @@ ingress: annotations: { } # -- The ingress class. ingressClassName: "" - # -- The host field of the Ingress rule. - host: "" local: # -- If enabled, nebulous can run Pods in the local cluster. enabled: false +bucket: + # -- The name of the bucket to use for Nebulous. + name: "nebulous-rs" + # -- The region of the bucket to use for Nebulous. + region: "us-east-1" + +# -- The owner of the Nebulous root. +rootOwner: "agentsea" + messageQueue: # -- The message queue type. The currently only supported value is "redis". type: "redis" @@ -53,14 +63,12 @@ tailscale: authKey: "" secret: - # -- Name of the secret with the Redis connection string and password. + # -- Name of the secret with the Tailscale auth key and API key. name: "tailscale-secret" keys: - # -- The key in the secret containing the Tailscale host. - loginServer: "LOGIN_SERVER" - # -- The key in the secret containing the Tailscale API key + # -- The key in the secret containing the Tailscale API key. apiKey: "API_KEY" - # -- The key in the secret containing the Tailscale auth key + # -- The key in the secret containing the Tailscale auth key. authKey: "AUTH_KEY" encryptionKey: @@ -74,6 +82,23 @@ encryptionKey: # -- The 32 byte encryption key encoded in base64. Not recommended for production. encodedValue: "" +openmeter: + # -- Enable usage monitoring with OpenMeter. + enabled: false + + # -- The URL to report OpenMeter data to. + url: https://openmeter.cloud + + secret: + # -- The name of the secrets containing the OpenMeter API token. + name: "openmeter-secret" + keys: + # -- The key in the eecret containing the OpenMeter API token. + token: "TOKEN" + + # -- The OpenMeter API token. Not recommended for production. + token: "" + ## Storage configuration storage: huggingface: @@ -111,7 +136,7 @@ postgres: name: "postgres-secret" keys: # -- The key in the secret containing the Postgres connection string. - connection_string: "CONNECTION_STRING" + connectionString: "CONNECTION_STRING" # -- Manual configuration of the Postgres connection. Except for 'host', this information is also used if 'create' is true. auth: @@ -145,7 +170,7 @@ redis: name: "redis-secret" keys: # -- The key in the secret containing the Redis connection string. - connection_string: "CONNECTION_STRING" + connectionString: "CONNECTION_STRING" # -- The key in the secret containing the Redis password. password: "PASSWORD" @@ -167,16 +192,19 @@ redis: # -- Override the name of the Kubernetes service. nameOverride: "" - ingress: - # -- If enabled, create an Ingress resource. Ignored unless 'create' is true. - enabled: false + # -- The name of the Kubernetes service account for the Redis Pod. + serviceAccountName: "redis" - # -- Annotations to add to the Ingress resource. - annotations: { } - # -- The ingress class. - ingressClassName: "" - # -- The host field of the Ingress rule. - host: "" + tailscale: + # -- The Tailscale auth key for Redis. If headscale.enabled is true, this is ignored. + authKey: "" + + secret: + # -- Name of the secret with the Tailscale auth key for Redis. + name: "tailscale-redis-secret" + keys: + # -- The key in the secret containing the Tailscale auth key. + authKey: "AUTH_KEY" ## Headscale configuration headscale: @@ -186,7 +214,8 @@ headscale: namespaceOverride: "" # -- The Headscale image tag. imageTag: "latest" - # -- The domain under which the Headscale server is exposed. + # -- The domain under which the Headscale server is exposed. Required if create is true. + # The headscale server must be reachable at https://${domain}:443. domain: "" # -- Prefixes to allocate tailaddresses from. Must be within the IP ranges supported by the Tailscale client. @@ -198,7 +227,13 @@ headscale: dns: # -- The base domain for MagicDNS hostnames. Cannot be the same as the Headscale server's domain. # Refer to https://github.com/juanfont/headscale/blob/main/config-example.yaml for details. - base_domain: "" + baseDomain: "" + + log: + # -- The log level of the Headscale server. Options are "off", "trace", "debug", "info", "warn", "error". + level: "info" + # -- The log format of the Headscale server. Options are "text" or "json". + format: "text" # -- The Headscale DERP configuration. Either 'externalMapUrls' or 'configMap' must be set. derp: @@ -232,9 +267,28 @@ headscale: # -- The storage class of the PersistentVolumeClaim created for the Headscale private keys. storageClassName: "" + tls: + letsencrypt: + # -- The hostname for the Let's Encrypt certificate. Has to match the domain of the Headscale server. + hostname: "" + # -- The email address for the Let's Encrypt certificate. + email: "" + + # -- The name of the PersistentVolumeClaim for the Headscale Let's Encrypt cache. + claimName: "headscale-tls-pvc" + + # -- If true, create a PersistentVolumeClaim for the Headscale Let's Encrypt cache. + createPersistentVolumeClaim: true + # -- The size of the PersistentVolumeClaim created for the Headscale Let's Encrypt cache. + size: "16Mi" + # -- The storage class of the PersistentVolumeClaim created for the Headscale Let's Encrypt cache. + storageClassName: "" + service: # -- The port of the Kubernetes service. port: 80 + # -- The type of the Kubernetes service. Options are "ClusterIP", "NodePort", and "LoadBalancer". + type: "ClusterIP" # -- The annotations to add to the Kubernetes service. annotations: { } # -- Override the name of the Kubernetes service. @@ -274,12 +328,15 @@ providers: enabled: false secret: - # -- The name of the secret containing the API key. + # -- The name of the secret containing the Runpod credentials. name: "runpod-secret" keys: # -- The key in the secret containing the API key. apiKey: "RUNPOD_API_KEY" + # -- The key in the secret containing the container registry auth ID. + containerRegistryAuthId: "RUNPOD_CONTAINER_REGISTRY_AUTH_ID" - # -- Manual configuration of the Runpod API key. Not recommended for production. + # -- Manual configuration of the Runpod credentials. Not recommended for production. auth: apiKey: "" + containerRegistryAuthId: "" diff --git a/deploy/docker/README.md b/deploy/docker/README.md new file mode 100644 index 0000000..d68ac58 --- /dev/null +++ b/deploy/docker/README.md @@ -0,0 +1,10 @@ +```bash +docker compose up +``` + + +You can also launch it through the CLI: + +```bash +nebu serve --docker +``` diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml new file mode 100644 index 0000000..95ac6f2 --- /dev/null +++ b/deploy/docker/docker-compose.yml @@ -0,0 +1,55 @@ +services: + tailscale: + image: tailscale/tailscale:latest + container_name: tailscale + hostname: nebulous + environment: + - TS_STATE_DIR=/var/lib/tailscale + - TS_USERSPACE=true + - TS_AUTH_KEY= + - TS_EXTRA_ARGS=--login-server https://headscale.nebulous.sh + volumes: + - nebu-ts-authkey:/var/lib/tailscale + - /dev/net/tun:/dev/net/tun + - nebu-ts-sock:/var/run/tailscale + - nebu-tmp:/tmp + cap_add: + - NET_ADMIN + - SYS_MODULE + restart: unless-stopped + + nebulous: + image: us-docker.pkg.dev/agentsea-dev/nebulous/server:latest + command: ["sh", "-c", "exec nebu serve --host 0.0.0.0 --port 3000"] + environment: + DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres + REDIS_URL: redis://localhost:6379 + RUST_LOG: debug + NEBU_BUCKET_NAME: nebulous + NEBU_BUCKET_REGION: us-east-1 + NEBU_ROOT_OWNER: me + volumes: + - nebu-ts-sock:/var/run/tailscale + - nebu-tmp:/tmp + network_mode: service:tailscale + + redis: + image: redis:latest + network_mode: service:tailscale + restart: unless-stopped + + postgres: + image: postgres:latest + environment: + POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + restart: unless-stopped + +volumes: + nebu-ts-authkey: + driver: local + nebu-ts-sock: + driver: local + nebu-tmp: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index fe936eb..a6b9488 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,12 @@ services: ports: - "5432:5432" + redis: + image: redis:latest + command: [ "redis-server", "--requirepass", "myredispassword" ] + ports: + - "6379:6379" + nebu: build: context: . @@ -13,6 +19,7 @@ services: container_name: nebu environment: DATABASE_URL: postgres://postgres@postgres:5432/postgres + REDIS_URL: redis://:myredispassword@redis:6379 RUST_LOG: debug NEBU_BUCKET_NAME: nebulous NEBU_BUCKET_REGION: us-east-1 diff --git a/src/agent/mod.rs b/src/agent/mod.rs index 7ec70c6..97a1165 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -1,3 +1,3 @@ pub mod agent; pub mod aws; -pub mod ns; \ No newline at end of file +pub mod ns; diff --git a/src/agent/ns.rs b/src/agent/ns.rs index fa3b4bf..63445ef 100644 --- a/src/agent/ns.rs +++ b/src/agent/ns.rs @@ -1,4 +1,4 @@ -use crate::config::CONFIG; +use crate::config::SERVER_CONFIG; use crate::entities::namespaces; use anyhow::Result; use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; @@ -25,7 +25,7 @@ pub async fn auth_ns( if namespace == "root" { debug!("Namespace is root"); - let root_owner = CONFIG.root_owner.clone(); + let root_owner = SERVER_CONFIG.root_owner.clone(); if !owner_ids.contains(&root_owner) { error!("User not authorized to access root namespace"); return Err(anyhow::anyhow!("User not authorized to access namespace")); diff --git a/src/auth/api.rs b/src/auth/api.rs index 8c57ea4..9553659 100644 --- a/src/auth/api.rs +++ b/src/auth/api.rs @@ -72,7 +72,7 @@ pub async fn validate_api_key( let parts: Vec<&str> = full_key.split('.').collect(); if parts.len() == 2 { let (id, key) = (parts[0], parts[1]); - if let Some(mut api_key) = db::Entity::find_by_id(id).one(db_conn).await? { + if let Some(api_key) = db::Entity::find_by_id(id).one(db_conn).await? { if api_key.revoked_at.is_some() { return Ok(false); } diff --git a/src/cli.rs b/src/cli.rs index 6217d4f..efacb09 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,6 +1,6 @@ use clap::{arg, ArgAction, Args, Parser, Subcommand}; -/// Orign CLI. +/// Nebulous CLI #[derive(Parser)] #[command(author, version, about, long_about = None)] pub struct Cli { @@ -43,7 +43,7 @@ pub enum Commands { /// Serve the API. Serve { - /// The address to bind to. + /// The address to bind to. Ignored when launching through docker. #[arg(long, default_value = "127.0.0.1")] host: String, @@ -51,13 +51,17 @@ pub enum Commands { #[arg(long, default_value_t = 3000)] port: u16, - /// Disable internal auth server - #[arg(long, default_value_t = true)] - internal_auth: bool, + /// Disable the internal auth server + #[arg(long, default_value_t = false)] + disable_internal_auth: bool, /// The port to bind the internal auth server to. #[arg(long, default_value_t = 8080)] auth_port: u16, + + /// Launch through Docker + #[arg(long, default_value_t = false)] + docker: bool, }, /// Proxy services. @@ -97,6 +101,14 @@ pub enum Commands { #[arg(default_value = "http://127.0.0.1")] url: String, + /// Name of the server + #[arg(long, default_value = "nebulous")] + name: String, + + /// Update the server if it already exists + #[arg(long, default_value_t = false)] + update: bool, + /// Address of the Auth server #[arg(long, default_value = None)] auth: Option, @@ -114,6 +126,18 @@ pub enum Commands { #[command(subcommand)] command: AuthCommands, }, + + /// Manage configuration + Config { + #[command(subcommand)] + command: ConfigCommands, + }, + + /// Manage a headscale server + Headscale { + #[command(subcommand)] + command: HeadscaleCommands, + }, } /// Select a checkpoint. @@ -457,3 +481,61 @@ pub enum ApiKeyActions { id: String, }, } + +#[derive(Subcommand)] +pub enum ConfigCommands { + /// Show the full configuration + Show, + + /// Manage the current configuration + Current { + /// The action to perform. + #[command(subcommand)] + action: CurrentConfigActions, + }, +} + +#[derive(Subcommand)] +pub enum CurrentConfigActions { + /// Show the current configuration + Show, + /// Set the current configuration + Set { + /// The name of the server to set as current + server: String, + }, +} + +#[derive(Subcommand)] +pub enum HeadscaleCommands { + /// Manage API keys + Apikey { + /// The action to perform + #[command(subcommand)] + action: HeadscaleApiKeyActions, + }, +} + +#[derive(Subcommand)] +pub enum HeadscaleApiKeyActions { + /// Generate a new API key + Create { + /// The API key's validity period + #[arg(long)] + expiration: String, + }, + + /// Validate an API key + Validate { + /// The API key's prefix + #[arg(long)] + prefix: String, + }, + + /// Revoke an API key. + Revoke { + /// The API key's prefix + #[arg(long)] + prefix: String, + }, +} diff --git a/src/client/client.rs b/src/client/client.rs index 24e368e..2037ad7 100644 --- a/src/client/client.rs +++ b/src/client/client.rs @@ -1,4 +1,4 @@ -use crate::config::GlobalConfig; +use crate::config::ClientConfig; use crate::resources::v1::containers::models::{ V1Container, V1ContainerRequest, V1ContainerSearch, V1Containers, V1UpdateContainer, }; @@ -45,9 +45,9 @@ impl NebulousClient { /// Creates a new NebulousClient by reading from the global config. /// You could also pass server and api key directly if preferred. pub fn new_from_config() -> Result> { - let config = GlobalConfig::read()?; + let config = ClientConfig::read()?; let current_server = config - .get_current_server_config() + .get_current_server() .ok_or("No current server config found")?; let server_url = current_server .server diff --git a/src/commands/auth_cmd.rs b/src/commands/auth_cmd.rs index 7ca3719..7db673a 100644 --- a/src/commands/auth_cmd.rs +++ b/src/commands/auth_cmd.rs @@ -1,11 +1,9 @@ use crate::commands::request::server_request; use nebulous::auth::models::SanitizedApiKey; use nebulous::auth::server::handlers::{ApiKeyListResponse, ApiKeyRequest, RawApiKeyResponse}; +use nebulous::config::ClientConfig; use std::error::Error; -// TODO: Make the auth server's port configurable -const SERVER: &str = "http://localhost:8080"; - fn pretty_print_api_key(api_key: SanitizedApiKey) { println!("ID: {}", api_key.id); println!("Active: {}", api_key.is_active); @@ -42,7 +40,9 @@ pub async fn get_api_key(id: &str) -> Result<(), Box> { } pub async fn generate_api_key() -> Result<(), Box> { - let url = format!("{}/api-key/generate", SERVER); + let config = ClientConfig::read()?; + let internal_auth_url = format!("http://localhost:{}", config.internal_auth_port.expect("No internal auth port configured. Note that this command only works on localhost and when the internal auth server is active.")); + let url = format!("{}/api-key/generate", internal_auth_url); match reqwest::Client::new().get(&url).send().await { Ok(response) => { let api_key = response.json::().await?; @@ -61,7 +61,9 @@ pub async fn generate_api_key() -> Result<(), Box> { } pub async fn revoke_api_key(id: &str) -> Result<(), Box> { - let url = format!("{}/api-key/revoke", SERVER); + let config = ClientConfig::read()?; + let internal_auth_url = format!("http://localhost:{}", config.internal_auth_port.expect("No internal auth port configured. Note that this command only works on localhost and when the internal auth server is active.")); + let url = format!("{}/api-key/revoke", internal_auth_url); let payload = ApiKeyRequest { id: id.to_string() }; match reqwest::Client::new() .post(&url) diff --git a/src/commands/config_cmd.rs b/src/commands/config_cmd.rs new file mode 100644 index 0000000..cdf24b7 --- /dev/null +++ b/src/commands/config_cmd.rs @@ -0,0 +1,55 @@ +use std::error::Error; + +pub async fn show_config() -> Result<(), Box> { + match nebulous::config::ClientConfig::read() { + Ok(config) => match serde_yaml::to_string(&config) { + Ok(yaml) => println!("{}", yaml), + Err(e) => eprintln!("Error formatting config as YAML: {}", e), + }, + Err(e) => { + eprintln!("Error reading config: {}", e); + } + } + Ok(()) +} + +pub async fn show_current() -> Result<(), Box> { + match nebulous::config::ClientConfig::read() { + Ok(config) => { + if let Some(current_server) = config.get_current_server() { + let mut current_server = current_server.clone(); + current_server.api_key = Some("".to_string()); + match serde_yaml::to_string(¤t_server) { + Ok(yaml) => println!("{}", yaml), + Err(e) => eprintln!("Error formatting current server config as YAML: {}", e), + } + } else { + eprintln!("No current server configuration found."); + } + } + Err(e) => { + eprintln!("Error reading config: {}", e); + } + } + Ok(()) +} + +pub async fn set_current(name: &str) -> Result<(), Box> { + match nebulous::config::ClientConfig::read() { + Ok(mut config) => { + if let Some(server_config) = config.get_server(name) { + config.current_server = Some(server_config.name.clone()); + match config.write() { + Ok(_) => println!("Current server set to '{}'", name), + Err(e) => eprintln!("Error writing config: {}", e), + } + } else { + eprintln!("Server '{}' not found in configuration.", name); + } + } + Err(e) => { + eprintln!("Error reading config: {}", e); + } + } + Ok(()) +} diff --git a/src/commands/configure_cmd.rs b/src/commands/configure_cmd.rs deleted file mode 100644 index cd9cb3d..0000000 --- a/src/commands/configure_cmd.rs +++ /dev/null @@ -1,5 +0,0 @@ -use std::error::Error; - -pub async fn configure() -> Result<(), Box> { - Ok(()) -} diff --git a/src/commands/headscale_cmd.rs b/src/commands/headscale_cmd.rs new file mode 100644 index 0000000..c036959 --- /dev/null +++ b/src/commands/headscale_cmd.rs @@ -0,0 +1,99 @@ +use futures::{StreamExt, TryStreamExt}; +use k8s_openapi::api::core::v1::Pod; +use kube::api::{AttachParams, AttachedProcess}; +use kube::{Api, Client}; + +#[derive(Debug, Clone)] +struct HeadscalePod { + name: String, + namespace: String, +} + +impl HeadscalePod { + fn new() -> Self { + let name = std::env::var("HEADSCALE_POD_NAME").expect("HEADSCALE_POD_NAME not set"); + let namespace = + std::env::var("HEADSCALE_POD_NAMESPACE").expect("HEADSCALE_POD_NAMESPACE not set"); + HeadscalePod { name, namespace } + } +} + +async fn get_output(mut attached: AttachedProcess) -> String { + let stdout = tokio_util::io::ReaderStream::new(attached.stdout().unwrap()); + let out = stdout + .filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) }) + .collect::>() + .await + .join(""); + attached.join().await.unwrap(); + out +} + +async fn headscale_cmd(cmd: Vec<&str>) -> Result> { + let headscale_pod = HeadscalePod::new(); + + let client = Client::try_default().await?; + let api: Api = Api::namespaced(client, headscale_pod.namespace.as_str()); + + let attached = api + .exec( + headscale_pod.name.as_str(), + cmd, + &AttachParams::default().stderr(false), + ) + .await?; + let output = get_output(attached).await; + Ok(output) +} + +pub async fn create_api_key(expiration: &str) -> Result> { + let cmd = vec!["headscale", "apikeys", "create", "--expiration", expiration]; + let api_key = headscale_cmd(cmd.into()).await?; + Ok(api_key) +} + +pub async fn validate_api_key(prefix: &str) -> Result> { + let cmd = vec!["headscale", "apikeys", "list", "-o", "json"]; + let output = headscale_cmd(cmd).await?; + let api_keys = serde_json::from_str::>(&output)?; + + let cmd = vec!["date", "+%s%N"]; + let timestamp = headscale_cmd(cmd).await?; + + for apikey in api_keys { + if let Some(other_prefix) = apikey.get("prefix") { + if other_prefix.as_str() == Some(prefix) { + println!("API key with prefix {} found", prefix); + + return if let Some(expiration) = apikey.get("expiration") { + let seconds = expiration + .get("seconds") + .expect("API key has no expiration seconds"); + let nanos = expiration + .get("nanos") + .expect("API key has no expiration nanos"); + let expiration_time = + seconds.as_i64().unwrap() * 1_000_000_000 + nanos.as_i64().unwrap(); + + if expiration_time < timestamp.parse::().unwrap() { + println!("API key has expired"); + Ok(false) + } else { + println!("API key is valid"); + Ok(true) + } + } else { + Err(format!("API key with prefix {} has no expiration time", prefix).into()) + }; + } + } + } + println!("API key with prefix {} not found", prefix); + Ok(false) +} + +pub async fn revoke_api_key(prefix: &str) -> Result<(), Box> { + let cmd = vec!["headscale", "apikeys", "expire", "--prefix", prefix]; + headscale_cmd(cmd).await?; + Ok(()) +} diff --git a/src/commands/log_cmd.rs b/src/commands/log_cmd.rs index 17e6cfa..2bffa7f 100644 --- a/src/commands/log_cmd.rs +++ b/src/commands/log_cmd.rs @@ -1,4 +1,4 @@ -use nebulous::config::GlobalConfig; +use nebulous::config::ClientConfig; use std::error::Error as StdError; use reqwest::Client; @@ -12,8 +12,8 @@ pub async fn fetch_container_logs( let container_id = fetch_container_id_from_api(&namespace, &name).await?; // Load config the same way as in get_cmd.rs - let config = GlobalConfig::read()?; - let current_server = config.get_current_server_config().unwrap(); + let config = ClientConfig::read()?; + let current_server = config.get_current_server().unwrap(); let _server = current_server.server.as_ref().unwrap(); let api_key = current_server.api_key.as_ref().unwrap(); @@ -43,8 +43,8 @@ async fn fetch_container_id_from_api( namespace: &str, name: &str, ) -> Result> { - let config = nebulous::config::GlobalConfig::read()?; - let current_server = config.get_current_server_config().unwrap(); + let config = nebulous::config::ClientConfig::read()?; + let current_server = config.get_current_server().unwrap(); let server = current_server.server.as_ref().unwrap(); let api_key = current_server.api_key.as_ref().unwrap(); // Adjust base URL/host as needed: diff --git a/src/commands/login_cmd.rs b/src/commands/login_cmd.rs index 10c3793..b53e1d1 100644 --- a/src/commands/login_cmd.rs +++ b/src/commands/login_cmd.rs @@ -1,12 +1,16 @@ use std::error::Error; use std::io::{self, Write}; -use nebulous::config::{GlobalConfig, ServerConfig}; +use crate::commands::request::server_request; +use nebulous::config::{ClientConfig, ClientServerConfig}; +use nebulous::models::V1UserProfile; use open; use rpassword; pub async fn execute( nebu_url: String, + name: String, + update: bool, auth: Option, hub: Option, ) -> Result<(), Box> { @@ -17,7 +21,12 @@ pub async fn execute( let nebu_url = nebu_url.trim().trim_end_matches("/").to_string(); - let mut config = GlobalConfig::read()?; + let mut config = ClientConfig::read()?; + + if config.contains_server(&name) && !update { + eprintln!("Server with name '{}' already exists. Please choose a different name or set --update flag.", name); + return Ok(()); + } if auth.is_some() && hub.is_some() { let auth_url = auth.unwrap().trim().trim_end_matches("/").to_string(); @@ -35,13 +44,15 @@ pub async fn execute( io::stdout().flush()?; let api_key = rpassword::read_password()?; - config.servers.push(ServerConfig { - name: Some("cloud".to_string()), - server: Some(nebu_url), - api_key: Some(api_key), - auth_server: Some(auth_url), - }); - config.current_server = Some("cloud".to_string()); + config.update_server( + ClientServerConfig { + name, + server: Some(nebu_url), + api_key: Some(api_key), + auth_server: Some(auth_url), + }, + true, + ); } else { println!( r#"Configuring the Nebulous CLI to use the integrated auth server. @@ -63,18 +74,25 @@ When you're running nebulous on Kubernetes, use: io::stdout().flush()?; let api_key = rpassword::read_password()?; - config.servers.push(ServerConfig { - name: Some("nebu".to_string()), - server: Some(nebu_url), - api_key: Some(api_key), - auth_server: None, - }); - config.current_server = Some("nebu".to_string()); + config.update_server( + ClientServerConfig { + name, + server: Some(nebu_url), + api_key: Some(api_key), + auth_server: None, + }, + true, + ); } - config.write()?; - // TODO: Check that we can actually reach and authenticate with the server + let response = server_request("/v1/users/me", reqwest::Method::GET).await?; + let profile: V1UserProfile = response.json().await?; + println!( + "\nSuccessfully logged into '{}' as '{}'", + config.current_server.clone().unwrap(), + profile.email + ); - println!("\nLogin successful!"); + config.write()?; Ok(()) } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index b848b76..bc1ea33 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,10 +1,11 @@ pub mod auth_cmd; -pub mod configure_cmd; +pub mod config_cmd; pub mod create_cmd; pub mod daemon_cmd; pub mod delete_cmd; pub mod exec_cmd; pub mod get_cmd; +pub mod headscale_cmd; pub mod log_cmd; pub mod login_cmd; pub mod proxy_cmd; diff --git a/src/commands/request.rs b/src/commands/request.rs index f7999f7..9c27e70 100644 --- a/src/commands/request.rs +++ b/src/commands/request.rs @@ -4,9 +4,9 @@ fn prepare_request( path: &str, method: reqwest::Method, ) -> Result> { - let config = nebulous::config::GlobalConfig::read()?; + let config = nebulous::config::ClientConfig::read()?; let current_server = config - .get_current_server_config() + .get_current_server() .ok_or("Failed to get current server configuration")?; let server = current_server .server diff --git a/src/commands/serve_cmd.rs b/src/commands/serve_cmd.rs index b1a9f09..34f72d5 100644 --- a/src/commands/serve_cmd.rs +++ b/src/commands/serve_cmd.rs @@ -1,16 +1,19 @@ -// src/commands/serve.rs - +use nebulous::config::ClientConfig; use nebulous::create_app; use nebulous::create_app_state; use nebulous::proxy::server::start_proxy; use nebulous::resources::v1::containers::controller::ContainerController; use nebulous::resources::v1::processors::controller::ProcessorController; +use serde_json::Value; use std::error::Error; +use std::io::Write; +use std::ops::Add; +use std::process::{Command, Stdio}; -pub async fn execute( +pub async fn launch_server( host: String, port: u16, - internal_auth: bool, + disable_internal_auth: bool, auth_port: u16, ) -> Result<(), Box> { let app_state = create_app_state().await?; @@ -37,8 +40,12 @@ pub async fn execute( }); println!("Proxy server started in background"); - if internal_auth { + if !disable_internal_auth { println!("Starting auth server"); + let mut config = ClientConfig::read()?; + config.set_internal_auth_port(auth_port); + config.write()?; + tokio::spawn({ let auth_state = app_state.clone(); async move { @@ -63,3 +70,71 @@ pub async fn execute( Ok(()) } + +const BASE_COMPOSE: &str = include_str!("../../deploy/docker/docker-compose.yml"); + +pub async fn launch_docker( + port: u16, + disable_internal_auth: bool, + auth_port: u16, +) -> Result<(), Box> { + Command::new("docker") + .args(&["compose", "version"]) + .output() + .expect( + "Did not find docker compose. Please ensure that docker is installed and in your PATH.", + ); + + // Ask the user for the tailscale login server and auth key + let tailscale_login_server = std::env::var("TS_LOGIN_SERVER").unwrap_or_else(|_| { + println!("Please enter the Tailscale login server URL (or leave blank for default):"); + let mut input = String::new(); + std::io::stdin().read_line(&mut input).unwrap(); + input.trim().to_string() + }); + let tailscale_auth_key = std::env::var("TS_AUTH_KEY").unwrap_or_else(|_| { + println!("Please enter the Tailscale auth key:"); + let mut input = String::new(); + std::io::stdin().read_line(&mut input).unwrap(); + input.trim().to_string() + }); + + let mut doc: Value = serde_yaml::from_str(BASE_COMPOSE)?; + + doc["services"]["tailscale"]["environment"][2] = + Value::String(format!("TS_AUTH_KEY={}", tailscale_auth_key)); + if !tailscale_login_server.is_empty() { + doc["services"]["tailscale"]["environment"][3] = Value::String(format!( + "TS_EXTRA_ARGS=--login-server {}", + tailscale_login_server + )); + } + + let mut command = format!( + "exec nebu serve --host 0.0.0.0 --port {} --auth-port {}", + port, auth_port + ); + if disable_internal_auth { + command = command.add(" --disable-internal-auth"); + }; + doc["services"]["nebulous"]["command"][2] = Value::String(command); + + let yaml = serde_yaml::to_string(&doc)?; + + let mut child = Command::new("docker") + .args(&["compose", "-f", "-", "up"]) + .stdin(Stdio::piped()) + .spawn()?; + + { + let stdin = child.stdin.as_mut().expect("Failed to open stdin"); + stdin.write_all(yaml.as_bytes())?; + } + + let status = child.wait()?; + if !status.success() { + Err("docker compose failed".into()) + } else { + Ok(()) + } +} diff --git a/src/config.rs b/src/config.rs index f0daf37..047d949 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,24 +7,25 @@ use std::fs; use std::path::PathBuf; #[derive(Serialize, Deserialize, Default, Debug)] -pub struct GlobalConfig { - pub servers: Vec, +pub struct ClientConfig { + pub servers: Vec, pub current_server: Option, + pub internal_auth_port: Option, } #[derive(Serialize, Deserialize, Default, Clone, Debug)] -pub struct ServerConfig { - /// Optional identifier for your server config. - pub name: Option, +pub struct ClientServerConfig { + pub name: String, pub api_key: Option, pub server: Option, pub auth_server: Option, } -impl GlobalConfig { - /// Read the config from disk, or create a default one. - /// Then ensure that we either find or create a matching server in `self.servers` - /// based on environment variables, and set that as the `default_server`. +/// Configuration for a Nebulous client. +impl ClientConfig { + /// Read client configuration from disk, or create a default one. + /// If a server is configured through environment variables, + /// it will be added temporarily and marked as the current server. pub fn read() -> Result> { let config_path = get_config_file_path()?; let path_exists = config_path.exists(); @@ -32,12 +33,22 @@ impl GlobalConfig { // Load or create default let mut config = if path_exists { let yaml = fs::read_to_string(&config_path)?; - serde_yaml::from_str::(&yaml)? + serde_yaml::from_str::(&yaml)? } else { - GlobalConfig::default() + ClientConfig::default() }; - // Collect environment variables (NO fallback defaults here) + // Only write if the file didn't already exist + if !path_exists { + config.write()?; + } + + config.create_config_from_environment(); + + Ok(config) + } + + fn create_config_from_environment(&mut self) { let env_api_key = env::var("NEBU_API_KEY") .or_else(|_| env::var("AGENTSEA_API_KEY")) .ok(); @@ -48,50 +59,32 @@ impl GlobalConfig { .or_else(|_| env::var("AGENTSEA_AUTH_SERVER")) .ok(); - // Only proceed if all three environment variables are present. if let (Some(env_api_key), Some(env_server), Some(env_auth_server)) = (env_api_key, env_server, env_auth_server) { // Find a matching server (all three fields match). - let found_server = config.servers.iter_mut().find(|srv| { + let found_server = self.servers.iter_mut().find(|srv| { srv.api_key.as_deref() == Some(&env_api_key) && srv.server.as_deref() == Some(&env_server) && srv.auth_server.as_deref() == Some(&env_auth_server) }); // If found, use that. If not, create a new entry. - let server_name = "env-based-server".to_string(); - let chosen_name = if let Some(srv) = found_server { - // Make sure it has a name, so we can set default_server to it - if srv.name.is_none() { - srv.name = Some(server_name.clone()); - } - srv.name.clone().unwrap() + if let Some(srv) = found_server { + self.current_server = Some(srv.name.clone()); } else { - // Need to create a new server entry - let new_server = ServerConfig { - name: Some(server_name.clone()), + let new_server = ClientServerConfig { + name: "env-based-server".to_string(), api_key: Some(env_api_key), server: Some(env_server), auth_server: Some(env_auth_server), }; - config.servers.push(new_server); - server_name + self.update_server(new_server, true); }; - - // Set that server as the “current” or default - config.current_server = Some(chosen_name); } - - // Only write if the file didn't already exist - if !path_exists { - config.write()?; - } - - Ok(config) } - /// Write the current GlobalConfig to disk (YAML). + /// Write the current ClientConfig to disk (YAML). pub fn write(&self) -> Result<(), Box> { let config_path = get_config_file_path()?; @@ -106,15 +99,66 @@ impl GlobalConfig { Ok(()) } - /// Get the server config for the current `default_server`. - /// Returns `None` if `default_server` is unset or if no server - /// with that name is found. - pub fn get_current_server_config(&self) -> Option<&ServerConfig> { - self.current_server.as_deref().and_then(|name| { - self.servers - .iter() - .find(|srv| srv.name.as_deref() == Some(name)) - }) + /// Get the current server. + pub fn get_current_server(&self) -> Option<&ClientServerConfig> { + self.current_server + .as_deref() + .and_then(|name| self.servers.iter().find(|srv| srv.name == name)) + } + + /// Get a server by name. + pub fn get_server(&self, name: &str) -> Option<&ClientServerConfig> { + self.servers.iter().find(|srv| srv.name == name) + } + + /// Remove a server. + pub fn drop_server(&mut self, name: &str) { + if let Some(pos) = self.servers.iter().position(|srv| srv.name == name) { + self.servers.remove(pos); + + // If the removed server was the current one, clear it. + if self.current_server == Some(name.to_string()) { + self.current_server = None; + } + } + } + + /// Update or add a server. + pub fn update_server(&mut self, new_config: ClientServerConfig, make_current: bool) { + if let Some(pos) = self + .servers + .iter() + .position(|srv| srv.name == new_config.name) + { + self.servers[pos] = new_config; + } else { + if make_current { + self.current_server = Some(new_config.name.clone()); + } + self.servers.push(new_config); + } + } + + /// Add a server. + pub fn add_server(&mut self, config: ClientServerConfig, make_current: bool) { + if self.contains_server(&config.name) { + eprintln!( + "Server with name '{}' already exists. Please choose a different name.", + config.name + ); + return; + } + self.update_server(config, make_current); + } + + /// Check if a server with the given name exists. + pub fn contains_server(&self, name: &str) -> bool { + self.servers.iter().any(|srv| srv.name == name) + } + + /// Set the internal auth port + pub fn set_internal_auth_port(&mut self, port: u16) { + self.internal_auth_port = Some(port); } } @@ -126,40 +170,152 @@ fn get_config_file_path() -> Result> { } #[derive(Debug, Clone)] -pub struct Config { - pub message_queue_type: String, - pub kafka_bootstrap_servers: String, - pub kafka_timeout_ms: String, - pub redis_host: String, - pub redis_port: String, - pub redis_password: Option, - pub redis_url: Option, +pub struct ServerConfig { pub database_url: String, - pub tailscale_api_key: Option, - pub tailscale_tailnet: Option, + pub message_queue_type: String, + pub redis_url: String, + + pub tailscale: Option, + + pub auth: ServerAuthConfig, + pub bucket_name: String, pub bucket_region: String, pub root_owner: String, } -impl Config { +#[derive(Debug, Clone)] +pub struct DatabaseConfig { + pub host: String, + pub port: u16, + pub user: String, + pub password: Option, + pub name: String, +} + +impl DatabaseConfig { + pub fn new() -> Self { + dotenv().ok(); + + Self { + host: env::var("DATABASE_HOST").unwrap_or_else(|_| "localhost".to_string()), + port: env::var("DATABASE_PORT") + .unwrap_or_else(|_| "5432".to_string()) + .parse() + .expect("Invalid value for DATABASE_PORT."), + user: env::var("DATABASE_USER").unwrap_or_else(|_| "postgres".to_string()), + password: env::var("DATABASE_PASSWORD").ok(), + name: env::var("DATABASE_NAME").unwrap_or_else(|_| "postgres".to_string()), + } + } +} + +#[derive(Debug, Clone)] +pub struct RedisConfig { + pub host: String, + pub port: u16, + pub user: Option, + pub password: Option, + pub database: u16, +} + +impl RedisConfig { + pub fn new() -> Self { + dotenv().ok(); + + Self { + host: env::var("REDIS_HOST").unwrap_or_else(|_| "localhost".to_string()), + port: env::var("REDIS_PORT") + .unwrap_or_else(|_| "6379".to_string()) + .parse() + .expect("Invalid value for REDIS_PORT."), + user: env::var("REDIS_USER").ok(), + password: env::var("REDIS_PASSWORD").ok(), + database: env::var("REDIS_DATABASE") + .unwrap_or_else(|_| "0".to_string()) + .parse() + .expect("Invalid value for REDIS_DATABASE."), + } + } +} + +#[derive(Debug, Clone)] +pub struct TailscaleConfig { + pub api_key: String, + pub tailnet: String, +} + +#[derive(Debug, Clone)] +pub struct ServerAuthConfig { + pub url: Option, +} + +impl ServerAuthConfig { + pub fn new() -> Self { + dotenv().ok(); + + let url = env::var("NEBU_AUTH_URL").ok(); + + Self { + url, + } + } +} + +/// Configuration for a Nebulous server. +impl ServerConfig { pub fn new() -> Self { dotenv().ok(); + let database_url = env::var("DATABASE_URL").unwrap_or_else(|_| { + let db_config = DatabaseConfig::new(); + format!( + "postgres://{}:{}@{}:{}/{}", + db_config.user, + db_config.password.as_deref().unwrap_or(""), + db_config.host, + db_config.port, + db_config.name + ) + }); + + let message_queue_type = match env::var("MESSAGE_QUEUE_TYPE") { + Ok(queue_type) => { + if queue_type == "redis" { + queue_type + } else { + panic!("Invalid MESSAGE_QUEUE_TYPE. Only 'redis' is supported. (You can safely omit this value.)") + } + } + Err(_) => "redis".to_string(), + }; + + let redis_url = env::var("REDIS_URL").unwrap_or_else(|_| { + let redis_config = RedisConfig::new(); + format!( + "redis://{}:{}@{}:{}/{}", + redis_config.user.as_deref().unwrap_or(""), + redis_config.password.as_deref().unwrap_or(""), + redis_config.host, + redis_config.port, + redis_config.database + ) + }); + + let tailscale = match (env::var("TS_API_KEY"), env::var("TS_TAILNET")) { + (Ok(api_key), Ok(tailnet)) => Some(TailscaleConfig { api_key, tailnet }), + _ => None, + }; + + let auth = ServerAuthConfig::new(); + Self { - message_queue_type: env::var("MESSAGE_QUEUE_TYPE") - .unwrap_or_else(|_| "redis".to_string()), - kafka_bootstrap_servers: env::var("KAFKA_BOOTSTRAP_SERVERS") - .unwrap_or_else(|_| "localhost:9092".to_string()), - kafka_timeout_ms: env::var("KAFKA_TIMEOUT_MS").unwrap_or_else(|_| "5000".to_string()), - redis_host: env::var("REDIS_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()), - redis_port: env::var("REDIS_PORT").unwrap_or_else(|_| "6379".to_string()), - redis_password: env::var("REDIS_PASSWORD").ok(), - redis_url: env::var("REDIS_URL").ok(), - database_url: env::var("DATABASE_URL") - .unwrap_or_else(|_| "sqlite://.data/data.db".to_string()), - tailscale_api_key: env::var("TAILSCALE_API_KEY").ok(), - tailscale_tailnet: env::var("TAILSCALE_TAILNET").ok(), + database_url, + message_queue_type, + redis_url, + tailscale, + auth, + // TODO: Move this to dedicated config bucket_name: env::var("NEBU_BUCKET_NAME") .unwrap_or_else(|_| panic!("NEBU_BUCKET_NAME environment variable must be set")), bucket_region: env::var("NEBU_BUCKET_REGION") @@ -168,6 +324,23 @@ impl Config { .unwrap_or_else(|_| panic!("NEBU_ROOT_OWNER environment variable must be set")), } } + pub fn get_redis_url(&self, user: String, password: String) -> String { + let protocol = self + .redis_url + .clone() + .split("://") + .next() + .unwrap_or_else(|| panic!("Invalid Redis URL {}", self.redis_url)) + .to_string(); + let base_url = self + .redis_url + .clone() + .split("@") + .nth(1) + .unwrap_or_else(|| panic!("Invalid Redis URL {}", self.redis_url)) + .to_string(); + format!("{protocol}://{user}:{password}@{base_url}") + } } // Global static CONFIG instance -pub static CONFIG: Lazy = Lazy::new(Config::new); +pub static SERVER_CONFIG: Lazy = Lazy::new(ServerConfig::new); diff --git a/src/controllers/containers.rs b/src/controllers/containers.rs new file mode 100644 index 0000000..1244e1c --- /dev/null +++ b/src/controllers/containers.rs @@ -0,0 +1,216 @@ +use crate::platforms::factory::create_platform_from_kind; +use crate::resources::v1::containers::models::V1Container; +use crate::resources::containers::ContainerModelVersion; +use crate::resources::v1::platforms::models::V1ContainerPlatform; +use crate::state::AppState; +use anyhow::{anyhow, Context}; +use std::sync::Arc; +use tracing::{error, info, span, Instrument, Level}; +use sea_orm::ActiveModelTrait; +use sea_orm::EntityTrait; +use crate::entities::containers::entity::Entity; + + +pub struct ContainerController +where + V: ContainerModelVersion, +{ + app_state: Arc, +} + +enum ContainerStatus { + Pending, + Scheduled, + Creating, + Running, + Unreachable, + Killed, + Stopping, + Finished, + Failed, +} + +impl ContainerStatus { + fn from_string(status: &str) -> Self { + match status { + "Pending" => ContainerStatus::Pending, + "Scheduled" => ContainerStatus::Scheduled, + "Creating" => ContainerStatus::Creating, + "Running" => ContainerStatus::Running, + "Unreachable" => ContainerStatus::Unreachable, + "Killed" => ContainerStatus::Killed, + "Stopping" => ContainerStatus::Stopping, + "Finished" => ContainerStatus::Finished, + "Failed" => ContainerStatus::Failed, + _ => panic!("Unknown container status"), + } + } +} + +#[derive(Debug, Clone)] +struct ControllerContainer { + id: String, + status: ContainerStatus, + platform: Option, +} + +trait ToControllerContainer { + fn to_controller(&self) -> ControllerContainer; +} + +impl ToControllerContainer for V1Container { + fn to_controller(&self) -> ControllerContainer { + ControllerContainer { + id: self.metadata.id.clone(), + platform: Some(self.platform.clone()), + // TODO: Handle the status conversion (and make it a required field?) + status: ContainerStatus::from_string(&self.status.clone().unwrap().status.unwrap()), + } + } +} + +impl ContainerController +where + V: ContainerModelVersion, + V::Container: ToControllerContainer, +{ + pub fn new(app_state: Arc) -> Self { + Self { app_state } + } + + pub async fn reconcile(&self) { + let span = span!(Level::INFO, "ContainerController"); + + async { + info!("Starting reconciliation process"); + let paginator = entities::containers::Entity::find() + .paginate(&self.app_state.db_pool, 100) + .await?; + let total_pages = paginator.num_pages().await?; + + for page in 1..=total_pages { + let platforms = paginator.fetch_page(page).await?; + for mut platform in platforms { + self.handle(platform).await; + } + } + info!("Finished reconciliation process"); + } + .instrument(span) + .await; + } + + pub async fn handle(&self, mut container: V::Container) -> anyhow::Result<()> { + let container_status: ContainerStatus = ContainerStatus::Scheduled; + + match container_status { + ContainerStatus::Pending | ContainerStatus::Scheduled => { + self.schedule(container).await?; + } + ContainerStatus::Creating + | ContainerStatus::Running + | ContainerStatus::Unreachable + | ContainerStatus::Stopping => { + self.watch(container).await?; + } + ContainerStatus::Killed => { + self.kill(container).await?; + } + ContainerStatus::Finished | ContainerStatus::Failed => {} + } + + Ok(()) + } + + /// Schedules a container to run on a suitable platform. + pub async fn schedule(&self, container: V::Container) -> anyhow::Result { + let platform_spec = self.find_platform(container.clone()).await?; + let platform: Box> = + create_platform_from_kind(platform_spec).await?; + // TODO: Determine if the platform can take the container right now + // and adjust status accordingly + + // TODO: Run as separate task and return with "Creating" + platform.create(container).await? + } + + async fn find_platform( + &self, + container: V::Container, + ) -> anyhow::Result { + let container_spec = container.to_controller(); + match container_spec.platform { + Some(platform) => self.get_platform(platform).await.with_context(|| { + format!( + "Did not find platform {} requested by container {}.", + platform, container_spec.id + ) + })?, + None => { + // TODO: Look for a platform this container could run on + error!( + "No platform specified for container {}. We cannot handle this case yet.", + container_spec.id + ); + anyhow::bail!("No platform specified for container {}", container_spec.id); + } + } + } + + async fn get_platform(&self, platform: String) -> anyhow::Result { + V::ContainerPlatform::find() + .filter(platform.eq(&platform.clone())) + .first(&self.app_state.db_pool) + .await? + } + + pub async fn watch(&self, container: V::Container) -> anyhow::Result { + let container_spec = container.to_controller(); + match container_spec.platform { + Some(platform) => { + let platform_spec = self.get_platform(platform).await.with_context(|| { + format!( + "Did not find platform {} hosting container {}.", + platform, container_spec.id + ) + })?; // TODO: Handle gracefully. This can happen if the platform is removed/lost. + let platform: Box> = + create_platform_from_kind(platform_spec).await?; + // TODO: Run as separate task? + platform.get(container).await? + // TODO: Handle status updates in a compatible way + } + + None => { + anyhow::bail!( + "Cannot watch container {} because it has no platform even though its status is {}.", + container_spec.id, container_spec.status + ); + } + } + } + + pub async fn kill(&self, container: V::Container) -> anyhow::Result { + let container_spec = container.to_controller(); + match container_spec.platform { + Some(platform) => { + let platform_spec = self.get_platform(platform).await.with_context(|| { + format!( + "Did not find platform {} hosting container {}.", + platform, container_spec.id + ) + })?; // TODO: Handle gracefully. This can happen if the platform is removed/lost. + let platform: Box> = + create_platform_from_kind(platform_spec).await?; + // TODO: Run as separate task? + platform.delete(container).await? + } + None => { + anyhow::bail!( + "Cannot kill container {} because it has no platform even though its status is {}.", + container_spec.id, container_spec.status + ); + } + } + } +} diff --git a/src/controllers/mod.rs b/src/controllers/mod.rs new file mode 100644 index 0000000..d70acff --- /dev/null +++ b/src/controllers/mod.rs @@ -0,0 +1,2 @@ +pub mod containers; +pub mod platforms; diff --git a/src/controllers/platforms.rs b/src/controllers/platforms.rs new file mode 100644 index 0000000..6cd4148 --- /dev/null +++ b/src/controllers/platforms.rs @@ -0,0 +1,116 @@ +use crate::platforms::factory::create_platform_from_kind; +use crate::resources::containers::ContainerModelVersion; +use crate::resources::platforms::PlatformModelVersion; +use crate::state::AppState; +use std::sync::Arc; +use tracing::{info, span, Instrument, Level}; +use crate::resources::v1::containers::models::{V1Container}; +use crate::resources::v1::platforms::models::{V1ContainerPlatform}; + +pub struct ContainerPlatformController +where + V: ContainerModelVersion, +{ + version: std::marker::PhantomData, + app_state: Arc, +} + +enum ContainerPlatformStatus { + Started, + Initializing, + Available, + Unavailable, + Stopped, + Terminating, + Terminated, +} + +trait PlatformStatus { + fn get_status(&self) -> ContainerPlatformStatus; + fn set_status(&mut self, status: ContainerPlatformStatus); + +} + +impl PlatformStatus for V1ContainerPlatform { + fn get_status(&self) -> ContainerPlatformStatus { + ContainerPlatformStatus::Available + } + + fn set_status(&mut self, status: ContainerPlatformStatus) { + + } +} + +impl ContainerPlatformController +where + V: PlatformModelVersion, + V::ContainerPlatform: PlatformStatus +{ + pub fn new(app_state: Arc) -> Self { + Self { version: std::marker::PhantomData, app_state } + } + + pub async fn reconcile(&self) -> anyhow::Result<()> { + let span = span!(Level::INFO, "ContainerPlatformController"); + + async { + info!("Starting reconciliation process"); + let paginator = V::ContainerPlatform::find() + .paginate(&self.app_state.db_pool, 100) + .await; + let total_pages = paginator.num_pages().await; + + for page in 1..=total_pages { + let platforms = paginator.fetch_page(page).await; + for mut platform in platforms { + self.handle(platform).await.expect("TODO: panic message"); + } + } + info!("Finished reconciliation process"); + } + .instrument(span) + .await; + + Ok(()) + } + + pub async fn handle(&self, mut platform: V::ContainerPlatform) -> anyhow::Result<()> { + let platform_status: ContainerPlatformStatus = platform.get_status(); + + match platform_status { + ContainerPlatformStatus::Started => { + info!("Platform is started"); + self.initialize(platform).await?; + } + ContainerPlatformStatus::Initializing + | ContainerPlatformStatus::Available + | ContainerPlatformStatus::Unavailable + | ContainerPlatformStatus::Terminating => { + info!("Platform is active, getting status..."); + self.watch(platform).await?; + } + ContainerPlatformStatus::Stopped => { + info!("Platform is stopped"); + self.terminate(platform).await?; + } + ContainerPlatformStatus::Terminated => {} + } + + Ok(()) + } + + pub async fn initialize(&self, platform: V::ContainerPlatform) -> anyhow::Result { + let platform = create_platform_from_kind(platform).await?; + platform + } + + pub async fn watch(&self, platform: V::ContainerPlatform) -> anyhow::Result { + let platform = create_platform_from_kind(platform).await?; + platform + } + + pub async fn terminate(&self, platform: V::ContainerPlatform) -> anyhow::Result { + let platform = create_platform_from_kind(platform).await?; + platform + } +} diff --git a/src/conversion/macros.rs b/src/conversion/macros.rs new file mode 100644 index 0000000..34f1a91 --- /dev/null +++ b/src/conversion/macros.rs @@ -0,0 +1,28 @@ +#[macro_export] +macro_rules! impl_entity_conversion { + ( + model = $model:ty, + entity = $entity:ty, + fields = [ $( $field:ident ),* $(,)? ], + extra_to_entity = { $( $extra_to:tt )* }, + extra_from_entity = { $( $extra_from:tt )* } + ) => { + impl $crate::conversion::ToEntity<$entity> for $model { + fn to_entity(&self) -> $entity { + $entity { + $( $field: self.$field.clone(), )* + $( $extra_to )* + } + } + } + + impl $crate::conversion::FromEntity<$entity> for $model { + fn from_entity(entity: &$entity) -> Self { + Self { + $( $field: entity.$field.clone(), )* + $( $extra_from )* + } + } + } + }; +} diff --git a/src/conversion/mod.rs b/src/conversion/mod.rs new file mode 100644 index 0000000..7a2d89c --- /dev/null +++ b/src/conversion/mod.rs @@ -0,0 +1,9 @@ +pub mod macros; + +pub trait ToEntity { + fn to_entity(&self) -> E; +} + +pub trait FromEntity: Sized { + fn from_entity(entity: &E) -> Self; +} diff --git a/src/db.rs b/src/db.rs index 6516485..f45d580 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,11 +1,11 @@ // src/db.rs -use crate::config::CONFIG; +use crate::config::SERVER_CONFIG; use sea_orm::{ConnectionTrait, Database, DatabaseConnection, DbErr, Schema}; pub type DbPool = DatabaseConnection; pub async fn init_db() -> Result { - let database_url = &CONFIG.database_url; + let database_url = &SERVER_CONFIG.database_url; println!("Connecting to database at: {}", database_url); // Create the data directory if it doesn't exist @@ -100,7 +100,7 @@ async fn create_tables(db: &DbPool) -> Result<(), DbErr> { .if_not_exists(), ), ) - .await?; + .await?; db.execute( db.get_database_backend().build( diff --git a/src/entities/containers.rs b/src/entities/containers.rs index 39b2950..38fd1d0 100644 --- a/src/entities/containers.rs +++ b/src/entities/containers.rs @@ -1,4 +1,3 @@ -// src/entities/containers.rs use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; @@ -12,11 +11,13 @@ use crate::resources::v1::containers::models::{ }; use crate::resources::v1::volumes::models::V1VolumePath; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] #[sea_orm(table_name = "containers")] pub struct Model { #[sea_orm(primary_key, column_type = "Text", auto_increment = false)] pub id: String, + pub version: i32, pub namespace: String, pub name: String, #[sea_orm(unique, column_type = "Text")] @@ -169,57 +170,4 @@ impl Model { } } - /// Construct a full V1Container from the current model row. - /// Returns a serde_json Error if any JSON parsing in subfields fails. - pub fn to_v1_container(&self) -> Result { - let env = self.parse_env()?; - let volumes = self.parse_volumes()?; - let status = self.parse_status()?; - let labels = self.parse_labels()?; - let meters = self.parse_meters()?; - let resources = self.parse_resources()?; - let ssh_keys = self.parse_ssh_keys()?; - let ports = self.parse_ports()?; - let authz = self.parse_authz()?; - let health_check = self.parse_health_check()?; - - // Build metadata; fill with defaults or unwrap as needed - let metadata = crate::models::V1ResourceMeta { - name: self.name.clone(), - namespace: self.namespace.clone(), - id: self.id.clone(), - owner: self.owner.clone(), - owner_ref: self.owner_ref.clone(), - created_at: self.created_at.timestamp(), - updated_at: self.updated_at.timestamp(), - created_by: self.created_by.clone().unwrap_or_default(), - labels, - }; - - // Construct final V1Container - let container = V1Container { - kind: "Container".to_owned(), // or use default_container_kind() if needed - platform: self.platform.clone().unwrap_or_default(), - metadata, - image: self.image.clone(), - env, - command: self.command.clone(), - args: self.args.clone(), - volumes, - accelerators: self.accelerators.clone(), - meters, - restart: self.restart.clone(), - queue: self.queue.clone(), - timeout: self.timeout.clone(), - status, - resources, - health_check, - ssh_keys, - ports: ports.clone(), - proxy_port: self.proxy_port.clone(), - authz, - }; - - Ok(container) - } } diff --git a/src/entities/mod.rs b/src/entities/mod.rs index fcf533b..b3ab17f 100644 --- a/src/entities/mod.rs +++ b/src/entities/mod.rs @@ -4,3 +4,6 @@ pub mod namespaces; pub mod processors; pub mod secrets; pub mod volumes; +pub mod platforms; + + diff --git a/src/entities/platforms.rs b/src/entities/platforms.rs new file mode 100644 index 0000000..db4f73f --- /dev/null +++ b/src/entities/platforms.rs @@ -0,0 +1,16 @@ +use sea_orm::DeriveEntityModel; +use serde::{Deserialize, Serialize}; +use sea_orm::entity::prelude::*; + + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "platforms")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text", auto_increment = false)] + pub id: String, + pub version: i32, + pub name: String +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} diff --git a/src/handlers/v1/namespaces.rs b/src/handlers/v1/namespaces.rs index 9147035..9c72ea5 100644 --- a/src/handlers/v1/namespaces.rs +++ b/src/handlers/v1/namespaces.rs @@ -1,6 +1,6 @@ // src/handlers/containers.rs -use crate::config::CONFIG; +use crate::config::SERVER_CONFIG; use crate::entities::namespaces::{self, ActiveModel as NamespaceActiveModel}; use crate::handlers::v1::volumes::ensure_volume; use crate::models::V1UserProfile; @@ -158,7 +158,7 @@ pub async fn create_namespace( &namespace_entity.owner.clone(), &format!( "s3://{}/data/{}", - &CONFIG.bucket_name, + &SERVER_CONFIG.bucket_name, &namespace_entity.name.clone() ), &namespace_entity.created_by.clone(), @@ -330,7 +330,7 @@ pub async fn ensure_ns_and_resources( name, owner, owner, - format!("s3://{}", &CONFIG.bucket_name).as_str(), + format!("s3://{}", &SERVER_CONFIG.bucket_name).as_str(), created_by, labels, ) diff --git a/src/lib.rs b/src/lib.rs index c41f6b3..e0114cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,23 +28,22 @@ pub mod streams; pub mod validate; pub mod volumes; -use crate::config::CONFIG; +pub mod conversion; +pub mod controllers; +pub mod platforms; + +use crate::config::SERVER_CONFIG; use crate::handlers::v1::namespaces::ensure_namespace; use crate::handlers::v1::volumes::ensure_volume; use axum::Router; use db::init_db; -use rdkafka::admin::AdminClient; -use rdkafka::producer::FutureProducer; -use rdkafka::ClientConfig; use routes::create_routes; use sea_orm::DatabaseConnection; use state::AppState; use state::MessageQueue; -use std::env; use std::sync::Arc; use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; -use url::Url; /// Create and return the application state. pub async fn create_app_state() -> Result> { @@ -53,46 +52,9 @@ pub async fn create_app_state() -> Result> println!("Database pool created"); // Initialize the appropriate message queue based on configuration - let message_queue = match CONFIG.message_queue_type.to_lowercase().as_str() { + let message_queue = match SERVER_CONFIG.message_queue_type.to_lowercase().as_str() { "redis" => { - let redis_url = match &CONFIG.redis_url { - Some(url) if !url.is_empty() => { - // Redis URL exists, so use it directly but also parse it to set env vars - if let Ok(parsed_url) = Url::parse(url) { - // Extract and set host - if let Some(host) = parsed_url.host_str() { - env::set_var("REDIS_HOST", host); - } - - // Extract and set port - if let Some(port) = parsed_url.port() { - env::set_var("REDIS_PORT", port.to_string()); - } else { - // Default redis port if not specified in URL - env::set_var("REDIS_PORT", "6379"); - } - - // Extract and set password if present - if let Some(password) = parsed_url.password() { - env::set_var("REDIS_PASSWORD", password); - } - } - - url.clone() - } - _ => { - // Redis URL not present or empty, build from components - let host = &CONFIG.redis_host; - let port = &CONFIG.redis_port; - - match &CONFIG.redis_password { - Some(password) if !password.is_empty() => { - format!("redis://:{}@{}:{}", password, host, port) - } - _ => format!("redis://{}:{}", host, port), - } - } - }; + let redis_url = &SERVER_CONFIG.redis_url; // Create the Redis client using the constructed URL let redis_client = Arc::new(redis::Client::open(redis_url.as_str())?); @@ -101,17 +63,6 @@ pub async fn create_app_state() -> Result> client: redis_client, } } - "kafka" => { - let mut client_config = ClientConfig::new(); - let kafka_config = client_config - .set("bootstrap.servers", &CONFIG.kafka_bootstrap_servers) - .set("message.timeout.ms", &CONFIG.kafka_timeout_ms); - - let producer = Arc::new(kafka_config.clone().create::()?); - let admin = Arc::new(kafka_config.create::>()?); - - MessageQueue::Kafka { producer, admin } - } unsupported => { return Err(format!("Unsupported message queue type: {}", unsupported).into()) } @@ -151,8 +102,8 @@ pub async fn ensure_base_resources( match ensure_namespace( db_pool, "root", - &CONFIG.root_owner, - &CONFIG.root_owner, + &SERVER_CONFIG.root_owner, + &SERVER_CONFIG.root_owner, None, ) .await @@ -165,8 +116,8 @@ pub async fn ensure_base_resources( db_pool, "root", "root", - &CONFIG.root_owner, - format!("s3://{}", &CONFIG.bucket_name).as_str(), + &SERVER_CONFIG.root_owner, + format!("s3://{}", &SERVER_CONFIG.bucket_name).as_str(), "root", None, ) diff --git a/src/main.rs b/src/main.rs index 7fd9f0f..e405eef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,8 @@ mod models; use std::path::Path; use crate::cli::{ - ApiKeyActions, AuthCommands, Cli, Commands, CreateCommands, DeleteCommands, GetCommands, + ApiKeyActions, AuthCommands, Cli, Commands, ConfigCommands, CreateCommands, + CurrentConfigActions, DeleteCommands, GetCommands, HeadscaleApiKeyActions, HeadscaleCommands, ProxyCommands, SelectCommands, }; use clap::Parser; @@ -29,10 +30,16 @@ async fn main() -> Result<(), Box> { Commands::Serve { host, port, - internal_auth, + disable_internal_auth, auth_port, + docker, } => { - commands::serve_cmd::execute(host, port, internal_auth, auth_port).await?; + if docker { + commands::serve_cmd::launch_docker(port, disable_internal_auth, auth_port).await?; + } else { + commands::serve_cmd::launch_server(host, port, disable_internal_auth, auth_port) + .await?; + } } Commands::Sync { command } => match command { SyncCommands::Volumes { @@ -125,8 +132,14 @@ async fn main() -> Result<(), Box> { Commands::Logs { name, namespace } => { commands::log_cmd::fetch_container_logs(name, namespace).await?; } - Commands::Login { url, auth, hub } => { - commands::login_cmd::execute(url, auth, hub).await?; + Commands::Login { + url, + name, + update, + auth, + hub, + } => { + commands::login_cmd::execute(url, name, update, auth, hub).await?; } Commands::Exec(args) => { commands::exec_cmd::exec_cmd(args).await?; @@ -147,6 +160,32 @@ async fn main() -> Result<(), Box> { } }, }, + Commands::Config { command } => match command { + ConfigCommands::Show => { + commands::config_cmd::show_config().await?; + } + ConfigCommands::Current { action } => match action { + CurrentConfigActions::Show => { + commands::config_cmd::show_current().await?; + } + CurrentConfigActions::Set { server } => { + commands::config_cmd::set_current(&server).await?; + } + }, + }, + Commands::Headscale { command } => match command { + HeadscaleCommands::Apikey { action } => match action { + HeadscaleApiKeyActions::Create { expiration } => { + commands::headscale_cmd::create_api_key(&expiration).await?; + } + HeadscaleApiKeyActions::Validate { prefix } => { + commands::headscale_cmd::validate_api_key(&prefix).await?; + } + HeadscaleApiKeyActions::Revoke { prefix } => { + commands::headscale_cmd::revoke_api_key(&prefix).await?; + } + }, + }, } Ok(()) diff --git a/src/middleware.rs b/src/middleware.rs index cc14099..a2d5ba5 100644 --- a/src/middleware.rs +++ b/src/middleware.rs @@ -1,4 +1,5 @@ use crate::auth; +use crate::config::SERVER_CONFIG; use crate::models::V1UserProfile; use crate::AppState; use axum::{ @@ -86,14 +87,11 @@ async fn internal_auth( } async fn external_auth(auth_header: &String, mut request: Request, next: Next) -> Response { - let config = crate::config::GlobalConfig::read().unwrap(); - - let auth_url = config - .get_current_server_config() - .unwrap() - .auth_server - .as_ref() - .unwrap(); + let auth_url = SERVER_CONFIG + .auth + .url + .clone() + .expect("No external auth URL configured."); println!("🔐 Making auth request to: {}", auth_url); diff --git a/src/platforms/docker.rs b/src/platforms/docker.rs new file mode 100644 index 0000000..c8298e6 --- /dev/null +++ b/src/platforms/docker.rs @@ -0,0 +1,204 @@ +use super::local::LocalShell; +use super::platform::{ + ContainerPlatform, ContainerPlatformBuilder, ContainerPlatformStatus, PlatformConnection, + ShellConnection, +}; +use crate::resources::v1::containers::models::v1::V1Container; +use crate::resources::v1::containers::models::ContainerModelVersion; +use crate::resources::v1::containers::platforms::ssh::SSHConnection; +use async_trait::async_trait; +use serde::Deserialize; + +pub struct DockerPlatform +where + C: PlatformConnection + ShellConnection, + V: ContainerModelVersion, +{ + pub(crate) connection: C, + pub(crate) version: std::marker::PhantomData, +} + +struct DockerContainer { + pub image: String, +} + +trait ToDockerContainer { + fn to_docker(&self) -> DockerContainer; +} + +trait FromDockerContainer { + fn from_docker(docker_container: DockerContainer) -> Self; +} + +impl ToDockerContainer for V1Container { + fn to_docker(&self) -> DockerContainer { + DockerContainer { + image: self.image.clone(), + } + } +} + +impl FromDockerContainer for V1Container { + fn from_docker(docker_container: DockerContainer) -> Self { + V1Container { + image: docker_container.image, + ..Default::default() + } + } +} + +#[allow(non_snake_case)] +#[derive(Debug, Deserialize)] +struct DockerInspect { + pub Id: String, + pub RepoTags: Option>, + pub RepoDigests: Option>, + pub Parent: Option, + pub Comment: Option, + pub Created: Option, + pub DockerVersion: Option, + pub Author: Option, + pub Config: Option, + pub Architecture: Option, + pub Os: Option, + pub Size: Option, + pub GraphDriver: Option, + pub RootFS: Option, + pub Metadata: Option, +} + +#[allow(non_snake_case)] +#[derive(Debug, Deserialize)] +struct DockerInspectConfig { + pub Hostname: Option, + pub Domainname: Option, + pub User: Option, + pub AttachStdin: Option, + pub AttachStdout: Option, + pub AttachStderr: Option, + pub ExposedPorts: Option>, + pub Tty: Option, + pub OpenStdin: Option, + pub StdinOnce: Option, + pub Env: Option>, + pub Cmd: Option>, + pub Image: Option, + pub Volumes: Option, + pub WorkingDir: Option, + pub Entrypoint: Option>, + pub OnBuild: Option, + pub Labels: Option>, + pub StopSignal: Option, +} + +#[allow(non_snake_case)] +#[derive(Debug, Deserialize)] +struct DockerInspectGraphDriver { + pub Data: Option, + pub Name: Option, +} + +#[allow(non_snake_case)] +#[derive(Debug, Deserialize)] +struct DockerInspectGraphDriverData { + pub LowerDir: Option, + pub MergedDir: Option, + pub UpperDir: Option, + pub WorkDir: Option, +} + +#[allow(non_snake_case)] +#[derive(Debug, Deserialize)] +struct DockerInspectRootFS { + pub Type: Option, + pub Layers: Option>, +} + +#[allow(non_snake_case)] +#[derive(Debug, Deserialize)] +struct DockerInspectMetadata { + pub LastTagTime: Option, +} + +impl ToDockerContainer for DockerInspect { + fn to_docker(&self) -> DockerContainer { + DockerContainer { + image: self + .Config + .as_ref() + .and_then(|config| config.Image.clone()) + .unwrap_or_default(), + } + } +} + +impl ContainerPlatformBuilder for DockerPlatform +where + C: PlatformConnection + ShellConnection, + V: ContainerModelVersion, +{ + fn from_spec(spec: V::ContainerPlatform) -> Self { + let connection = C::from_spec(spec); + DockerPlatform { + connection, + version: std::marker::PhantomData, + } + } +} + +#[async_trait] +impl ContainerPlatform for DockerPlatform +where + C: PlatformConnection + ShellConnection, + V: ContainerModelVersion, + V::Container: FromDockerContainer + ToDockerContainer, +{ + async fn create(&self, container: V::Container) -> anyhow::Result { + let docker_container = container.to_docker(); + let command = format!("docker run -d {}", docker_container.image); + let id = self.connection.run_command(&command).await?; + Ok(container) + } + + async fn get(&self, id: &str) -> anyhow::Result { + let command = format!("docker inspect {}", id); + let output = self.connection.run_command(&command).await?; + let docker_inspect = serde_json::from_str::(&output)?; + let docker_container = docker_inspect.to_docker(); + let container = V::Container::from_docker(docker_container); + Ok(container) + } + + async fn delete(&self, id: &str) -> anyhow::Result { + let command = format!("docker rm -f {}", id); + let output = self.connection.run_command(&command).await?; + if !output.is_empty() { + return Err(anyhow::anyhow!("Failed to delete container: {}", output)); + } + Ok(()) + } + + async fn logs(&self, id: &str) -> anyhow::Result { + let command = format!("docker logs --tail 1000 {}", id); + let logs = self.connection.run_command(&command).await?; + Ok(logs) + } + + async fn exec(&self, id: &str, command: &str) -> anyhow::Result { + let command = format!("docker exec -it {} {}", id, command); + let output = self.connection.run_command(&command).await?; + Ok(output) + } + + async fn status(&self) -> anyhow::Result { + // TODO: Check capacity + if self.connection.is_connected().await? { + Ok(ContainerPlatformStatus::Ready) + } else { + Ok(ContainerPlatformStatus::DoNotSchedule) + } + } +} + +pub(crate) type LocalDockerPlatform = DockerPlatform, V>; +pub(crate) type RemoteDockerPlatform = DockerPlatform, V>; diff --git a/src/platforms/ec2.rs b/src/platforms/ec2.rs new file mode 100644 index 0000000..b69a162 --- /dev/null +++ b/src/platforms/ec2.rs @@ -0,0 +1,4 @@ +use super::docker::DockerPlatform; +use super::ssh::SSHConnection; + +pub(crate) type EC2Platform = DockerPlatform, V>; diff --git a/src/platforms/factory.rs b/src/platforms/factory.rs new file mode 100644 index 0000000..1b68cdc --- /dev/null +++ b/src/platforms/factory.rs @@ -0,0 +1,48 @@ +use super::platform::ContainerPlatform; +use super::{ + docker::LocalDockerPlatform, docker::RemoteDockerPlatform, ec2::EC2Platform, gce::GCEPlatform, + nebulous::NebulousPlatform, +}; +use crate::resources::v1::containers::models::ContainerModelVersion; + +#[derive(Debug)] +enum PlatformKind { + LocalDocker, + RemoteDocker, + EC2, + GCE, + Nebulous, +} + +impl std::str::FromStr for PlatformKind { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "local-docker" => Ok(Self::LocalDocker), + "remote-docker" => Ok(Self::RemoteDocker), + "ec2" => Ok(Self::EC2), + "gce" => Ok(Self::GCE), + "nebulous" => Ok(Self::Nebulous), + _ => Err(anyhow::anyhow!("Unknown platform kind: {}", s)), + } + } +} + +pub(crate) fn create_platform_from_kind( + spec: V::ContainerPlatform, +) -> anyhow::Result>> +where + V: ContainerModelVersion, + V::ContainerPlatform: ExtractPlatformKind, +{ + let kind = spec.get_kind()?; + + match kind { + PlatformKind::LocalDocker => Ok(LocalDockerPlatform::::from_spec(spec)), + PlatformKind::RemoteDocker => Ok(RemoteDockerPlatform::::from_spec(spec)), + PlatformKind::EC2 => Ok(EC2Platform::::from_spec(spec)), + PlatformKind::GCE => Ok(GCEPlatform::::from_spec(spec)), + PlatformKind::Nebulous => Ok(NebulousPlatform::::from_spec(spec)), + } +} diff --git a/src/platforms/gce.rs b/src/platforms/gce.rs new file mode 100644 index 0000000..6113100 --- /dev/null +++ b/src/platforms/gce.rs @@ -0,0 +1,76 @@ +use super::docker::DockerPlatform; +use super::platform::{ContainerPlatform, ContainerPlatformBuilder, ContainerPlatformStatus}; +use super::ssh::SSHConnection; +use crate::resources::v1::containers::models::ContainerModelVersion; +use async_trait::async_trait; + +pub struct GCEPlatform { + inner: DockerPlatform, V>, + gce_zone: String, + gce_instance: String, +} + +impl GCEPlatform { + pub fn new(gce_zone: String, gce_instance: String, connection: SSHConnection) -> Self { + let inner = DockerPlatform { + connection, + version: std::marker::PhantomData, + }; + GCEPlatform { + inner, + gce_zone, + gce_instance, + } + } +} + +impl ContainerPlatformBuilder for GCEPlatform +where + V: ContainerModelVersion, +{ + fn validate(spec: &V::ContainerPlatform) -> anyhow::Result<()> { + if spec.gce.gce_zone.is_empty() { + return Err(anyhow::anyhow!("GCE zone is required")); + } + if spec.gce.gce_instance.is_empty() { + return Err(anyhow::anyhow!("GCE instance is required")); + } + Ok(()) + } + fn from_spec(spec: V::ContainerPlatform) -> Self { + let gce_zone = spec.gce.gce_zone; + let gce_instance = spec.gce.gce_instance; + let connection = SSHConnection::from_spec(spec); + GCEPlatform::new(gce_zone, gce_instance, connection) + } +} + +#[async_trait] +impl ContainerPlatform for GCEPlatform +where + V: ContainerModelVersion, +{ + async fn create(&self, container: V::Container) -> anyhow::Result { + self.inner.create(container).await + } + + async fn get(&self, id: &str) -> anyhow::Result { + self.inner.get(id).await + } + + async fn delete(&self, id: &str) -> anyhow::Result { + self.inner.delete(id).await + } + + async fn logs(&self, id: &str) -> anyhow::Result { + todo!() + } + + async fn exec(&self, id: &str, command: &str) -> anyhow::Result { + todo!() + } + + async fn status(&self) -> anyhow::Result { + todo!() + } +} diff --git a/src/platforms/http.rs b/src/platforms/http.rs new file mode 100644 index 0000000..2379baf --- /dev/null +++ b/src/platforms/http.rs @@ -0,0 +1,80 @@ +use async_trait::async_trait; +use crate::resources::v1::containers::models::v1::V1ContainerPlatform; +use crate::resources::v1::containers::models::ContainerModelVersion; +use crate::resources::v1::containers::platforms::platform::{PlatformConnection, RESTConnection}; + +pub struct HTTPConnection { + version: std::marker::PhantomData, + host: String, + health_check_path: String, +} + +trait GetHttpConnectionDetails { + fn host(&self) -> String; + fn health_check_path(&self) -> String; +} + +impl GetHttpConnectionDetails for V1ContainerPlatform { + fn host(&self) -> String { + todo!() + } + + fn health_check_path(&self) -> String { + todo!() + } +} + +#[async_trait] +impl PlatformConnection for HTTPConnection +where + V: ContainerModelVersion, +{ + fn from_spec(spec: V::ContainerPlatform) -> Self { + todo!() + } + + async fn connect(&self) -> anyhow::Result<()> { + self.is_connected().await?; + Ok(()) + } + + async fn disconnect(&self) -> anyhow::Result<()> { + Ok(()) + } + + async fn is_connected(&self) -> bool { + match self.get(&self.health_check_path).await { + Ok(_) => true, + Err(_) => false, + } + } +} + +#[async_trait] +impl RESTConnection for HTTPConnection +where + V: ContainerModelVersion, +{ + async fn get(&self, path: &str) -> anyhow::Result { + let client = reqwest::Client::new(); + client + .get(format!("{}/{}", self.host, path)) + .send() + .await? + .text() + .await + .map_err(|e| anyhow::anyhow!("Failed to read response: {}", e)) + } + + async fn post(&self, path: &str, body: &str) -> anyhow::Result { + let client = reqwest::Client::new(); + client + .post(format!("{}/{}", self.host, path)) + .body(body.to_string()) + .send() + .await? + .text() + .await + .map_err(|e| anyhow::anyhow!("Failed to read response: {}", e)) + } +} diff --git a/src/platforms/local.rs b/src/platforms/local.rs new file mode 100644 index 0000000..9bd4496 --- /dev/null +++ b/src/platforms/local.rs @@ -0,0 +1,63 @@ +use super::platform::{PlatformConnection, ShellConnection}; +use crate::resources::v1::containers::models::ContainerModelVersion; +use anyhow::Context; +use async_trait::async_trait; + +pub struct LocalShell { + version: std::marker::PhantomData, +} + +#[async_trait] +impl PlatformConnection for LocalShell +where + V: ContainerModelVersion, +{ + fn from_spec(spec: V::ContainerPlatform) -> Self { + LocalShell { + version: std::marker::PhantomData, + } + } + async fn connect(&self) -> anyhow::Result<()> { + match self.is_connected().await { + true => Ok(()), + false => Err(anyhow::anyhow!("Failed to connect to local shell")), + } + } + + async fn disconnect(&self) -> anyhow::Result<()> { + Ok(()) + } + + async fn is_connected(&self) -> bool { + let output = std::process::Command::new("sh") + .arg("-c") + .arg("which sh") + .output() + .await?; + output.status.success() + } +} + +#[async_trait] +impl ShellConnection for LocalShell +where + V: ContainerModelVersion, +{ + async fn run_command(&self, command: &str) -> anyhow::Result { + let output = std::process::Command::new("sh") + .arg("-c") + .arg(command) + .output() + .await?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "Command failed with status: {}", + output.status + )); + } + let stdout = String::from_utf8(&output.stdout).with_context(|| { + format!("Output for 'sh -c {}' is incompatible with UTF-8.", command) + })?; + Ok(stdout) + } +} diff --git a/src/platforms/mod.rs b/src/platforms/mod.rs new file mode 100644 index 0000000..7df558a --- /dev/null +++ b/src/platforms/mod.rs @@ -0,0 +1,9 @@ +pub mod docker; +pub mod ec2; +pub mod factory; +pub mod gce; +pub mod local; +mod nebulous; +pub mod platform; +pub mod ssh; +mod http; diff --git a/src/platforms/nebulous.rs b/src/platforms/nebulous.rs new file mode 100644 index 0000000..f04781f --- /dev/null +++ b/src/platforms/nebulous.rs @@ -0,0 +1,61 @@ +use async_trait::async_trait; +use crate::resources::v1::containers::models::ContainerModelVersion; +use crate::resources::v1::containers::platforms::http::HTTPConnection; +use crate::resources::v1::containers::platforms::platform::{ContainerPlatform, ContainerPlatformBuilder, ContainerPlatformStatus, PlatformConnection, RESTConnection}; + +pub struct NebulousPlatform +where + C: PlatformConnection + RESTConnection, + V: ContainerModelVersion, +{ + version: std::marker::PhantomData, + connection: C, +} + +impl ContainerPlatformBuilder for NebulousPlatform +where + C: PlatformConnection + RESTConnection, + V: ContainerModelVersion, +{ + fn from_spec(spec: V::ContainerPlatform) -> Self { + let connection = C::from_spec(spec); + NebulousPlatform { + version: std::marker::PhantomData, + connection, + } + } +} + +#[async_trait] +impl ContainerPlatform for NebulousPlatform +where + C: PlatformConnection + RESTConnection, + V: ContainerModelVersion, +{ + + async fn create(&self, container: V::Container) -> anyhow::Result { + todo!() + } + + async fn get(&self, id: &str) -> anyhow::Result { + todo!() + } + + async fn delete(&self, id: &str) -> anyhow::Result { + todo!() + } + + async fn logs(&self, id: &str) -> anyhow::Result { + todo!() + } + + async fn exec(&self, id: &str, command: &str) -> anyhow::Result { + todo!() + } + + async fn status(&self) -> anyhow::Result { + todo!() + } +} + +pub(crate) type HttpNebulousPlatform = NebulousPlatform, V>; diff --git a/src/platforms/platform.rs b/src/platforms/platform.rs new file mode 100644 index 0000000..a8e51ea --- /dev/null +++ b/src/platforms/platform.rs @@ -0,0 +1,61 @@ +use crate::resources::containers::ContainerModelVersion; +use crate::resources::platforms::PlatformModelVersion; +use async_trait::async_trait; + +pub enum ContainerPlatformStatus { + Initializing, + Ready, + Unavailable, + DoNotSchedule, + Terminating, +} + + +pub trait ContainerPlatformBuilder { + + fn validate(spec: &V::ContainerPlatform) -> anyhow::Result<()> { + let _ = Self::from_spec(spec.clone()); + Ok(()) + } + + fn from_spec(spec: V::ContainerPlatform) -> Self; +} + + +#[async_trait] +pub trait ContainerPlatform { + async fn create(&self, container: C::Container) -> anyhow::Result; + async fn get(&self, id: &str) -> anyhow::Result; + async fn delete(&self, id: &str) -> anyhow::Result; + async fn logs(&self, id: &str) -> anyhow::Result; + + // TODO: Design API + async fn exec(&self, id: &str, command: &str) -> anyhow::Result; + + // TODO: Add platform-level monitoring, status, and properties + async fn status(&self) -> anyhow::Result; +} + +#[async_trait] +pub trait PlatformConnection { + fn validate(spec: &V::ContainerPlatform) -> anyhow::Result<()> { + let _ = Self::from_spec(spec.clone()); + Ok(()) + } + fn from_spec(spec: V::ContainerPlatform) -> Self; + async fn connect(&self) -> anyhow::Result<()>; + async fn disconnect(&self) -> anyhow::Result<()>; + async fn is_connected(&self) -> bool; +} + +#[async_trait] +pub trait ShellConnection { + async fn run_command(&self, command: &str) -> anyhow::Result; +} + +#[async_trait] +pub trait RESTConnection { + async fn get(&self, path: &str) -> anyhow::Result; + + async fn post(&self, path: &str, body: &str) -> anyhow::Result; +} diff --git a/src/platforms/ssh.rs b/src/platforms/ssh.rs new file mode 100644 index 0000000..575a70e --- /dev/null +++ b/src/platforms/ssh.rs @@ -0,0 +1,54 @@ +use async_trait::async_trait; +use crate::resources::v1::containers::models::ContainerModelVersion; +use crate::resources::v1::containers::platforms::platform::{PlatformConnection, ShellConnection}; + +pub struct SSHConnection { + version: std::marker::PhantomData, + pub host: String, + pub port: u16, + pub username: String, + pub private_key: String, +} + + +#[async_trait] +impl PlatformConnection for SSHConnection +where + V: ContainerModelVersion, +{ + fn from_spec(spec: V::ContainerPlatform) -> Self { + SSHConnection { + version: std::marker::PhantomData, + host: spec.ssh.host, + port: spec.ssh.port, + username: spec.ssh.username, + private_key: spec.ssh.private_key, + } + } + async fn connect(&self) -> anyhow::Result<()> { + // Implement SSH connection logic here + Ok(()) + } + + async fn disconnect(&self) -> anyhow::Result<()> { + // Implement SSH disconnection logic here + Ok(()) + } + + async fn is_connected(&self) -> bool { + // Implement check for SSH connection status + true + } +} + + +#[async_trait] +impl ShellConnection for SSHConnection +where + V: ContainerModelVersion, +{ + async fn run_command(&self, method: &str) -> anyhow::Result { + // Implement SSH command execution logic here + Ok(format!("Executed command: {}", method)) + } +} diff --git a/src/resources/containers.rs b/src/resources/containers.rs new file mode 100644 index 0000000..6c532c6 --- /dev/null +++ b/src/resources/containers.rs @@ -0,0 +1,17 @@ +pub trait ContainerModelVersion { + type Container; + type ContainerMetaRequest; + type ContainerHealthCheck; + type ContainerStatus; + type EnvVar; + type ErrorResponse; + type ContainerResource; + type Port; + type PortRequest; + type SSHKey; + + type UpdateContainer; + + type ContainerSearch; + +} diff --git a/src/resources/mod.rs b/src/resources/mod.rs index a3a6d96..ea2bbcc 100644 --- a/src/resources/mod.rs +++ b/src/resources/mod.rs @@ -1 +1,3 @@ pub mod v1; +pub mod containers; +pub mod platforms; diff --git a/src/resources/platforms.rs b/src/resources/platforms.rs new file mode 100644 index 0000000..80fbbae --- /dev/null +++ b/src/resources/platforms.rs @@ -0,0 +1,4 @@ +pub trait PlatformModelVersion { + type ContainerPlatform; +} + diff --git a/src/resources/v1/containers/base.rs b/src/resources/v1/containers/base.rs index 983024b..cd40615 100644 --- a/src/resources/v1/containers/base.rs +++ b/src/resources/v1/containers/base.rs @@ -1,7 +1,7 @@ use crate::agent::agent::create_agent_key; use crate::agent::aws::create_s3_scoped_user; -use crate::config::GlobalConfig; -use crate::config::CONFIG; +use crate::config::ClientConfig; +use crate::config::SERVER_CONFIG; use crate::entities::containers; use crate::handlers::v1::volumes::ensure_volume; use crate::models::{V1CreateAgentKeyRequest, V1UserProfile}; @@ -184,7 +184,7 @@ pub trait ContainerPlatform { model: &containers::Model, db: &DatabaseConnection, ) -> HashMap { - let config = GlobalConfig::read().unwrap(); + let config = ClientConfig::read().unwrap(); let mut env = HashMap::new(); debug!("Getting agent key"); @@ -196,7 +196,10 @@ pub trait ContainerPlatform { } }; - let source = format!("s3://{}/data/{}", CONFIG.bucket_name, model.namespace); + let source = format!( + "s3://{}/data/{}", + SERVER_CONFIG.bucket_name, model.namespace + ); debug!("Ensuring volume: {:?}", source); let _ = match ensure_volume( @@ -219,7 +222,9 @@ pub trait ContainerPlatform { debug!("Creating s3 token"); let s3_token = - match create_s3_scoped_user(&CONFIG.bucket_name, &model.namespace, &model.id).await { + match create_s3_scoped_user(&SERVER_CONFIG.bucket_name, &model.namespace, &model.id) + .await + { Ok(token) => token, Err(e) => { error!("Error creating s3 token: {:?}", e); @@ -248,14 +253,14 @@ pub trait ContainerPlatform { ); env.insert( "RCLONE_CONFIG_S3REMOTE_REGION".to_string(), - CONFIG.bucket_region.clone(), + SERVER_CONFIG.bucket_region.clone(), ); env.insert("RCLONE_S3_NO_CHECK_BUCKET".to_string(), "true".to_string()); env.insert("NEBU_API_KEY".to_string(), agent_key.unwrap()); env.insert( "NEBU_SERVER".to_string(), config - .get_current_server_config() + .get_current_server() .unwrap() .server .as_ref() @@ -314,19 +319,21 @@ pub trait ContainerPlatform { } async fn get_tailscale_client(&self) -> TailscaleClient { - let tailscale_api_key = CONFIG - .tailscale_api_key + let tailscale_api_key = SERVER_CONFIG + .tailscale .clone() - .expect("TAILSCALE_API_KEY not found in config"); + .expect("No Tailscale configuration") + .api_key; debug!("Tailscale key: {}", tailscale_api_key); TailscaleClient::new(tailscale_api_key) } async fn get_tailscale_device_key(&self, model: &containers::Model) -> String { - let tailnet = CONFIG - .tailscale_tailnet + let tailnet = SERVER_CONFIG + .tailscale .clone() - .expect("tailscale_tailnet not found in config"); + .expect("No Tailscale configuration") + .tailnet; debug!("Tailnet: {}", tailnet); @@ -386,7 +393,7 @@ pub trait ContainerPlatform { &self, user_profile: &V1UserProfile, ) -> Result> { - let config = crate::config::GlobalConfig::read().unwrap(); + let config = crate::config::ClientConfig::read().unwrap(); debug!("[DEBUG] get_agent_key: Entering function"); debug!("[DEBUG] get_agent_key: user_profile = {:?}", user_profile); @@ -405,7 +412,7 @@ pub trait ContainerPlatform { }; debug!("[DEBUG] get_agent_key: Getting server config"); - let server_config = match config.get_current_server_config() { + let server_config = match config.get_current_server() { Some(cfg) => cfg, None => { error!("[ERROR] get_agent_key: No current server config found"); diff --git a/src/resources/v1/containers/conversion.rs b/src/resources/v1/containers/conversion.rs new file mode 100644 index 0000000..3b9430c --- /dev/null +++ b/src/resources/v1/containers/conversion.rs @@ -0,0 +1,117 @@ +use crate::conversion::{FromEntity, ToEntity}; +use crate::entities; +use crate::resources::v1::containers::models::V1Container; + +impl FromEntity for V1Container +{ + /// Construct a full V1Container from the current model row. + /// Returns a serde_json Error if any JSON parsing in subfields fails. + fn from_entity(entity: entities::containers::Model) -> Result { + let env = entity.parse_env()?; + let volumes = entity.parse_volumes()?; + let status = entity.parse_status()?; + let labels = entity.parse_labels()?; + let meters = entity.parse_meters()?; + let resources = entity.parse_resources()?; + let ssh_keys = entity.parse_ssh_keys()?; + let ports = entity.parse_ports()?; + let authz = entity.parse_authz()?; + let health_check = entity.parse_health_check()?; + + // Build metadata; fill with defaults or unwrap as needed + let metadata = crate::models::V1ResourceMeta { + name: entity.name.clone(), + namespace: entity.namespace.clone(), + id: entity.id.clone(), + owner: entity.owner.clone(), + owner_ref: entity.owner_ref.clone(), + created_at: entity.created_at.timestamp(), + updated_at: entity.updated_at.timestamp(), + created_by: entity.created_by.clone().unwrap_or_default(), + labels, + }; + + // Construct final V1Container + let container = V1Container { + kind: "Container".to_owned(), // or use default_container_kind() if needed + platform: entity.platform.clone().unwrap_or_default(), + metadata, + image: entity.image.clone(), + env, + command: entity.command.clone(), + args: entity.args.clone(), + volumes, + accelerators: entity.accelerators.clone(), + meters, + restart: entity.restart.clone(), + queue: entity.queue.clone(), + timeout: entity.timeout.clone(), + status, + resources, + health_check, + ssh_keys, + ports: ports.clone(), + proxy_port: entity.proxy_port.clone(), + authz, + }; + + Ok(container) + } +} + + +impl ToEntity for V1Container { + /// Convert a V1Container into a Model. + /// Returns a serde_json Error if any JSON parsing in subfields fails. + fn to_entity(&self) -> Result { + // Convert the metadata + let metadata = self.metadata.clone(); + + // Convert the container + let model = entities::containers::Model { + id: metadata.id, + version: 1, + name: metadata.name.clone(), + full_name: format!("{}:{}", metadata.namespace, metadata.name), // TODO: Check that this is correct + namespace: metadata.namespace, + owner: metadata.owner, + owner_ref: metadata.owner_ref, + image: self.image.clone(), + env: self.env.clone(), + volumes: self.volumes.clone(), + local_volumes: None, + accelerators: self.accelerators.clone(), + cpu_request: self.resources.as_ref().and_then(|r| r.cpu_request.clone()), + memory_request: self.resources.as_ref().and_then(|r| r.memory_request.clone()), + status: self.status.clone(), + platform: Some(self.platform.clone()), + platforms: None, + resource_name: None, + resource_namespace: None, + resource_cost_per_hr: None, + command: self.command.clone(), + args: self.args.clone(), + labels: Some(metadata.labels), + meters: self.meters.clone(), + queue: self.queue.clone(), + ports: self.ports.clone(), + proxy_port: self.proxy_port, + timeout: self.timeout.clone(), + resources: Some(self.resources), + health_check: Some(self.health_check), + restart: self.restart.clone(), + authz: Some(self.authz), + public_addr: None, + tailnet_ip: None, + created_by: Some(metadata.created_by), + desired_status: None, + controller_data: None, + container_user: None, + ssh_keys: Some(self.ssh_keys), + updated_at: chrono::Utc::now().timestamp(), + created_at: chrono::Utc::now().timestamp(), + }; + + Ok(model) + } +} \ No newline at end of file diff --git a/src/resources/v1/containers/ec2.rs b/src/resources/v1/containers/ec2.rs deleted file mode 100644 index 70b786d..0000000 --- a/src/resources/v1/containers/ec2.rs +++ /dev/null @@ -1 +0,0 @@ -// TODO diff --git a/src/resources/v1/containers/gce.rs b/src/resources/v1/containers/gce.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/resources/v1/containers/mod.rs b/src/resources/v1/containers/mod.rs index bafe3ae..3a353c3 100644 --- a/src/resources/v1/containers/mod.rs +++ b/src/resources/v1/containers/mod.rs @@ -4,3 +4,4 @@ pub mod factory; pub mod kube; pub mod models; pub mod runpod; +mod conversion; diff --git a/src/resources/v1/containers/models.rs b/src/resources/v1/containers/models.rs index e1cb772..6dd40b9 100644 --- a/src/resources/v1/containers/models.rs +++ b/src/resources/v1/containers/models.rs @@ -1,6 +1,7 @@ use crate::models::{ V1AuthzConfig, V1Meter, V1ResourceMeta, V1ResourceMetaRequest, V1ResourceReference, }; +use crate::resources::containers::ContainerModelVersion; use crate::resources::v1::volumes::models::V1VolumePath; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -220,3 +221,20 @@ pub struct V1ContainerSearch { pub proxy_port: Option, pub authz: Option, } + +pub struct V1; + +impl ContainerModelVersion for V1 { + type Container = V1Container; + type ContainerMetaRequest = V1ContainerMetaRequest; + type ContainerHealthCheck = V1ContainerHealthCheck; + type ContainerStatus = V1ContainerStatus; + type EnvVar = V1EnvVar; + type ErrorResponse = V1ErrorResponse; + type ContainerResource = V1ContainerResources; + type Port = V1Port; + type PortRequest = V1PortRequest; + type SSHKey = V1SSHKey; + type UpdateContainer = V1UpdateContainer; + type ContainerSearch = V1ContainerSearch; +} diff --git a/src/resources/v1/containers/runpod.rs b/src/resources/v1/containers/runpod.rs index 2f04ade..eede8e1 100644 --- a/src/resources/v1/containers/runpod.rs +++ b/src/resources/v1/containers/runpod.rs @@ -1,6 +1,6 @@ use crate::accelerator::base::AcceleratorProvider; use crate::accelerator::runpod::RunPodProvider; -use crate::config::CONFIG; +use crate::config::SERVER_CONFIG; use crate::entities::containers; use crate::models::{V1Meter, V1UserProfile}; use crate::mutation::{self, Mutation}; diff --git a/src/resources/v1/mod.rs b/src/resources/v1/mod.rs index 87c28bc..3cad2c2 100644 --- a/src/resources/v1/mod.rs +++ b/src/resources/v1/mod.rs @@ -5,3 +5,4 @@ pub mod processors; pub mod secrets; pub mod services; pub mod volumes; +pub(crate) mod platforms; diff --git a/src/resources/v1/platforms/mod.rs b/src/resources/v1/platforms/mod.rs new file mode 100644 index 0000000..640c751 --- /dev/null +++ b/src/resources/v1/platforms/mod.rs @@ -0,0 +1 @@ +pub(crate) mod models; \ No newline at end of file diff --git a/src/resources/v1/platforms/models.rs b/src/resources/v1/platforms/models.rs new file mode 100644 index 0000000..ec70bf1 --- /dev/null +++ b/src/resources/v1/platforms/models.rs @@ -0,0 +1,12 @@ +use crate::resources::platforms::PlatformModelVersion; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq)] +pub struct V1ContainerPlatform {} + +pub struct V1; + +impl PlatformModelVersion for V1 { + type ContainerPlatform = V1ContainerPlatform; +} + diff --git a/src/resources/v1/processors/standard.rs b/src/resources/v1/processors/standard.rs index 1d54258..13533ad 100644 --- a/src/resources/v1/processors/standard.rs +++ b/src/resources/v1/processors/standard.rs @@ -1,5 +1,5 @@ use crate::agent::agent::create_agent_key; -use crate::config::CONFIG; +use crate::config::SERVER_CONFIG; use crate::entities::containers; use crate::entities::processors; use crate::models::V1CreateAgentKeyRequest; @@ -135,13 +135,7 @@ impl StandardProcessor { }); // Redis URL with credentials - prioritize REDIS_URL if set - let redis_url = match CONFIG.redis_url.clone() { - Some(url) => url, - None => format!( - "redis://{}:{}@{}:{}", - username, password, CONFIG.redis_host, CONFIG.redis_port - ), - }; + let redis_url = SERVER_CONFIG.get_redis_url(username, password); // Add all Redis env vars env.push(V1EnvVar { @@ -149,16 +143,6 @@ impl StandardProcessor { value: Some(redis_url), secret_name: None, }); - env.push(V1EnvVar { - key: "REDIS_HOST".to_string(), - value: Some(CONFIG.redis_host.clone()), - secret_name: None, - }); - env.push(V1EnvVar { - key: "REDIS_PORT".to_string(), - value: Some(CONFIG.redis_port.clone()), - secret_name: None, - }); env.push(V1EnvVar { key: "REDIS_CONSUMER_GROUP".to_string(), value: Some(processor.id.clone()), @@ -832,10 +816,10 @@ impl ProcessorPlatform for StandardProcessor { // Assume a function exists to create the key using user profile // We need the auth server URL, user token, desired agent ID, name, and duration. - let config = crate::config::GlobalConfig::read() + let config = crate::config::ClientConfig::read() .map_err(|e| format!("Failed to read global config: {}", e))?; let auth_server = config - .get_current_server_config() + .get_current_server() .and_then(|cfg| cfg.auth_server.clone()) .ok_or_else(|| "Auth server URL not configured".to_string())?; let user_token = user_profile @@ -986,10 +970,10 @@ impl ProcessorPlatform for StandardProcessor { .decrypt_value() .map_err(|e| format!("Failed to decrypt agent key: {}", e))?; - let config = crate::config::GlobalConfig::read() + let config = crate::config::ClientConfig::read() .map_err(|e| format!("Failed to read global config: {}", e))?; let auth_server = config - .get_current_server_config() + .get_current_server() // Use .as_ref() to avoid moving out of shared reference .as_ref() .ok_or_else(|| "Current server config not found".to_string())? diff --git a/src/resources/v1/volumes/base.rs b/src/resources/v1/volumes/base.rs index e69de29..8b13789 100644 --- a/src/resources/v1/volumes/base.rs +++ b/src/resources/v1/volumes/base.rs @@ -0,0 +1 @@ +