diff --git a/src/.vitepress/routes/sidebar/deployment.ts b/src/.vitepress/routes/sidebar/deployment.ts index 7bc0421f..61392039 100644 --- a/src/.vitepress/routes/sidebar/deployment.ts +++ b/src/.vitepress/routes/sidebar/deployment.ts @@ -1,22 +1,35 @@ export const deploymentRoutes = [ - // { - // text: 'System Requirements', - // link: '/hub-introduction', - // }, - { - text: 'Registering in the Hub', - link: '/node-registration' - }, { text: 'Installing k8s', collapsed: true, items: [ - {text: 'microk8s', link: '/microk8s-quickstart'}, - {text: 'minikube', link: '/minikube-quickstart'}, + { text: 'microk8s', link: '/microk8s-quickstart' }, + { text: 'minikube', link: '/minikube-quickstart' }, + ] + }, + { + text: 'Hub Deployment', + items: [ + { text: 'Helm Chart Installation', link: '/hub-installation' }, + { text: 'Storage Setup', link: '/hub-storage' }, + { text: 'Docker Compose (Dev)', link: '/hub-docker-compose' }, ] }, { - text: 'FLAME Node Deployment', - link: '/node-installation' + text: 'Node Deployment', + items: [ + { + text: 'Installation', + link: '/node-installation' + }, + { + text: 'Registering in the Hub', + link: '/node-registration' + }, + { + text: 'Troubleshooting', + link: '/node-troubleshooting' + }, + ] }, ] diff --git a/src/guide/deployment/central-app.md b/src/guide/deployment/central-app.md deleted file mode 100644 index beeb310e..00000000 --- a/src/guide/deployment/central-app.md +++ /dev/null @@ -1,105 +0,0 @@ -# App - -The central app contains the following services: -- user-interface, -- api (authorization- & resource-server) -- realtime-server -- train-manager -- ... - -Those are managed in a mono-repository on GitHUb. -Contributions are very welcome and be performed by following the guideline of the README in the repository. - -::: info Info -Assure yourself that you are not using `dash` as default shell. -Therefore, run the command `dpkg-reconfigure dash` to make the adjustment if necessary. -::: - -## Installation -```shell -git clone https://github.com/PHT-Medic/central-deployment -cd central-deployment -``` - -## Configuration - -```shell -$ ./env.sh -``` - -This command will create environment files in the following directories: -- config/.env -- config/api/.env -- config/realtime/.env -- config/third-party/authup/.env -- config/train-manager/.env -- config/ui/.env - -Change the values to your needs. - -::: warning Info -Don't forget to replace the placeholders with the actual values: -- `[APP_URL]`: Web address (e.g. https://app.example.com/) -- `[APP_API_URL]`: Web address (e.g. https://app.example.com/api/) -- `[AUTHUP_API_URL]`: Web address of the authup service (e.g. https://app.example.com/auth/) -::: - -The following values need to be adjusted before running the setup script! - -**`config/api/.env`** -```dotenv -API_URL=[API_URL] -APP_URL=[APP_URL] -AUTHUP_API_URL=[AUTHUP_API_URL] -``` - -**`config/realtime/.env`** -```dotenv -AUTHUP_API_URL=[AUTHUP_API_URL] -``` - -**`config/third-party/authup/.env`** -```dotenv -AUTHORIZE_REDIRECT_URL=[APP_URL] -``` - -**`config/train-manager/.env`** -```dotenv -API_URL=[API_URL] -AUTHUP_API_URL=[AUTHUP_API_URL] -``` - -**`config/ui/.env`** -```dotenv -API_URL=[API_URL] -APP_URL=[APP_URL] -AUTHUP_API_URL=[AUTHUP_API_URL] -``` - -## Setup -To set up everything just run the following command: -``` -$ ./setup.sh -``` -This will download the docker image and run initial project setup πŸ”₯. - -## Usage -Start πŸ›« -``` -$ ./third-party.sh start -$ ./start.sh -``` -Stop πŸ›¬ -``` -$ ./third-party.sh stop -$ ./stop.sh -``` - -Reset πŸͺ‚ -``` -$ ./reset.sh -``` - - - - diff --git a/src/guide/deployment/central-harbor.md b/src/guide/deployment/central-harbor.md deleted file mode 100644 index 7365edc9..00000000 --- a/src/guide/deployment/central-harbor.md +++ /dev/null @@ -1,275 +0,0 @@ -# Harbor - -The open source software [harbor](https://goharbor.io/) is a registry for docker images. It is used to store and distribute -the docker images of the PHT-meDIC infrastructure, as well as the PHT-meDIC train images. -The harbor instance runs behind by the same reverse proxy used by the central application. - -## Prerequisites - -- [Docker](https://docs.docker.com/engine/install/) -- [Docker Compose](https://docs.docker.com/compose/install/) -- CPU: Minimum 2, Recommended 4 -- RAM: Minimum 4GB, Recommended 8GB -- Disk: Minimum 40GB, Recommended 160GB -- HTTPS: A valid `[HARBOR_SSL_CRT]` + `[HARBOR_SSL_KEY]` for the `[HARBOR_DOMAIN]` name -- DNS: A valid DNS entry for the `[HARBOR_DOMAIN]` name - - -## Installation - -Follow the instructions in the [harbor documentation](https://goharbor.io/docs/2.4.0/install-config/) to install harbor. - - -## Configuration -The following shows an example configuration file for harbor. - -::: warning Info -Don't forget to replace the placeholders with the actual values: - -- `[HARBOR_DOMAIN]`: Domain name (e.g. harbor.example.com) -- `[HARBOR_STORAGE]`: Storage path (e.g. /data/harbor) -- `[HARBOR_SSL_CRT]`: Path to certificate file (.crt) -- `[HARBOR_SSL_KEY]`: Path to certificate key file (.key) -::: - -```yaml -# Configuration file of Harbor - -# The IP address or hostname to access admin UI and registry service. -# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -hostname: [HARBOR_DOMAIN] - -# http related config -http: - # port for http, default is 80. If https enabled, this port will redirect to https port - port: 80 - -# https related config -https: - # https port for harbor, default is 443 - port: 443 - # The path of cert and key files for nginx - certificate: [HARBOR_SSL_CRT] - private_key: [HARBOR_SSL_KEY] - -# # Uncomment following will enable tls communication between all harbor components -# internal_tls: -# # set enabled to true means internal tls is enabled -# enabled: true -# # put your cert and key files on dir -# dir: /etc/harbor/tls/internal - -# Uncomment external_url if you want to enable external proxy -# And when it enabled the hostname will no longer used -external_url: https://[HARBOR_DOMAIN] - -# The initial password of Harbor admin -# It only works in first time to install harbor -# Remember Change the admin password from UI after launching Harbor. -harbor_admin_password: Harbor12345 - -# Harbor DB configuration -database: - # The password for the root user of Harbor DB. Change this before any production use. - password: root123 - # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. - max_idle_conns: 100 - # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. - # Note: the default number of connections is 1024 for postgres of harbor. - max_open_conns: 900 - -# The default data volume -data_volume: [HARBOR_STORAGE] - -# Harbor Storage settings by default is using /data dir on local filesystem -# Uncomment storage_service setting If you want to using external storage -# storage_service: -# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore -# # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. -# ca_bundle: - -# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss -# # for more info about this configuration please refer https://docs.docker.com/registry/configuration/ -# filesystem: -# maxthreads: 100 -# # set disable to true when you want to disable registry redirect -# redirect: -# disabled: false - -# Trivy configuration -# -# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. -# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached -# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it -# should download a newer version from the Internet or use the cached one. Currently, the database is updated every -# 12 hours and published as a new release to GitHub. -trivy: - # ignoreUnfixed The flag to display only fixed vulnerabilities - ignore_unfixed: false - # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub - # - # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues. - # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and - # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path. - skip_update: false - # - # The offline_scan option prevents Trivy from sending API requests to identify dependencies. - # Scanning JAR files and pom.xml may require Internet access for better detection, but this option tries to avoid it. - # For example, the offline mode will not try to resolve transitive dependencies in pom.xml when the dependency doesn't - # exist in the local repositories. It means a number of detected vulnerabilities might be fewer in offline mode. - # It would work if all the dependencies are in local. - # This option doesn’t affect DB download. You need to specify "skip-update" as well as "offline-scan" in an air-gapped environment. - offline_scan: false - # - # insecure The flag to skip verifying registry certificate - insecure: false - # github_token The GitHub access token to download Trivy DB - # - # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough - # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000 - # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult - # https://developer.github.com/v3/#rate-limiting - # - # You can create a GitHub token by following the instructions in - # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line - # - # github_token: xxx - -jobservice: - # Maximum number of job workers in job service - max_job_workers: 10 - -notification: - # Maximum retry count for webhook job - webhook_job_max_retry: 10 - -chart: - # Change the value of absolute_url to enabled can enable absolute url in chart - absolute_url: disabled - -# Log configurations -log: - # options are debug, info, warning, error, fatal - level: info - # configs for logs in local storage - local: - # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. - rotate_count: 50 - # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. - # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G - # are all valid. - rotate_size: 200M - # The directory on your host that store log - location: /var/log/harbor - - # Uncomment following lines to enable external syslog endpoint. - # external_endpoint: - # # protocol used to transmit log to external endpoint, options is tcp or udp - # protocol: tcp - # # The host of external endpoint - # host: localhost - # # Port of external endpoint - # port: 5140 - -#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 2.4.0 - -# Uncomment external_database if using external database. -# external_database: -# harbor: -# host: harbor_db_host -# port: harbor_db_port -# db_name: harbor_db_name -# username: harbor_db_username -# password: harbor_db_password -# ssl_mode: disable -# max_idle_conns: 2 -# max_open_conns: 0 -# notary_signer: -# host: notary_signer_db_host -# port: notary_signer_db_port -# db_name: notary_signer_db_name -# username: notary_signer_db_username -# password: notary_signer_db_password -# ssl_mode: disable -# notary_server: -# host: notary_server_db_host -# port: notary_server_db_port -# db_name: notary_server_db_name -# username: notary_server_db_username -# password: notary_server_db_password -# ssl_mode: disable - -# Uncomment external_redis if using external Redis server -# external_redis: -# # support redis, redis+sentinel -# # host for redis: : -# # host for redis+sentinel: -# # :,:,: -# host: redis:6379 -# password: -# # sentinel_master_set must be set to support redis+sentinel -# #sentinel_master_set: -# # db_index 0 is for core, it's unchangeable -# registry_db_index: 1 -# jobservice_db_index: 2 -# chartmuseum_db_index: 3 -# trivy_db_index: 5 -# idle_timeout_seconds: 30 - -# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. -# uaa: -# ca_file: /path/to/ca - -# Global proxy -# Config http proxy for components, e.g. http://my.proxy.com:3128 -# Components doesn't need to connect to each others via http proxy. -# Remove component from `components` array if want disable proxy -# for it. If you want use proxy for replication, MUST enable proxy -# for core and jobservice, and set `http_proxy` and `https_proxy`. -# Add domain to the `no_proxy` field, when you want disable proxy -# for some special registry. -proxy: - http_proxy: - https_proxy: - no_proxy: - components: - - core - - jobservice - - trivy - -# metric: -# enabled: false -# port: 9090 -# path: /metrics - -# Trace related config -# only can enable one trace provider(jaeger or otel) at the same time, -# and when using jaeger as provider, can only enable it with agent mode or collector mode. -# if using jaeger collector mode, uncomment endpoint and uncomment username, password if needed -# if using jaeger agetn mode uncomment agent_host and agent_port -# trace: -# enabled: true -# # set sample_rate to 1 if you wanna sampling 100% of trace data; set 0.5 if you wanna sampling 50% of trace data, and so forth -# sample_rate: 1 -# # # namespace used to differenciate different harbor services -# # namespace: -# # # attributes is a key value dict contains user defined attributes used to initialize trace provider -# # attributes: -# # application: harbor -# # # jaeger should be 1.26 or newer. -# # jaeger: -# # endpoint: http://hostname:14268/api/traces -# # username: -# # password: -# # agent_host: hostname -# # # export trace data by jaeger.thrift in compact mode -# # agent_port: 6831 -# # otel: -# # endpoint: hostname:4318 -# # url_path: /v1/traces -# # compression: false -# # insecure: true -# # timeout: 10s - -``` diff --git a/src/guide/deployment/central-nginx.md b/src/guide/deployment/central-nginx.md deleted file mode 100644 index 6eb92652..00000000 --- a/src/guide/deployment/central-nginx.md +++ /dev/null @@ -1,159 +0,0 @@ -# Nginx -To forward requests to the docker-containers, it is required to configure nginx as a reverse proxy. -Therefore, two separate configurations for [harbor](#harbor) and the [app](#app) must be created. - -## Harbor -For harbor, we create a new file (e.g. `harbor`) in the directory `/etc/nginx/sites-enabled` with the following content: - -::: warning Info -Don't forget to replace the placeholders with the actual values: - -- `[HARBOR_DOMAIN]`: Domain name (e.g. harbor.example.com) -- `[HARBOR_ADDRESS]`: Ip Address:Port of the harbor instance rev. proxy (e.g. 192.168.1.1:443) -- `[HARBOR_SSL_CRT]`: Path to certificate file (.crt) -- `[HARBOR_SSL_KEY]`: Path to certificate key file (.key) -::: - -```txt -server { - server_name [HARBOR_DOMAIN]; - - gzip on; - gzip_types text/plain application/xml text/css application/javascript; - gzip_min_length 1000; - - client_max_body_size 0; - chunked_transfer_encoding on; - - listen 443 ssl; - ssl_certificate [HARBOR_SSL_CRT]; - ssl_certificate_key [HARBOR_SSL_KEY]; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - location / { - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass [HARBOR_ADDRESS]; - - proxy_buffering off; - proxy_request_buffering off; - } - -} -server { - if ($host = [HARBOR_DOMAIN]) { - return 301 https://$host$request_uri; - } - - - server_name [HARBOR_DOMAIN]; - listen 80; - return 404; - -} - -``` - -## App -For the app we need to create a new file (e.g. `app`) in the directory `/etc/nginx/sites-enabled` with the following content: - -::: warning Info -Don't forget to replace the placeholders with the actual values: -- `[APP_DOMAIN]` Domain name (e.g. app.example.com) -- `[APP_SSL_CRT]`: Certificate file (.crt) -- `[APP_SSL_KEY]`: Certificate key file (.key) -::: - -```txt -map $sent_http_content_type $expires { - "text/html" epoch; - "text/html; charset=utf-8" epoch; - default off; -} - -server { - server_name [APP_DOMAIN]; - listen 443 ssl; - - ssl_certificate [APP_SSL_CRT]; - ssl_certificate_key [APP_SSL_KEY]; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - gzip on; - gzip_types text/plain application/xml text/css application/javascript; - gzip_min_length 1000; - - client_max_body_size 0; - chunked_transfer_encoding on; - - location / { - expires $expires; - - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 1m; - proxy_connect_timeout 1m; - proxy_pass http://127.0.0.1:3000; - } - - location /api/ { - rewrite ^/api(/.*)$ $1 break; - - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 2m; - proxy_connect_timeout 2m; - proxy_pass http://127.0.0.1:3002; - } - - location /auth/ { - rewrite ^/auth(/.*)$ $1 break; - - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 2m; - proxy_connect_timeout 2m; - proxy_pass http://127.0.0.1:3010; - } - - location /socket.io/ { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_pass http://127.0.0.1:3001; - - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } -} - -server { - if ($host = [APP_DOMAIN]) { - return 301 https://$host$request_uri; - } - - - listen 80; - server_name [APP_DOMAIN]; - return 404; -} -``` - -## Installer diff --git a/src/guide/deployment/hub-docker-compose.md b/src/guide/deployment/hub-docker-compose.md new file mode 100644 index 00000000..72ca778a --- /dev/null +++ b/src/guide/deployment/hub-docker-compose.md @@ -0,0 +1,33 @@ +# FLAME Hub Docker Compose + +Clone the [Hub Deployment repository](https://github.com/privateaim/hub-deployment.git): +```bash +git clone https://github.com/privateaim/hub-deployment.git +``` + +## Note on Harbor +This setup does not include the Harbor service, you must either provide the credentials to an existing Harbor or [install Harbor separately](https://goharbor.io/docs/latest/install-config/). + +## Configuration +Basic configuration occurs via environment variables in an `.env` file. +An example (`.env.example`) is located in the root of the repository. + +| Variable | Mandatory | Default | Use/Meaning | +|-----------------|:---------:|---------|-------------------------------------------------------------------| +| `HUB_IMAGE` | ❌ | `ghcr.io/privateaim/hub` | Used to override the default image for the `HUB` docker image | +| `HUB_IMAGE_TAG` | ❌ | `latest` | Used to override the default image tag for the `HUB` docker image | +| `SUBNET` | ❌ | `172.40.1.0/24` | Used to change the default docker subnet. | + +To provide credentials to Harbor (either local or external), use the following Variable: +> ⚠️ Important: Harbor must be accessible via https (plain http will fail). This is a limitation of Harbor. + +| Variable | Mandatory | Use/Meaning | +|-----------------|:---------:|-------------------------------------------------------------------| +| `HARBOR_URL` | βœ… | Construct using `:@` (no `https://` ) | + +## Run + +Run using: +```bash +docker compose up -d +``` diff --git a/src/guide/deployment/hub-installation.md b/src/guide/deployment/hub-installation.md new file mode 100644 index 00000000..5f20de63 --- /dev/null +++ b/src/guide/deployment/hub-installation.md @@ -0,0 +1,71 @@ +# FLAME Hub Helm Chart + +The FLAME Hub Chart provides a default configuration in `values.yaml` but is meant to be installed with your own adjusted override values file, e.g. `values_local.yaml`. +To get an overview of all configuration options, see the [`values.yaml` in the Helm Repository](https://github.com/PrivateAIM/helm/blob/master/charts/flame-hub/values.yaml) +You can choose if you want to use the chart's harbor or a separate external harbor instance. + +## Prerequisites: +* Existing k8s distribution (see [microk8s-quickstart](./microk8s-quickstart) or [minikube-quickstart](./minikube-quickstart) for a guide) + * Ingress addon + * Default StorageClass (e.g. hostpath-storage addon in microk8s) +* Two domain names (one for the Hub, one for Harbor). If you run the Hub on your local machine, you can choose them freely and set them in your hosts file (`/etc/hosts`) + +## Storage +The chart will use whichever storage class is the default in your cluster, unless you specify otherwise in the values. See [Storage Setup](./hub-storage) for instructions how to setup Mayastor Storage Replication. This requires 3+ nodes in your cluster and will replicate persistent volumes across them. + +## Ingress +See the `values.yaml` for ingress options. The default ingress configuration will use path-based routing for all services except Harbor (which requires its own hostname). You will have to provide an extra (sub)domain if you want to use the harbor component of this chart. The current setup does not automatically acquire TLS certificates. + +## Installing the FLAME Hub Chart + +### 1. Option: Official Chart Repo +```bash +helm repo add flame https://PrivateAIM.github.io/helm +helm repo update +``` +Create your custom values file (instructions below). +```bash +helm install -f flame/hub +``` + +### 2. Option: Chart Source Code +> Choose this option if you want to +> +> a) modify not only the chart values, but also the chart files. +> +> b) run the newest, not yet released version of the chart. + +```bash +git clone https://github.com/PrivateAIM/helm.git +cd helm +cd charts/flame-hub +``` +Create your custom values file (instructions below). + +```bash +helm install -f . +``` + +## Minimal configuration using custom values file + +This is the values-file mentioned in the sections above. Call it `override_values.yaml` or whatever you want. + +```yaml +global: + flameHub: + ingress: + enabled: true + ssl: true + hostname: "hub.local" + +grafana: + # disable plugins until plugin issues are fixed + plugins: + +harbor: + enabled: true + externalURL: "https://harbor.hub.local/" # don't forget https:// +``` + +## Accessing the hub +For accessing the hub in a **minikube** installation, refer to the [Minikube Setup: Accessing Deployments](./minikube-quickstart) diff --git a/src/guide/deployment/hub-storage.md b/src/guide/deployment/hub-storage.md new file mode 100644 index 00000000..7f23df0b --- /dev/null +++ b/src/guide/deployment/hub-storage.md @@ -0,0 +1,103 @@ +# OpenEBS/Mayastor for FLAME (optional) + +OpenEBS is a chart that provides extra storage options. We will be using Mayastor Replicated Storage to synchronize Persistent Volumes across multiple nodes in the cluster. This aims to ensure no data is lost when a node fails. With extra configuration, it enables automatic failover. + +If you want to enable storage replication, you need to: +1. Prepare your k8s nodes +2. Install OpenEBS into your k8s cluster +3. Configure your Application (FLAME Hub) to use the new storage class + +## Node Setup +Before installing OpenEBS, you must prepare the k8s nodes you want to use for mayastor. + +### Automatic script πŸ€– +Use the script `prepare_for_mayastor.sh`. +> The script has been tested in Debian. +> +> The script can safely be run multiple times. + +Download the script: [prepare_for_mayastor.sh](https://github.com/PrivateAIM/hub-deployment/tree/master/scripts) + +```bash +chmod +x prepare_for_mayastor.sh +sudo ./prepare_for_mayastor.sh +``` +Alternatively, you can manually do the following: + +### 1. Configure hugepages +Enable hugepages: +```bash +echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +``` +Persist hugepages and apply: +```bash +echo "vm.nr_hugepages = 1024" > /etc/sysctl.d/20-microk8s-hugepages.conf +sysctl --system +``` + +### 2. Load NVMe-oF module +Load the module immediately: +```bash +modprobe nvme-tcp +``` +Persist the module across reboots: + * If `/etc/modules-load.d/nvme-tcp.conf` does not exist or does not + contain `nvme-tcp`, create it with: +```bash +echo "nvme-tcp" > /etc/modules-load.d/nvme-tcp.conf +``` + +### 3. Verify +Check hugepages: +```bash +grep HugePages_Total /proc/meminfo +``` +Check the module is active: +```bash +lsmod | grep nvme_tcp +``` + +### 4. Label the node for Mayastor +```bash +kubectl label node openebs.io/engine=mayastor +``` + +## Installing OpenEBS + +The FLAME Helm repository provides a [wrapper chart for OpenEBS](https://github.com/PrivateAIM/helm/blob/master/charts/third-party/openebs) with some suggested default values. +Installing this chart will add a new StorageClass to your cluster. You can tell workloads of the Flame Hub to use this StorageClass by specifying it in `flame-hub/values.yaml`. Usually, only stateful workloads (StatefulSets) need replicated storage. + + +1. **Clone the [Flame Helm Repository](https://github.com/PrivateAIM/helm/) and navigate to `charts/third-party/openebs`** +2. Make sure you have 3 nodes in your cluster. +3. Make sure you have read the previous section on node preparation for mayastor. + * If not already labeled, label your nodes: +`kubectl label node openebs.io/engine=mayastor` +4. Clone `values.yaml` to `values_local.yaml`. +5. Fill in your kubelet path and populate the disk pools section with your unmounted drives. +6. Install the chart: + +```bash +helm dependency update . +``` +```bash +helm install openebs . --namespace openebs --create-namespace -f values_local.yaml +``` + +7. Verify disk pools and `mayastor` storage classes: +```bash +kubectl get diskpools -n openebs +kubectl get storageclasses +``` +8. Edit your FLAME-Hub values-file to use the new StorageClass. A reinstall of the Hub will probably be required. + +## Use the new storage class + +In your `values.yaml` (or better `values_override.yaml`) you can specify the storage class for the different components of the FLAME Hub. + +Example: +```yaml +minio: + persistence: + storageClass: "mayastor-replicated" +``` \ No newline at end of file diff --git a/src/guide/deployment/index.md b/src/guide/deployment/index.md index d54c9cf6..8166938b 100644 --- a/src/guide/deployment/index.md +++ b/src/guide/deployment/index.md @@ -2,17 +2,32 @@ ## Introduction -This deployment guide contains instructions on how to deploy the FLAME Node software on your server, thereby creating -a node that analysts can use for their federated learning models. +The FLAME project consists of two major components: +- **FLAME Hub**: The central component coordinating the nodes. +- **FLAME Node**: The component installed at data sites (hospitals, research centers) to execute federated learning tasks. + +This guide contains instructions for deploying both components. ## Overview -Below are links to the tasks that need to be completed to successfully create a FLAME Node: +### Hub Deployment + +Instructions for deploying the FLAME Hub. + +- **Production (Kubernetes)** + - [Helm Chart Installation](/guide/deployment/hub-installation) + - [Storage Setup (Mayastor) (optional)](/guide/deployment/hub-storage) +- **Development / Test** + - [Hub Docker Compose](/guide/deployment/hub-docker-compose) + +### Node Deployment + +Instructions for deploying a FLAME Node. -**`Deployment`** +**Tasks:** - [Hub Registration](/guide/deployment/node-registration) -- Installing k8s (choose one) +- Cluster Setup (choose one) - [microk8s](/guide/deployment/microk8s-quickstart) - [minikube](/guide/deployment/minikube-quickstart) -- [FLAME Node Deployment](/guide/deployment/node-installation) +- [FLAME Node Installation](/guide/deployment/node-installation) diff --git a/src/guide/deployment/microk8s-quickstart.md b/src/guide/deployment/microk8s-quickstart.md index a5a62fb1..1c82ef68 100644 --- a/src/guide/deployment/microk8s-quickstart.md +++ b/src/guide/deployment/microk8s-quickstart.md @@ -2,6 +2,28 @@ This guide demonstrates how to get microk8s and Helm running on a debian-based server (e.g. ubuntu). ## Install microk8s and Helm + +### Automatic Installation script πŸ€– +Instead of following the below installation guide, you can also run the bash script: +> The script has been tested in Debian. + +Download: [microk8s_setup.sh](https://github.com/PrivateAIM/hub-deployment/tree/master/scripts) + +```bash +chmod +x microk8s_setup.sh +./microk8s_setup.sh +``` + +This script will optionally: +* Setup non default MicroK8s storage path (see section Storage) +* Install Snap and MicroK8s +* Configure roles and permissions +* Export Kubeconfig +* Add /snap/bin to PATH + +### Manual Installation +If you prefer to install manually, follow the steps below. + ### Update System Packages Assuming you are working with a freshly installed OS, be sure to first update the system packages before proceeding: ```bash @@ -26,8 +48,45 @@ following command: echo 'export PATH=$PATH:/snap/bin' >> ~/.bashrc && source ~/.bashrc ``` -### Use snap to install microk8s -Next, install the `core` dependency along with microk8s using snap: +### Prepare Storage before Install (Optional) +Because we need to be able to download and store several container images, we need to ensure that there is enough disk +space available. By default, microk8s will store everything on your root file system located at `/`. +Depending on how your server was setup, bulk storage may be located on a different logical volume (LV) than your root +file system. You can check this by running: +```bash +df -h +``` +
+LV sizes +
Example output. Here the root file system, installed at / +(yellow arrow) only has 13GB available, +but a different LV mounted at /mnt/vdb1 (green arrow) has 435GB.
+ +If you need to use a different LV for storage, you can either: + +**Option A (Recommended for simplicity):** Mount (or symlink) the SNAP_COMMON directory to your preferred destination. +::: tip +This storage configuration step should be performed **before** installing microk8s to avoid needing to migrate data later. +::: + +Ensure the target directories exist and then mount the volume: +```bash +# Create the target directory and the mount point +mkdir -p /mnt/vdb1/microk8s/common +sudo mkdir -p /var/snap/microk8s/common + +# Bind mount +sudo mount --bind /mnt/vdb1/microk8s/common /var/snap/microk8s/common + +# for persistence +echo "/mnt/vdb1/microk8s/common /var/snap/microk8s/common none bind 0 0" | sudo tee -a /etc/fstab +``` + +**Option B (after installation):** Configure containerd to use a different root. +This option is performed **after** installation and shown in the section "Setup microk8s". + +### Install microK8s +Install the `core` dependency along with microk8s using snap: ```bash sudo snap install core && sudo snap install microk8s --classic --channel=1.32 ``` @@ -54,22 +113,10 @@ chmod 700 get_helm.sh ``` ## Setup microk8s -### Update Storage Location -Because we need to be able to download and store several container images, we need to ensure that there is enough disk -space available. By default, microk8s will store everything on your root file system located at `/`. -Depending on how your server was setup, bulk storage may be located on a different logical volume (LV) than your root -file system. You can check this by running: -```bash -df -h -``` -
-LV sizes -
Example output. Here the root file system, installed at / -(yellow arrow) only has 13GB available, -but a different LV mounted at /mnt/vdb1 (green arrow) has 435GB.
+### Storage Setup after Installation (optional) +As discussed before, we may need to change the micro8s filesystem path to another drive with more available storage. If you have not mounted the SNAP_COMMON path before the microk8s installation, you can edit the config after installation as shown here. -If you need to use a different LV for storage, replace `` in the following command with the mount path -of the LV you want to use for storage and then execute the command: +Replace `` in the following command with the mount path of the LV you want to use for storage and then execute the command: ```bash cat < /var/snap/microk8s/current/args/containerd --config \${SNAP_DATA}/args/containerd.toml