From 84912ca2e08ee8b4f273447ec6ea9cd70d0896f0 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Thu, 5 Feb 2026 15:34:03 +0200 Subject: [PATCH] MDBF-1163 - Replace minio with Garage Concurrent S3 tests can operate within a single bucket given that MDEV-38754 is accepted and merged up to main, thus, no need for create/delete bucket tasks, nor using master's `mc` client. --- common_factories.py | 44 +++---------- configuration/builders/sequences/helpers.py | 27 ++------ configuration/steps/commands/util.py | 44 ------------- docker-compose/docker-compose.yaml | 13 ---- docker-compose/generate-config.py | 5 +- master-private.cfg-sample | 1 - minio/README.md | 52 --------------- minio/docker-compose.yaml | 41 ------------ minio/nginx/conf.d/minio.conf | 73 --------------------- minio/nginx/nginx.conf | 73 --------------------- rsync.exclude | 1 - 11 files changed, 15 insertions(+), 359 deletions(-) delete mode 100644 minio/README.md delete mode 100644 minio/docker-compose.yaml delete mode 100644 minio/nginx/conf.d/minio.conf delete mode 100644 minio/nginx/nginx.conf diff --git a/common_factories.py b/common_factories.py index a95f6acfd..1cc2eec5f 100644 --- a/common_factories.py +++ b/common_factories.py @@ -475,24 +475,12 @@ def addGaleraTests(factory, mtrDbPool): def addS3Tests(factory, mtrDbPool): runS3 = lambda props: hasS3(props) and props.hasProperty("compile_step_completed") - factory.addStep( - steps.MasterShellCommand( - name="Create minio S3 bucket", - alwaysRun=True, - command=[ - "mc", - "mb", - util.Interpolate("minio/%(prop:buildername)s-%(prop:buildnumber)s"), - ], - doStepIf=runS3, - ) - ) factory.addStep( steps.MTR( - name="S3 minio tests", + name="MTR - S3", alwaysRun=True, - description="testing S3 minio", - descriptionDone="test s3 minio", + description="testing S3", + descriptionDone="test s3", logfiles={"mysqld*": "./buildbot/mysql_logs.html"}, test_type="s3", command=[ @@ -512,34 +500,18 @@ def addS3Tests(factory, mtrDbPool): dbpool=mtrDbPool, autoCreateTables=True, env={ - "S3_HOST_NAME": "minio.mariadb.org", + "S3_HOST_NAME": "s3bb.mariadb.org", "S3_PORT": "443", - "S3_ACCESS_KEY": util.Interpolate("%(secret:minio_access_key)s"), - "S3_SECRET_KEY": util.Interpolate("%(secret:minio_secret_key)s"), - "S3_BUCKET": util.Interpolate( - "%(prop:buildername)s-%(prop:buildnumber)s" - ), + "S3_ACCESS_KEY": util.Interpolate("%(secret:s3_access_key)s"), + "S3_SECRET_KEY": util.Interpolate("%(secret:s3_secret_key)s"), + "S3_BUCKET": "mariadb-buildbot", "S3_USE_HTTP": "OFF", "S3_PROTOCOL_VERSION": "Path", + "S3_REGION": "garage", }, doStepIf=runS3, ) ) - - factory.addStep( - steps.MasterShellCommand( - name="Delete minio S3 bucket", - alwaysRun=True, - command=[ - "mc", - "rb", - "--force", - util.Interpolate("minio/%(prop:buildername)s-%(prop:buildnumber)s"), - ], - doStepIf=runS3, - ) - ) - factory.addStep( steps.ShellCommand( name="move mariadb S3 log files", diff --git a/configuration/builders/sequences/helpers.py b/configuration/builders/sequences/helpers.py index d907886b9..814beb6a0 100644 --- a/configuration/builders/sequences/helpers.py +++ b/configuration/builders/sequences/helpers.py @@ -5,11 +5,7 @@ from configuration.steps.base import StepOptions from configuration.steps.commands.base import URL from configuration.steps.commands.mtr import MTRReporter, MTRTest -from configuration.steps.commands.util import ( - CreateS3Bucket, - DeleteS3Bucket, - SaveCompressedTar, -) +from configuration.steps.commands.util import SaveCompressedTar from configuration.steps.generators.mtr.generator import MTRGenerator from configuration.steps.generators.mtr.options import ( MTR, @@ -224,12 +220,6 @@ def get_mtr_s3_steps( env_vars: list[tuple] = [], ): steps = [] - steps.append( - MasterShellStep( - command=CreateS3Bucket(bucket=f"%(prop:buildername)s-%(prop:buildnumber)s") - ) - ) - steps.append( step_wrapping_fn( ShellStep( @@ -254,13 +244,14 @@ def get_mtr_s3_steps( ), ), env_vars=[ - ("S3_HOST_NAME", "minio.mariadb.org"), + ("S3_HOST_NAME", "s3bb.mariadb.org"), ("S3_PORT", "443"), - ("S3_ACCESS_KEY", "%(secret:minio_access_key)s"), - ("S3_SECRET_KEY", "%(secret:minio_secret_key)s"), - ("S3_BUCKET", "%(prop:buildername)s-%(prop:buildnumber)s"), + ("S3_ACCESS_KEY", "%(secret:s3_access_key)s"), + ("S3_SECRET_KEY", "%(secret:s3_secret_key)s"), + ("S3_BUCKET", "mariadb-buildbot"), ("S3_USE_HTTPS", "OFF"), ("S3_PROTOCOL_VERSION", "Path"), + ("S3_REGION", "garage"), ] + env_vars, options=StepOptions( @@ -269,12 +260,6 @@ def get_mtr_s3_steps( ) ) ) - steps.append( - MasterShellStep( - command=DeleteS3Bucket(bucket=f"%(prop:buildername)s-%(prop:buildnumber)s"), - options=StepOptions(alwaysRun=True), - ), - ) return steps diff --git a/configuration/steps/commands/util.py b/configuration/steps/commands/util.py index 2420ba57d..ba86cc074 100644 --- a/configuration/steps/commands/util.py +++ b/configuration/steps/commands/util.py @@ -4,50 +4,6 @@ from configuration.steps.commands.base import BashScriptCommand, Command -class CreateS3Bucket(Command): - """ - A command to create an S3 bucket using MinIO client (mc). - This command initializes a new S3 bucket in the MinIO server. - Attributes: - bucket (str): The name of the S3 bucket to create. - workdir (PurePath): The working directory for the command. - """ - - def __init__(self, bucket: str, workdir: PurePath = PurePath(".")): - name = "Create S3 bucket" - self.bucket = bucket - super().__init__(name=name, workdir=workdir) - - def as_cmd_arg(self) -> list[str]: - return [ - "bash", - "-exc", - util.Interpolate(f"mc mb minio/{self.bucket}"), - ] - - -class DeleteS3Bucket(Command): - """ - A command to delete an S3 bucket using MinIO client (mc). - This command removes an existing S3 bucket from the MinIO server. - Attributes: - bucket (str): The name of the S3 bucket to delete. - workdir (PurePath): The working directory for the command. - """ - - def __init__(self, bucket: str, workdir: PurePath = PurePath(".")): - name = "Delete S3 bucket" - self.bucket = bucket - super().__init__(name=name, workdir=workdir) - - def as_cmd_arg(self) -> list[str]: - return [ - "bash", - "-exc", - util.Interpolate(f"mc rb --force minio/{self.bucket}"), - ] - - class SaveCompressedTar(Command): """ A command to create a compressed tar archive of the current working directory. diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 7f61867b8..509f52fd1 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -111,7 +111,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=9995 - TITLE @@ -148,7 +147,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=9996 - TITLE @@ -184,7 +182,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=9997 - TITLE @@ -220,7 +217,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=9998 - TITLE @@ -256,7 +252,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=9999 - TITLE @@ -292,7 +287,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10000 - TITLE @@ -328,7 +322,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10001 - TITLE @@ -364,7 +357,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10002 - TITLE @@ -400,7 +392,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10003 - TITLE @@ -437,7 +428,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10004 - TITLE @@ -473,7 +463,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10005 - TITLE @@ -509,7 +498,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10006 - TITLE @@ -546,7 +534,6 @@ services: - MASTER_CREDENTIALS_DIR - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - - MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org - MQ_ROUTER_URL - PORT=10007 - TITLE diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py index 11d73f201..f46fade21 100755 --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -143,7 +143,7 @@ def construct_env_section(env_vars): for key, value in sorted(env_vars.items()): if key.startswith("NGINX_"): continue - elif key not in ["PORT", "MC_HOST_minio"]: + elif key not in ["PORT"]: env_section += f" - {key}\n" else: env_section += f" - {key}={value}\n" @@ -186,14 +186,12 @@ def main(args): f"container_name: master-web\n{construct_env_section(env_vars)}", ) - env_vars["MC_HOST_minio"] = "{mc_host}" # Modify the docker_compose_template to include the environment variables docker_compose_template = DOCKER_COMPOSE_TEMPLATE.replace( "container_name: {master_name}", f"container_name: {{master_name}}\n{construct_env_section(env_vars)}", ) - mc_host = config["private"]["minio_url"] starting_port = config["private"]["master-variables"]["starting_port"] master_web_port = 8010 # Generate startup scripts and Docker Compose pieces for each master directory @@ -223,7 +221,6 @@ def main(args): master_hostname=master_hostname, master_directory=master_directory, port=port, - mc_host=mc_host, volumes=generate_volumes(master_volumes[master_name]), environment="" if args.env == "prod" else "dev_", buildbot_stop_grace_period=BUILDBOT_STOP_GRACE_PERIOD, diff --git a/master-private.cfg-sample b/master-private.cfg-sample index bc39c416c..578853630 100644 --- a/master-private.cfg-sample +++ b/master-private.cfg-sample @@ -4,7 +4,6 @@ private["db_user"] = "buildmaster" private["db_password"] = "password" private["db_mtr_db"] = "buildbot" private["gh_secret"] = "gh_secret" -private["minio_url"] = "https://access_key:secret_key@minio.mariadb.org" private["zabbix_server"] = "https://zabbix.server" private["zabbix_token"] = "zabbix_token" private["user_pass"]= { diff --git a/minio/README.md b/minio/README.md deleted file mode 100644 index ccae8213e..000000000 --- a/minio/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# About - - MariaDB Foundation configuration of a Single-Node, Single-Drive MinIO deployment for running server S3 integration tests in BuildBot. - -Available at: [minio.mariadb.org](https://minio.mariadb.org) - -## Usage - -**Docker-compose.yaml** consists of two services: - -* **NGINX**, responsible for the SSL termination and the routing to the appropriate locations for the console and the API, see **minio.conf** - -* A single node/drive **MinIO** instance configured to expose the **console** and **API** on HTTP ports 8080/8081. The container is bind mounted to a data directory for persistent state (buckets, user configuration, and so on). - -**To run it locally**, one can only spin-up the MinIO service and access it on localhost, without any other prerequisites. Without providing the ENV variables ```MINIO_ROOT_{USER,PASSWORD}```, default credentials are used. Please consult the official documentation for more details. - -### Service stop/start - -``` -docker-compose down -docker-compose --env-file .env -d -``` - -### TLS configuration - -LE certificates are provided by **Certbot** with the **Webroot** plugin. This assumes NGINX is configured to handle ACME challenges on port 80. - -See [certbot](https://eff-certbot.readthedocs.io/en/stable/using.html) documentation on how to generate/renew certificates. - -We need to generate a certificate with 2 SAN's: - -1. minio.mariadb.org -1. minio.dev.mariadb.org (for future use) - -For certbot, create a **post-hook** that is able to: - -1. transfer the renewed certificates to the nginx SSL path (see volume mounts) -1. restart nginx container (via docker-compose) - - Make sure **dhparam** is available for NGINX. - - ``` - curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam - ``` - -### Certificate renewal - -On the host, under ```/etc/cron.d``` , replace the contents of the existing certbot file with: - -``` -0 */12 * * * root perl -e 'sleep int(rand(43200))' && certbot -q renew --post-hook "bash /etc/letsencrypt/post_hook.sh" -``` diff --git a/minio/docker-compose.yaml b/minio/docker-compose.yaml deleted file mode 100644 index ff5589b60..000000000 --- a/minio/docker-compose.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -services: - minio: - image: quay.io/minio/minio - restart: unless-stopped - container_name: minio - hostname: minio - environment: - - MINIO_ROOT_USER - - MINIO_ROOT_PASSWORD - - MINIO_BROWSER_REDIRECT_URL=https://minio.mariadb.org/minio/ui/ - - MC_HOST_local=http://localhost:8080 - network_mode: host - command: server /data --console-address ":8081" --address :8080 - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 5s - timeout: 5s - retries: 5 - volumes: - - /data/minio/:/data - logging: - driver: journald - options: - tag: "minio" - - nginx: - image: nginx:latest - restart: unless-stopped - container_name: nginx - hostname: nginx - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./nginx/conf.d/:/etc/nginx/conf.d/ - - ./certbot/www/:/var/www/certbot/:ro - - ./certbot/ssl/:/etc/nginx/ssl/:ro - network_mode: host - logging: - driver: journald - options: - tag: "minio-nginx" diff --git a/minio/nginx/conf.d/minio.conf b/minio/nginx/conf.d/minio.conf deleted file mode 100644 index 492b13e4f..000000000 --- a/minio/nginx/conf.d/minio.conf +++ /dev/null @@ -1,73 +0,0 @@ -server { - listen 80; - listen [::]:80; - - server_name minio.mariadb.org www.minio.mariadb.org; - server_tokens off; - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - - location / { - return 301 https://$server_name$request_uri; - } -} - -server { - listen 443 ssl; - listen [::]:443 ssl; - http2 on; - - server_name minio.mariadb.org; - - # Allow special characters in headers - ignore_invalid_headers off; - # Allow any size file to be uploaded. - # Set to a value such as 1000m; to restrict file size to a specific value - client_max_body_size 0; - # Disable buffering - proxy_buffering off; - proxy_request_buffering off; - - location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_connect_timeout 300; - # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 - proxy_http_version 1.1; - proxy_set_header Connection ""; - chunked_transfer_encoding off; - - proxy_pass http://127.0.0.1:8080; # This uses the upstream directive definition to load balance - } - - location /minio/ui/ { - rewrite ^/minio/ui/(.*) /$1 break; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-NginX-Proxy true; - - # This is necessary to pass the correct IP to be hashed - real_ip_header X-Real-IP; - - proxy_connect_timeout 300; - - # To support websockets in MinIO versions released after January 2023 - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - # Some environments may encounter CORS errors (Kubernetes + Nginx Ingress) - # Uncomment the following line to set the Origin request to an empty string - proxy_set_header Origin ''; - - chunked_transfer_encoding off; - - proxy_pass http://127.0.0.1:8081; - } -} diff --git a/minio/nginx/nginx.conf b/minio/nginx/nginx.conf deleted file mode 100644 index f9923f83a..000000000 --- a/minio/nginx/nginx.conf +++ /dev/null @@ -1,73 +0,0 @@ -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - # To allow special characters in headers - ignore_invalid_headers off; - # Allow any size file to be uploaded. - # Set to a value such as 1000m; to restrict file size to a specific value - client_max_body_size 0; - # To disable buffering - proxy_buffering off; - proxy_request_buffering off; - - sendfile on; - tcp_nopush on; - server_tokens off; - - keepalive_timeout 65; - - # Enable gzipped format - gzip on; - # Set level of compression - gzip_comp_level 3; - # Set mime types - gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - - # SSL configuration - ssl_certificate /etc/nginx/ssl/fullchain.pem; - ssl_certificate_key /etc/nginx/ssl/privkey.pem; - ssl_session_timeout 1d; - ssl_session_cache shared:MozSSL:10m; # about 40000 sessions - - # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam - ssl_dhparam /etc/nginx/ssl/dhparam; - - # intermediate configuration - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; - ssl_prefer_server_ciphers off; - - # HSTS (ngx_http_headers_module is required) (63072000 seconds) - add_header Strict-Transport-Security "max-age=63072000" always; - - # OCSP stapling - ssl_stapling on; - ssl_stapling_verify on; - - # verify chain of trust of OCSP response using Root CA and Intermediate certs - ssl_trusted_certificate /etc/nginx/ssl/fullchain.pem; - - # Disable embedding the site - add_header X-Frame-Options "SAMEORIGIN"; - # Enable XSS protection - add_header X-XSS-Protection "1;mode=block"; - - include /etc/nginx/conf.d/*.conf; -} diff --git a/rsync.exclude b/rsync.exclude index 1a53618b3..f200df5c4 100644 --- a/rsync.exclude +++ b/rsync.exclude @@ -12,4 +12,3 @@ master-libvirt/id_ed25519 master-libvirt/known_hosts master-private.cfg master-credential-provider -minio