BREAKING CHANGE: migrate elasticsearch to opensearch

.env.example

@@ -85,19 +85,17 @@ MS365_CLIENT_ID=<clientId>
 MS365_CLIENT_SECRET=<clientSecret>
 POLICY_PUBLICATION_SHIFT=3 years
 POLICY_RETENTION_SHIFT=-1 (indefinitely)
-ELASTICSEARCH_ENABLED=<"yes"|"no">
-ES_HOST="https://localhost:9200"
-ES_USERNAME="elastic"
-ES_PASSWORD="duo-password"
-ES_PORT=9200
-MONGODB_COLLECTION="Dataset"
-ES_MAX_RESULT=100000
-ES_FIELDS_LIMIT=400000
-ES_INDEX="dataset"
-ES_REFRESH=<"wait_for"|"false">
-STACK_VERSION="8.8.2"
-CLUSTER_NAME="es-cluster"
-MEM_LIMIT="4G"
+
+# "wait_for": waits for index refresh before returning, recommended for development and testing.
+# "false": returns immediately without waiting, recommended for production.
+OPENSEARCH_REFRESH=<"wait_for"|"false">
+OPENSEARCH_ENABLED=<"yes"|"no">
+OPENSEARCH_ENABLED="https://localhost:9200"
+OPENSEARCH_ENABLED="admin"
+OPENSEARCH_ENABLED="Scicat_default_password_2026"
+OPENSEARCH_DEFAULT_INDEX="dataset"
+OPENSEARCH_DATA_SYNC_BATCH_SIZE=50000
+
 FRONTEND_CONFIG_FILE="./src/config/frontend.config.json"
 FRONTEND_THEME_FILE="./src/config/frontend.theme.json"
 LOGGERS_CONFIG_FILE="loggers.json"

.github/workflows/test.yml

@@ -103,14 +103,14 @@ jobs:
         run: npm run build
 
   api_tests:
-    name: API tests with ElasticSearch enabled - ${{ matrix.elasticsearch_enabled }}
+    name: API tests with OpenSearch enabled - ${{ matrix.opensearch_enabled }}
     needs: [install-and-cache]
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
-        elasticsearch_enabled: ["yes", "no"]
+        opensearch_enabled: ["yes", "no"]
 
     steps:
       - name: Checkout code
@@ -130,12 +130,8 @@ jobs:
 
       - name: API tests
         env:
-          ELASTICSEARCH_ENABLED: ${{ matrix.elasticsearch_enabled }}
-          COMPOSE_PROFILES: ${{ matrix.elasticsearch_enabled == 'yes' && 'elasticsearch' || ''}}
-          CLUSTER_NAME: es-cluster
-          ES_PASSWORD: duo-password
-          MEM_LIMIT: 4G
-          STACK_VERSION: 8.8.2
+          COMPOSE_PROFILES: ${{ matrix.opensearch_enabled == 'yes' && 'opensearch' || ''}}
+          OPENSEARCH_ENABLED: ${{ matrix.opensearch_enabled }}
           OIDC_ISSUER: http://localhost:8080/local-test
           OIDC_CLIENT_ID: scicat-client-test
           OIDC_CLIENT_SECRET: secret
@@ -144,6 +140,7 @@ jobs:
 
         # Start mongo container and app before running api tests
         run: |
+          cp opensearchConfig.example.json opensearchConfig.json
           cp CI/ESS/docker-compose.api.yaml docker-compose.yaml
           docker compose up --build -d --wait
-          npm run test:api
+          OPENSEARCH_ENABLED=${{ matrix.opensearch_enabled }} npm run test:api

.gitignore

@@ -9,6 +9,7 @@ jobConfig.json
 jobConfig.yaml
 metricsConfig.json
 publishedDataConfig.json
+openSearchConfig.json
 
 # Configs
 .env

CI/E2E/.env.backend-next.example

@@ -39,12 +39,9 @@ DATASET_CREATION_VALIDATION_REGEX="^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][
 PROPOSAL_GROUPS="proposalingestor"
 SAMPLE_GROUPS=""
 
-ELASTICSEARCH_ENABLED='yes'
-ES_HOST=http://es01:9200
-ES_USERNAME=elastic
-ES_PASSWORD=duo-password
-MONGODB_COLLECTION=Dataset
-ES_MAX_RESULT=210000
-ES_FIELDS_LIMIT=400000
-ES_INDEX="dataset"
-ES_REFRESH="wait_for"
+OPENSEARCH_ENABLED='yes'
+OPENSEARCH_DEFAULT_INDEX="dataset"
+OPENSEARCH_HOST=https://localhost:9200
+OPENSEARCH_USERNAME=admin
+OPENSEARCH_PASSWORD=Scicat_default_password_2026
+OPENSEARCH_REFRESH="wait_for"

CI/E2E/backend.env

@@ -36,12 +36,9 @@ DATASET_CREATION_VALIDATION_REGEX="^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][
 PROPOSAL_GROUPS="proposalingestor"
 SAMPLE_GROUPS=""
 
-ELASTICSEARCH_ENABLED='yes'
-ES_HOST=http://es01:9200
-ES_USERNAME=elastic
-ES_PASSWORD=duo-password
-MONGODB_COLLECTION=Dataset
-ES_MAX_RESULT=210000
-ES_FIELDS_LIMIT=400000
-ES_INDEX="dataset"
-ES_REFRESH="wait_for"
+OPENSEARCH_ENABLED=yes
+OPENSEARCH_DEFAULT_INDEX="dataset"
+OPENSEARCH_HOST=https://localhost:9200
+OPENSEARCH_USERNAME=admin
+OPENSEARCH_PASSWORD=Scicat_default_password_2026
+OPENSEARCH_REFRESH="wait_for"

CI/E2E/docker-compose-local.yaml

@@ -1,11 +1,12 @@
 version: "3.2"
 services:
   mongodb:
-    image: bitnami/mongodb:latest
+    image: mongo:latest
     volumes:
-      - "mongodb_data:/bitnami"
+      - "mongodb_data:/data/db"
     ports:
       - "27017:27017"
+
   scichat-loopback:
     image: dacat/scichat-loopback:e2e
     command:
@@ -23,28 +24,44 @@ services:
     depends_on:
       - mongodb
 
-  es01:
+  opensearch:
     depends_on:
       - mongodb
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    image: opensearchproject/opensearch:3.5.0
     ports:
-      - ${ES_PORT}:9200
+      - 9200:9200
     environment:
-      - node.name=es01
-      - ES_JAVA_OPTS=-Xms2g -Xmx2g
-      - cluster.name=${CLUSTER_NAME}
-      - cluster.initial_master_nodes=es01
-      - ELASTIC_PASSWORD=${ES_PASSWORD}
+      - node.name=opensearch
+      - discovery.type=single-node
+      - cluster.name=os-cluster
       - bootstrap.memory_lock=true
-    mem_limit: ${MEM_LIMIT}
+      - OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Scicat_default_password_2026
+    volumes:
+      - opensearch_data:/usr/share/opensearch/data
+    mem_limit: 4G
     ulimits:
       memlock:
         soft: -1
         hard: -1
 
+
+ # Web UI for OpenSearch
+ # Enable this service if you want to use OpenSearch Dashboards to visualize and debug your OpenSearch data. 
+ # Access it at http://localhost:5601
+  # opensearch-dashboards:
+  #   image: opensearchproject/opensearch-dashboards:3.5.0
+  #   ports:
+  #     - "5601:5601"
+  #   environment:
+  #     - OPENSEARCH_HOSTS=https://opensearch:9200
+  #     - OPENSEARCH_USERNAME=admin
+  #     - OPENSEARCH_PASSWORD=Scicat_default_password_2026
+  #     - OPENSEARCH_SSL_VERIFICATIONMODE=none
+
 volumes:
   mongodb_data:
     driver: local
 
-  es01:
+  opensearch_data:
     driver: local

CI/E2E/docker-compose.yaml

@@ -8,11 +8,11 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
   mongodb:
-    image: "bitnami/mongodb:latest"
+    image: "mongo:latest"
     ports:
       - "27017:27017"
     volumes:
-      - "mongodb_data:/bitnami"
+      - "mongodb_data:/data/db"
     healthcheck:
       test: echo 'db.runCommand("ping").ok' | mongosh mongodb:27017/test --quiet
       interval: 10s
@@ -63,35 +63,28 @@ services:
       - "traefik.http.routers.frontend.rule=PathPrefix(`/`)"
       - "traefik.http.routers.frontend.entrypoints=web"
 
-  es01:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.2
+  opensearch:
+    depends_on:
+      - mongodb
+    image: opensearchproject/opensearch:3.5.0
     ports:
       - 9200:9200
     environment:
-      - xpack.security.enabled=false
-      - node.name=es01
-      - ES_JAVA_OPTS=-Xms512m -Xmx512m
-      - cluster.name=es-cluster
-      - cluster.initial_master_nodes=es01
+      - node.name=opensearch
+      - discovery.type=single-node
+      - cluster.name=os-cluster
       - bootstrap.memory_lock=true
-    mem_limit: 1g
+      - OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Scicat_default_password_2026
+    volumes:
+      - opensearch_data:/usr/share/opensearch/data
+    mem_limit: 4G
     ulimits:
       memlock:
         soft: -1
         hard: -1
-    healthcheck:
-      test:
-        [
-          "CMD-SHELL",
-          "curl -s -X GET 'http://es01:9200/_cluster/health?pretty' | grep status | grep -q '\\(green\\|yellow\\)'"
-        ]
-      interval: 30s
-      timeout: 10s
-      start_period: 60s
-      retries: 4
-
 volumes:
   mongodb_data:
     driver: local
-  es01:
+  opensearch_data:
     driver: local

CI/ESS/docker-compose.api.yaml

@@ -72,31 +72,36 @@ services:
   mongodb:
     image: mongo:latest
     volumes:
-      - mongodb_data:/bitnami
+      - "mongodb_data:/data/db"
     ports:
       - "27017:27017"
-  es01:
+  opensearch:
     profiles:
-      - elasticsearch
+      - opensearch
     depends_on:
       - mongodb
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    image: opensearchproject/opensearch:3.5.0
     ports:
       - "9200:9200"
     environment:
-      - node.name=es01
-      - ES_JAVA_OPTS=-Xms2g -Xmx2g
-      - cluster.name=${CLUSTER_NAME}
-      - cluster.initial_master_nodes=es01
-      - ELASTIC_PASSWORD=${ES_PASSWORD}
+      - node.name=opensearch
+      - discovery.type=single-node
+      - cluster.name=os-test-cluster
       - bootstrap.memory_lock=true
-    mem_limit: ${MEM_LIMIT}
+      - OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Scicat_default_password_2026
+    mem_limit: 4G
     ulimits:
       memlock:
         soft: -1
         hard: -1
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sk -u admin:Scicat_default_password_2026 https://localhost:9200/_cluster/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
+      start_period: 30s
 volumes:
   mongodb_data:
     driver: local
-  es01:
-    driver: local
+

CI/ESS/docker-compose.yaml

@@ -27,22 +27,27 @@ services:
     depends_on:
       - mongodb
 
-  es01:
+  opensearch:
     depends_on:
       - mongodb
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    image: opensearchproject/opensearch:3.5.0
+    ports:
+      - 9200:9200
     environment:
-      - node.name=es01
-      - ES_JAVA_OPTS=-Xms2g -Xmx2g
-      - cluster.name=${CLUSTER_NAME}
-      - cluster.initial_master_nodes=es01
-      - ELASTIC_PASSWORD=${ES_PASSWORD}
+      - node.name=opensearch
+      - discovery.type=single-node
+      - cluster.name=os-cluster
       - bootstrap.memory_lock=true
-    mem_limit: ${MEM_LIMIT}
+      - OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Scicat_default_password_2026
+    volumes:
+      - opensearch_data:/usr/share/opensearch/data
+    mem_limit: 4G
     ulimits:
       memlock:
         soft: -1
         hard: -1
+
 #  catanie:
 #    build: 
 #      context: catanie
@@ -114,5 +119,5 @@ volumes:
     driver: local
   rabbitmq_data:
     driver: local
-  es01:
-    driver: local
+  opensearch_data:
+    driver: local