From ed46d5e830b7830cdb71e14b19477d8fa7fbcdd8 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Mon, 22 Sep 2025 10:14:33 +0200 Subject: [PATCH 01/14] Fix doc on RefreshSchedulerConfig --- .../main/java/de/cxp/ocs/config/RefreshSchedulerConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search-service/src/main/java/de/cxp/ocs/config/RefreshSchedulerConfig.java b/search-service/src/main/java/de/cxp/ocs/config/RefreshSchedulerConfig.java index d51827cc7..f079dc6c5 100644 --- a/search-service/src/main/java/de/cxp/ocs/config/RefreshSchedulerConfig.java +++ b/search-service/src/main/java/de/cxp/ocs/config/RefreshSchedulerConfig.java @@ -16,11 +16,11 @@ *

set property 'ocs.scheduler.enabled=false' to disable completely.

* *

- * To enable refresh config, set the property 'ocs.scheduler.enabled.refresh-config=true'. + * To enable refresh config, set the property 'ocs.scheduler.refresh-config.enabled=true'. * The property 'ocs.scheduler.refresh-config-delay-ms' can be set to configure the fixed delay between each refresh (defaults to 60000 = 1 minute) *

*

- * To enable context refreshing, set the property 'ocs.scheduler.enabled.refresh-context=true'. + * To enable context refreshing, set the property 'ocs.scheduler.refresh-context.enabled=true'. * The property 'ocs.scheduler.refresh-context-delay-ms' can be set to configure the fixed delay between each refresh (defaults to 60000 = 1 minute) *

*

From cd8f78432a2051115c80c28a8413645413139ac4 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 11:23:09 +0100 Subject: [PATCH 02/14] Add es8 compatibility mode --- .../src/main/java/de/cxp/ocs/Application.java | 23 ++++++++++------ .../ocs/config/ConnectionConfiguration.java | 6 +++++ .../elasticsearch/ElasticSearchBuilder.java | 15 +++++++---- pom.xml | 2 +- .../src/main/java/de/cxp/ocs/Application.java | 26 ++++++++++++++----- .../ElasticsearchSuggestDataProvider.java | 5 +++- 6 files changed, 56 insertions(+), 21 deletions(-) diff --git a/indexer-service/src/main/java/de/cxp/ocs/Application.java b/indexer-service/src/main/java/de/cxp/ocs/Application.java index feb1e5968..c5317777e 100644 --- a/indexer-service/src/main/java/de/cxp/ocs/Application.java +++ b/indexer-service/src/main/java/de/cxp/ocs/Application.java @@ -4,8 +4,10 @@ import de.cxp.ocs.client.deserializer.DocumentDeserializer; import de.cxp.ocs.client.deserializer.ProductDeserializer; -import org.elasticsearch.client.RestClientBuilder; +import de.cxp.ocs.config.ConnectionConfiguration; +import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RestHighLevelClientBuilder; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cloud.context.config.annotation.RefreshScope; @@ -45,19 +47,25 @@ public static void main(String[] args) { } @Bean - public RestClientBuilder getRestClientBuilder(ApplicationProperties properties) { - return RestClientBuilderFactory.createRestClientBuilder(properties.getConnectionConfiguration()); + public RestClient getRestClient(ApplicationProperties properties) { + ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration(); + log.info("going to connect to Elasticsearch hosts {}", connectionConfig.getHosts()); + return RestClientBuilderFactory.createRestClientBuilder(connectionConfig).build(); } @Bean @SuppressWarnings("deprecation") - public RestHighLevelClient getRestHighLevelClient(RestClientBuilder clientBuilder) { - return new RestHighLevelClient(clientBuilder); + public RestHighLevelClient getRestHighLevelClient(RestClient restClient, ApplicationProperties properties) { + ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration(); + return new RestHighLevelClientBuilder(restClient) + .setApiCompatibilityMode(connectionConfig.isUseCompatibilityMode()) + .build(); } @Bean - public ElasticSearchBuilder getESBuilder(RestClientBuilder restClientBuilder) { - return new ElasticSearchBuilder(restClientBuilder); + public ElasticSearchBuilder getESBuilder(RestClient restClient, ApplicationProperties properties) { + ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration(); + return new ElasticSearchBuilder(restClient, connectionConfig.isUseCompatibilityMode()); } @Bean @@ -67,7 +75,6 @@ public PluginManager getPluginManager(ApplicationProperties properties) { @Bean public IndexerConfigurationProvider configurationProvider(PluginManager pluginManager, ApplicationProperties properties) { - log.info("going to connect to Elasticsearch hosts {}", properties.getConnectionConfiguration().getHosts()); DefaultIndexerConfigurationProvider defaultConfigProvider = new DefaultIndexerConfigurationProvider(properties); Optional indexerConfigurationProvider = pluginManager.loadPrefered(IndexerConfigurationProvider.class); indexerConfigurationProvider.ifPresent(icp -> icp.setDefaultProvider(defaultConfigProvider)); diff --git a/ocs-commons/src/main/java/de/cxp/ocs/config/ConnectionConfiguration.java b/ocs-commons/src/main/java/de/cxp/ocs/config/ConnectionConfiguration.java index a71058960..5a42d76f2 100644 --- a/ocs-commons/src/main/java/de/cxp/ocs/config/ConnectionConfiguration.java +++ b/ocs-commons/src/main/java/de/cxp/ocs/config/ConnectionConfiguration.java @@ -22,4 +22,10 @@ public class ConnectionConfiguration { * Example: "elastic:my$ecretPassw0rd" */ private String auth; + + /** + * If set to 'true' the rest-high-level-client version 7 can be used with elasticsearch version 8. + * see Compatibility with Elasticsearch 8.x + */ + private boolean useCompatibilityMode = false; } diff --git a/ocs-commons/src/main/java/de/cxp/ocs/elasticsearch/ElasticSearchBuilder.java b/ocs-commons/src/main/java/de/cxp/ocs/elasticsearch/ElasticSearchBuilder.java index b08f0049d..0bc01cab0 100644 --- a/ocs-commons/src/main/java/de/cxp/ocs/elasticsearch/ElasticSearchBuilder.java +++ b/ocs-commons/src/main/java/de/cxp/ocs/elasticsearch/ElasticSearchBuilder.java @@ -1,22 +1,27 @@ package de.cxp.ocs.elasticsearch; -import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RestHighLevelClientBuilder; public class ElasticSearchBuilder implements AutoCloseable { private RestHighLevelClient highLevelClient; - private final RestClientBuilder restClientBuilder; + private final RestClient restClient; + private final boolean useCompatibilityMode; - public ElasticSearchBuilder(RestClientBuilder clientBuilder) { - restClientBuilder = clientBuilder; + public ElasticSearchBuilder(RestClient restClient, boolean useCompatibilityMode) { + this.restClient = restClient; + this.useCompatibilityMode = useCompatibilityMode; } public RestHighLevelClient getRestHLClient() { if (highLevelClient == null) { synchronized (this) { if (highLevelClient == null) { - highLevelClient = new RestHighLevelClient(restClientBuilder); + highLevelClient = new RestHighLevelClientBuilder(restClient) + .setApiCompatibilityMode(useCompatibilityMode) + .build(); } } } diff --git a/pom.xml b/pom.xml index 52aeeb6ef..05d2d10c5 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ ${project.version} latest - 7.17.28 + 7.17.29 3.5.5 2025.0.0 1.8.0 diff --git a/search-service/src/main/java/de/cxp/ocs/Application.java b/search-service/src/main/java/de/cxp/ocs/Application.java index faf38f186..2e0b16172 100644 --- a/search-service/src/main/java/de/cxp/ocs/Application.java +++ b/search-service/src/main/java/de/cxp/ocs/Application.java @@ -2,8 +2,11 @@ import java.util.Optional; +import de.cxp.ocs.config.ConnectionConfiguration; import de.cxp.ocs.model.params.*; -import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RestHighLevelClientBuilder; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cloud.context.config.annotation.RefreshScope; @@ -37,14 +40,25 @@ public static void main(String[] args) { } @Bean - public ElasticSearchBuilder getESBuilder(RestClientBuilder restClientBuilder) { - return new ElasticSearchBuilder(restClientBuilder); + public RestClient getRestClient(ApplicationProperties properties) { + ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration(); + log.info("going to connect to Elasticsearch hosts {}", connectionConfig.getHosts()); + return RestClientBuilderFactory.createRestClientBuilder(connectionConfig).build(); } @Bean - public RestClientBuilder getRestClientBuilder(ApplicationProperties properties) { - log.info("going to connect to Elasticsearch hosts {}", properties.getConnectionConfiguration().getHosts()); - return RestClientBuilderFactory.createRestClientBuilder(properties.getConnectionConfiguration()); + @SuppressWarnings("deprecation") + public RestHighLevelClient getRestHighLevelClient(RestClient restClient, ApplicationProperties properties) { + ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration(); + return new RestHighLevelClientBuilder(restClient) + .setApiCompatibilityMode(connectionConfig.isUseCompatibilityMode()) + .build(); + } + + @Bean + public ElasticSearchBuilder getESBuilder(RestClient restClient, ApplicationProperties properties) { + ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration(); + return new ElasticSearchBuilder(restClient, connectionConfig.isUseCompatibilityMode()); } @Bean diff --git a/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/ElasticsearchSuggestDataProvider.java b/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/ElasticsearchSuggestDataProvider.java index ec715fe1e..e933181e4 100644 --- a/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/ElasticsearchSuggestDataProvider.java +++ b/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/ElasticsearchSuggestDataProvider.java @@ -18,6 +18,7 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RestHighLevelClientBuilder; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.client.indices.GetIndexRequest; @@ -62,7 +63,9 @@ public void configure(Map config) { ConnectionConfiguration connectionConf = settings.getConnectionConfig(); log.info("Connecting to Elasticsearch at {}", connectionConf.getHosts()); RestClientBuilder restClientBuilder = RestClientBuilderFactory.createRestClientBuilder(connectionConf); - client = new RestHighLevelClient(restClientBuilder); + client = new RestHighLevelClientBuilder(restClientBuilder.build()) + .setApiCompatibilityMode(connectionConf.isUseCompatibilityMode()) + .build(); } @Override From a98d5b68f9031f18125c71f809d8b302e7c5f8a0 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 11:24:17 +0100 Subject: [PATCH 03/14] Remove doc_values for potential text fields (did not work in es7 and causes errors starting from v8) --- .../elasticsearch/_component_templates/ocs_default.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/indexer-service/src/main/resources/elasticsearch/_component_templates/ocs_default.json b/indexer-service/src/main/resources/elasticsearch/_component_templates/ocs_default.json index 0dd0c8fc0..a149985e2 100644 --- a/indexer-service/src/main/resources/elasticsearch/_component_templates/ocs_default.json +++ b/indexer-service/src/main/resources/elasticsearch/_component_templates/ocs_default.json @@ -166,8 +166,7 @@ "result_data": { "path_match": "*resultData.*", "mapping": { - "index": false, - "doc_values": false + "index": false } } }, From 3a82949d8d7713c0402c74acfbe2030544259f6f Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 11:26:03 +0100 Subject: [PATCH 04/14] Bump versions for next release (0.73) --- config-service/pom.xml | 2 +- indexer-service/pom.xml | 4 ++-- ocs-commons/pom.xml | 2 +- ocs-plugin-spi/pom.xml | 2 +- pom.xml | 16 ++++++++-------- search-service/pom.xml | 4 ++-- .../ocs-suggest-data-provider/pom.xml | 4 ++-- suggest-service-parent/pom.xml | 4 ++-- suggest-service-parent/suggest-service/pom.xml | 4 ++-- 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/config-service/pom.xml b/config-service/pom.xml index 213fc3c5a..9e203c9aa 100644 --- a/config-service/pom.xml +++ b/config-service/pom.xml @@ -10,7 +10,7 @@ config-service - 0.2.1 + 0.3.0 diff --git a/indexer-service/pom.xml b/indexer-service/pom.xml index 4c8050828..613325da7 100644 --- a/indexer-service/pom.xml +++ b/indexer-service/pom.xml @@ -5,12 +5,12 @@ de.cxp.ocs ocs-parent - 0.73.0 + 0.74.0 ../pom.xml indexer-service - 0.45.7 + 0.46.0 jar diff --git a/ocs-commons/pom.xml b/ocs-commons/pom.xml index 141485e86..99dc8810c 100644 --- a/ocs-commons/pom.xml +++ b/ocs-commons/pom.xml @@ -12,7 +12,7 @@ ocs-commons - 0.34.3 + 0.35.0 jar Library that contains common models and utility for the open-commerce-search stack diff --git a/ocs-plugin-spi/pom.xml b/ocs-plugin-spi/pom.xml index 1d2b11f04..98e776fc4 100644 --- a/ocs-plugin-spi/pom.xml +++ b/ocs-plugin-spi/pom.xml @@ -10,7 +10,7 @@ ocs-plugin-spi - 1.18.2 + 1.19.0 jar Service Provider Interface for Plugins that could be optionally added to the services diff --git a/pom.xml b/pom.xml index 05d2d10c5..47d805065 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ de.cxp.ocs ocs-parent pom - 0.73.0 + 0.74.0 SearchHub Services - Parent @@ -47,39 +47,39 @@ de.cxp.ocs open-commerce-search-api - 0.26.2 + 0.27.0 de.cxp.ocs ocs-plugin-spi - 1.18.2 + 1.19.0 de.cxp.ocs ocs-commons - 0.34.3 + 0.35.0 de.cxp.ocs indexer-service - 0.45.7 + 0.46.0 de.cxp.ocs search-service - 0.63.3 + 0.64.0 de.cxp.ocs suggest-service - 0.32.1 + 0.33.0 de.cxp.ocs ocs-java-client - 0.21.2 + 0.22.0 diff --git a/search-service/pom.xml b/search-service/pom.xml index af004e4ad..7bff22312 100644 --- a/search-service/pom.xml +++ b/search-service/pom.xml @@ -5,12 +5,12 @@ de.cxp.ocs ocs-parent - 0.73.0 + 0.74.0 ../pom.xml search-service - 0.63.3 + 0.64.0 jar diff --git a/suggest-service-parent/ocs-suggest-data-provider/pom.xml b/suggest-service-parent/ocs-suggest-data-provider/pom.xml index 115cc0e0f..4b9806394 100644 --- a/suggest-service-parent/ocs-suggest-data-provider/pom.xml +++ b/suggest-service-parent/ocs-suggest-data-provider/pom.xml @@ -6,11 +6,11 @@ de.cxp.ocs suggest-service-parent - 0.73.0 + 0.74.0 ocs-suggest-data-provider - 0.24.1 + 0.25.0 jar OCS Suggest Data Provider diff --git a/suggest-service-parent/pom.xml b/suggest-service-parent/pom.xml index c287985a2..24da312d6 100644 --- a/suggest-service-parent/pom.xml +++ b/suggest-service-parent/pom.xml @@ -7,7 +7,7 @@ de.cxp.ocs ocs-parent - 0.73.0 + 0.74.0 ../pom.xml @@ -38,7 +38,7 @@ de.cxp.ocs ocs-suggest-data-provider - 0.24.1 + 0.25.0 de.cxp.ocs diff --git a/suggest-service-parent/suggest-service/pom.xml b/suggest-service-parent/suggest-service/pom.xml index e4d6b1852..781fa0540 100644 --- a/suggest-service-parent/suggest-service/pom.xml +++ b/suggest-service-parent/suggest-service/pom.xml @@ -5,12 +5,12 @@ de.cxp.ocs suggest-service-parent - 0.73.0 + 0.74.0 ../pom.xml suggest-service - 0.32.1 + 0.33.0 jar smartSuggest RESTful Service From 82ec3fce5f8037310eccfa8b33d1915eef64e01f Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 11:28:27 +0100 Subject: [PATCH 05/14] search-service: update ocs-smartquery-plugin version --- search-service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-service/pom.xml b/search-service/pom.xml index 7bff22312..13da15d04 100644 --- a/search-service/pom.xml +++ b/search-service/pom.xml @@ -303,7 +303,7 @@ io.searchhub ocs-smartquery-plugin - 2.3.1-ocs_spi_1.17.2 + 2.3.3-ocs_spi_1.17.3 From 4a88687c880d4a1104ac03c09f9b63c4bbcd510f Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 11:28:40 +0100 Subject: [PATCH 06/14] Remove duplicate dependency --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index 47d805065..51bdc854b 100644 --- a/pom.xml +++ b/pom.xml @@ -175,11 +175,6 @@ httpclient 4.5.13 - - org.apache.commons - commons-text - 1.10.0 - net.minidev json-smart From dc49b661101698065a337e8483b1d7f62a7c5280 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 12:56:26 +0100 Subject: [PATCH 07/14] Fix bean creation in ElasticsearchCRUDTest --- .../de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java index 738aa9774..7c6ea35e2 100644 --- a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java +++ b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.GetIndexRequest; @@ -73,10 +74,12 @@ public class ElasticsearchCRUDTest { static class TestConf extends Application { @Bean - public RestClientBuilder getRestClientBuilder(ApplicationProperties properties) { + @Override + public RestClient getRestClient(ApplicationProperties properties) { System.out.println("initializing ES client"); properties.getConnectionConfiguration().setHosts("127.0.0.1:" + HTTP_TEST_PORT); - return RestClientBuilderFactory.createRestClientBuilder(properties.getConnectionConfiguration()); + properties.getConnectionConfiguration().setUseCompatibilityMode(true); + return RestClientBuilderFactory.createRestClientBuilder(properties.getConnectionConfiguration()).build(); } } From 581a222781f09297b07129a0f8e2904a559cec18 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 12:56:47 +0100 Subject: [PATCH 08/14] Change to run CRUD-test with elasticsearch-8 --- .../de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java | 2 +- .../cxp/ocs/elasticsearch/ElasticsearchContainerUtil.java | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java index 7c6ea35e2..fa4edc22b 100644 --- a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java +++ b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java @@ -86,7 +86,7 @@ public RestClient getRestClient(ApplicationProperties properties) { @BeforeAll public static void spinUpEs() { - container = ElasticsearchContainerUtil.spinUpEs(); + container = ElasticsearchContainerUtil.spinUpEs("8.19.8"); HTTP_TEST_PORT = container.getMappedPort(ElasticsearchContainerUtil.ES_PORT); } diff --git a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchContainerUtil.java b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchContainerUtil.java index d52e64832..ef2914c44 100644 --- a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchContainerUtil.java +++ b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchContainerUtil.java @@ -19,13 +19,19 @@ public class ElasticsearchContainerUtil { public static final int ES_PORT = 9200; public static ElasticsearchContainer spinUpEs() { + return spinUpEs(Version.CURRENT.toString()); + } + + public static ElasticsearchContainer spinUpEs(String esVersion) { log.info("starting Elasticsearch container.."); ElasticsearchContainer container = new ElasticsearchContainer( DockerImageName .parse("docker.elastic.co/elasticsearch/elasticsearch") - .withTag(Version.CURRENT.toString())); + .withTag(esVersion)); container.addEnv("discovery.type", "single-node"); container.addEnv("ES_JAVA_OPTS", "-Xms1024m -Xmx1024m"); + container.addEnv("xpack.security.enabled", "false"); + container.addEnv("xpack.security.http.ssl.enabled", "false"); container.setWaitStrategy(new HttpWaitStrategy().forPort(ES_PORT)); container.withStartupTimeout(Duration.ofSeconds(60)); container.start(); From ed2e24d454cc852989c2e4177616f3b8696bb9a6 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 16:55:48 +0100 Subject: [PATCH 09/14] Update mockito-junit-jupiter version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 51bdc854b..d52d1e1ea 100644 --- a/pom.xml +++ b/pom.xml @@ -245,7 +245,7 @@ org.mockito mockito-junit-jupiter - 3.7.7 + 5.21.0 test From 41a9657c1e89f016c94c7477e9b3570f3f67c7b9 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 16:56:06 +0100 Subject: [PATCH 10/14] Add tests support for elasticsearch v8 --- Jenkinsfile | 5 +++++ .../de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java | 4 ++-- .../cxp/ocs/elasticsearch/ElasticsearchIndexerTest.java | 9 ++++----- integration-tests/src/test/java/de/cxp/ocs/OCSStack.java | 6 +++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2b01222c4..ec47e4343 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,6 +31,11 @@ pipeline { steps { withMaven(mavenSettingsConfig: '67c40a88-505a-4f78-94a3-d879cc1a29f6') { sh "mvn $MAVEN_CLI_OPTS integration-test -pl integration-tests" + + withEnv(["ES_CONTAINER_VERSION=8.19.8"]) { + // indexer-service also contains two "integration tests" with an elasticsearch container + sh "mvn $MAVEN_CLI_OPTS integration-test -pl indexer-service,integration-tests" + } } } } // end integration tests diff --git a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java index fa4edc22b..d4cca12f5 100644 --- a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java +++ b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java @@ -17,11 +17,11 @@ import java.util.*; import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.Version; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.GetIndexRequest; import org.junit.jupiter.api.AfterAll; @@ -86,7 +86,7 @@ public RestClient getRestClient(ApplicationProperties properties) { @BeforeAll public static void spinUpEs() { - container = ElasticsearchContainerUtil.spinUpEs("8.19.8"); + container = ElasticsearchContainerUtil.spinUpEs(Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).orElse(Version.CURRENT.toString())); HTTP_TEST_PORT = container.getMappedPort(ElasticsearchContainerUtil.ES_PORT); } diff --git a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchIndexerTest.java b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchIndexerTest.java index 22737ca58..c21d8a311 100644 --- a/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchIndexerTest.java +++ b/indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchIndexerTest.java @@ -5,16 +5,15 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.util.Optional; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.core.List; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.testcontainers.elasticsearch.ElasticsearchContainer; import de.cxp.ocs.api.indexer.ImportSession; @@ -29,7 +28,7 @@ public class ElasticsearchIndexerTest { @BeforeAll public static void startElasticsearch() { - container = ElasticsearchContainerUtil.spinUpEs(); + container = ElasticsearchContainerUtil.spinUpEs(Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).orElse(Version.CURRENT.toString())); } @AfterAll diff --git a/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java b/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java index 3fc14b2dc..b7ec18c99 100644 --- a/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java +++ b/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java @@ -3,6 +3,7 @@ import java.lang.reflect.Method; import java.net.URL; import java.util.Collections; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; @@ -71,8 +72,11 @@ public void beforeAll(ExtensionContext context) throws Exception { } } else { - elasticsearch = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + Version.CURRENT.toString()); + String esVersion = Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).orElse(Version.CURRENT.toString()); + elasticsearch = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + esVersion); elasticsearch.addEnv("discovery.type", "single-node"); + elasticsearch.addEnv("xpack.security.enabled", "false"); + elasticsearch.addEnv("xpack.security.http.ssl.enabled", "false"); elasticsearch.setExposedPorts(Collections.singletonList(ES_DEFAULT_PORT)); elasticsearch.withNetwork(Network.newNetwork()); elasticsearch.withNetworkAliases("elasticsearch"); From 269e4d8e807a59bb2ed3e923ab17bc408dc1bd2b Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 17:03:10 +0100 Subject: [PATCH 11/14] Fix wrong version bump of api --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d52d1e1ea..bd11b0121 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ de.cxp.ocs open-commerce-search-api - 0.27.0 + 0.26.2 de.cxp.ocs From 2ae11040d4f54dc48b1476c9f3001bba03041f30 Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 17:06:25 +0100 Subject: [PATCH 12/14] Fix wrong version bump for ocs-java-client --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bd11b0121..99f7416d7 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ de.cxp.ocs ocs-java-client - 0.22.0 + 0.21.2 From 544bee8ce98d01a787b50403364f13757e0278eb Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 17:50:29 +0100 Subject: [PATCH 13/14] itest: pass compatibility params to OCSStack --- .../src/test/java/de/cxp/ocs/OCSStack.java | 20 ++++++++++++++++--- .../cxp/ocs/elasticsearch/SettingsProxy.java | 1 + .../resources/ocs-suggest.default.properties | 3 +++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java b/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java index b7ec18c99..4700bd228 100644 --- a/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java +++ b/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java @@ -109,7 +109,12 @@ private CompletableFuture startIndexerService(CompletableFuture("commerceexperts/ocs-indexer-service:latest"); indexerService.addExposedPort(INDEXER_DEFAULT_PORT); - indexerService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,test"); + + String v8comp = ""; + if (Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).map(v -> v.startsWith("8")).orElse(false)) { + v8comp = " -Docs.connection-configuration.use-compatibility-mode=true"; + } + indexerService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,test" + v8comp); bindFile(indexerService, "indexer.application-test.yml", "application-test.yml"); @@ -140,7 +145,11 @@ private CompletableFuture startSearchService(CompletableFuture searchService.addExposedPort(SEARCH_DEFAULT_PORT); // searchService.setCommand("-Dspring.cloud.config.enabled=false", // "-Dspring.profiles.active=preset"); - searchService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,trace-searches,test"); + String v8comp = ""; + if (Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).map(v -> v.startsWith("8")).orElse(false)) { + v8comp = " -Docs.connection-configuration.use-compatibility-mode=true"; + } + searchService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,trace-searches,test" + v8comp); bindFile(searchService, "searcher.application-test.yml", "application-test.yml"); bindFile(searchService, "querqy-test-rules.txt", "querqy-test-rules.txt"); @@ -180,7 +189,12 @@ private CompletableFuture startSuggestService(CompletableFuture("commerceexperts/ocs-suggest-service:latest"); suggestService.addExposedPort(SUGGEST_DEFAULT_PORT); - suggestService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m"); + + String v8comp = ""; + if (Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).map(v -> v.startsWith("8")).orElse(false)) { + v8comp = " -Delasticsearch.useCompatibilityMode=true"; + } + suggestService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m" + v8comp); String esAddr; if (elasticsearch != null) { diff --git a/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/SettingsProxy.java b/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/SettingsProxy.java index 02296b353..042b15130 100644 --- a/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/SettingsProxy.java +++ b/suggest-service-parent/ocs-suggest-data-provider/src/main/java/de/cxp/ocs/elasticsearch/SettingsProxy.java @@ -77,6 +77,7 @@ public ConnectionConfiguration getConnectionConfig() { connectionConf = new ConnectionConfiguration(); connectionConf.setHosts(get("elasticsearch.hosts")); connectionConf.setAuth(get("elasticsearch.auth")); + connectionConf.setUseCompatibilityMode(Boolean.parseBoolean(get("elasticsearch.useCompatibilityMode"))); } return connectionConf; } diff --git a/suggest-service-parent/ocs-suggest-data-provider/src/main/resources/ocs-suggest.default.properties b/suggest-service-parent/ocs-suggest-data-provider/src/main/resources/ocs-suggest.default.properties index ca88f21c4..1afcfe1ce 100644 --- a/suggest-service-parent/ocs-suggest-data-provider/src/main/resources/ocs-suggest.default.properties +++ b/suggest-service-parent/ocs-suggest-data-provider/src/main/resources/ocs-suggest.default.properties @@ -4,6 +4,9 @@ elasticsearch.hosts=localhost:9200 # optional basic auth can be specified as well #elasticsearch.auth=user:password +# set to true, if you connect to a version 8 Elasticsearch cluster +#elasticsearch.useCompatibilityMode=true + # enable availability of index name like that #suggest.index..enable=true # if not set, the data provider will simply check, if index exists From 26b9ca6772d922c68f4ad9f9c9db8999733de2cc Mon Sep 17 00:00:00 2001 From: Rudolf Batt Date: Tue, 16 Dec 2025 18:11:47 +0100 Subject: [PATCH 14/14] Adjust test requests to restricted v8 api --- .../test/java/de/cxp/ocs/ITBulkIndexationWorks.java | 2 +- .../src/test/java/de/cxp/ocs/ITPartialUpdates.java | 2 +- .../src/test/java/de/cxp/ocs/ITUpdateApi.java | 6 +++--- .../src/test/java/de/cxp/ocs/OCSStack.java | 10 +++++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/integration-tests/src/test/java/de/cxp/ocs/ITBulkIndexationWorks.java b/integration-tests/src/test/java/de/cxp/ocs/ITBulkIndexationWorks.java index a5b3b3fcc..32d104cdf 100644 --- a/integration-tests/src/test/java/de/cxp/ocs/ITBulkIndexationWorks.java +++ b/integration-tests/src/test/java/de/cxp/ocs/ITBulkIndexationWorks.java @@ -45,6 +45,6 @@ public void testDefaultIndexation() throws Exception { } private void flushIndex() throws IOException { - getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush/synced")); + getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush?wait_if_ongoing=true")); } } diff --git a/integration-tests/src/test/java/de/cxp/ocs/ITPartialUpdates.java b/integration-tests/src/test/java/de/cxp/ocs/ITPartialUpdates.java index 71d5fb97a..e1feced6d 100644 --- a/integration-tests/src/test/java/de/cxp/ocs/ITPartialUpdates.java +++ b/integration-tests/src/test/java/de/cxp/ocs/ITPartialUpdates.java @@ -119,7 +119,7 @@ private SearchResult doSimpleSearch(String query) { } private void flushIndex() throws IOException { - getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush/synced")); + getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush?wait_if_ongoing=true")); } } diff --git a/integration-tests/src/test/java/de/cxp/ocs/ITUpdateApi.java b/integration-tests/src/test/java/de/cxp/ocs/ITUpdateApi.java index 2aa98f45b..e75594cea 100644 --- a/integration-tests/src/test/java/de/cxp/ocs/ITUpdateApi.java +++ b/integration-tests/src/test/java/de/cxp/ocs/ITUpdateApi.java @@ -32,13 +32,13 @@ public class ITUpdateApi { // no indexation, since we want to test if the update API creates the necessary index @AfterEach public void deleteIndexes() { - Request deleteRequest = new Request("DELETE", "ocs-*-" + indexName + "*"); + Request deleteRequest = new Request("DELETE", "ocs-1-" + indexName + "-en"); try { Response response = getElasticsearchClient().performRequest(deleteRequest); log.info("deleting index {} responded with {}", indexName, response); } catch (IOException e) { - log.error("failed to delete index {} after test", indexName); + log.error("failed to delete index {} after test: {}:{}", indexName, e.getClass(), e.getMessage()); } } @@ -93,7 +93,7 @@ private SearchResult doSimpleSearch(String query) { } private void flushIndex() throws IOException { - getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush/synced")); + getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush?wait_if_ongoing=true")); } } diff --git a/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java b/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java index 4700bd228..e990f70b5 100644 --- a/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java +++ b/integration-tests/src/test/java/de/cxp/ocs/OCSStack.java @@ -111,7 +111,7 @@ private CompletableFuture startIndexerService(CompletableFuture v.startsWith("8")).orElse(false)) { + if (isWithEs8Compatibility()) { v8comp = " -Docs.connection-configuration.use-compatibility-mode=true"; } indexerService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,test" + v8comp); @@ -146,7 +146,7 @@ private CompletableFuture startSearchService(CompletableFuture // searchService.setCommand("-Dspring.cloud.config.enabled=false", // "-Dspring.profiles.active=preset"); String v8comp = ""; - if (Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).map(v -> v.startsWith("8")).orElse(false)) { + if (isWithEs8Compatibility()) { v8comp = " -Docs.connection-configuration.use-compatibility-mode=true"; } searchService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,trace-searches,test" + v8comp); @@ -191,7 +191,7 @@ private CompletableFuture startSuggestService(CompletableFuture v.startsWith("8")).orElse(false)) { + if (isWithEs8Compatibility()) { v8comp = " -Delasticsearch.useCompatibilityMode=true"; } suggestService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m" + v8comp); @@ -220,6 +220,10 @@ private CompletableFuture startSuggestService(CompletableFuture v.startsWith("8")).orElse(false); + } + public static RestClient getElasticsearchClient() { assert isStarted.get() : "Stack not started yet!"; return esRestClient;