Release 0.74.0

Jenkinsfile

@@ -31,6 +31,11 @@ pipeline {
       steps {
         withMaven(mavenSettingsConfig: '67c40a88-505a-4f78-94a3-d879cc1a29f6') {
           sh "mvn $MAVEN_CLI_OPTS integration-test -pl integration-tests"
+
+          withEnv(["ES_CONTAINER_VERSION=8.19.8"]) {
+            // indexer-service also contains two "integration tests" with an elasticsearch container
+            sh "mvn $MAVEN_CLI_OPTS integration-test -pl indexer-service,integration-tests"
+          }
         }
       }
     } // end integration tests

config-service/pom.xml

@@ -10,7 +10,7 @@
     </parent>
 
     <artifactId>config-service</artifactId>
-    <version>0.2.1</version>
+    <version>0.3.0</version>
 
     <dependencyManagement>
         <dependencies>

indexer-service/pom.xml

@@ -5,12 +5,12 @@
     <parent>
         <groupId>de.cxp.ocs</groupId>
         <artifactId>ocs-parent</artifactId>
-        <version>0.73.0</version>
+        <version>0.74.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
     <artifactId>indexer-service</artifactId>
-    <version>0.45.7</version>
+    <version>0.46.0</version>
     <packaging>jar</packaging>
 
     <!-- Import dependency management from Spring Boot -->

indexer-service/src/main/java/de/cxp/ocs/Application.java

@@ -4,8 +4,10 @@
 
 import de.cxp.ocs.client.deserializer.DocumentDeserializer;
 import de.cxp.ocs.client.deserializer.ProductDeserializer;
-import org.elasticsearch.client.RestClientBuilder;
+import de.cxp.ocs.config.ConnectionConfiguration;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.RestHighLevelClientBuilder;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.context.config.annotation.RefreshScope;
@@ -45,19 +47,25 @@ public static void main(String[] args) {
 	}
 
 	@Bean
-	public RestClientBuilder getRestClientBuilder(ApplicationProperties properties) {
-		return RestClientBuilderFactory.createRestClientBuilder(properties.getConnectionConfiguration());
+	public RestClient getRestClient(ApplicationProperties properties) {
+		ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration();
+		log.info("going to connect to Elasticsearch hosts {}", connectionConfig.getHosts());
+		return RestClientBuilderFactory.createRestClientBuilder(connectionConfig).build();
 	}
 
 	@Bean
 	@SuppressWarnings("deprecation")
-	public RestHighLevelClient getRestHighLevelClient(RestClientBuilder clientBuilder) {
-		return new RestHighLevelClient(clientBuilder);
+	public RestHighLevelClient getRestHighLevelClient(RestClient restClient, ApplicationProperties properties) {
+		ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration();
+		return new RestHighLevelClientBuilder(restClient)
+				.setApiCompatibilityMode(connectionConfig.isUseCompatibilityMode())
+				.build();
 	}
 
 	@Bean
-	public ElasticSearchBuilder getESBuilder(RestClientBuilder restClientBuilder) {
-		return new ElasticSearchBuilder(restClientBuilder);
+	public ElasticSearchBuilder getESBuilder(RestClient restClient, ApplicationProperties properties) {
+		ConnectionConfiguration connectionConfig = properties.getConnectionConfiguration();
+		return new ElasticSearchBuilder(restClient, connectionConfig.isUseCompatibilityMode());
 	}
 
 	@Bean
@@ -67,7 +75,6 @@ public PluginManager getPluginManager(ApplicationProperties properties) {
 
 	@Bean
 	public IndexerConfigurationProvider configurationProvider(PluginManager pluginManager, ApplicationProperties properties) {
-		log.info("going to connect to Elasticsearch hosts {}", properties.getConnectionConfiguration().getHosts());
 		DefaultIndexerConfigurationProvider defaultConfigProvider = new DefaultIndexerConfigurationProvider(properties);
 		Optional<IndexerConfigurationProvider> indexerConfigurationProvider = pluginManager.loadPrefered(IndexerConfigurationProvider.class);
 		indexerConfigurationProvider.ifPresent(icp -> icp.setDefaultProvider(defaultConfigProvider));

indexer-service/src/main/resources/elasticsearch/_component_templates/ocs_default.json

@@ -166,8 +166,7 @@
           "result_data": {
             "path_match": "*resultData.*",
             "mapping": {
-              "index": false,
-              "doc_values": false
+              "index": false
             }
           }
         },

indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchCRUDTest.java

@@ -17,10 +17,11 @@
 import java.util.*;
 
 import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.indices.GetIndexRequest;
 import org.junit.jupiter.api.AfterAll;
@@ -73,17 +74,19 @@ public class ElasticsearchCRUDTest {
 	static class TestConf extends Application {
 
 		@Bean
-		public RestClientBuilder getRestClientBuilder(ApplicationProperties properties) {
+		@Override
+		public RestClient getRestClient(ApplicationProperties properties) {
 			System.out.println("initializing ES client");
 			properties.getConnectionConfiguration().setHosts("127.0.0.1:" + HTTP_TEST_PORT);
-			return RestClientBuilderFactory.createRestClientBuilder(properties.getConnectionConfiguration());
+			properties.getConnectionConfiguration().setUseCompatibilityMode(true);
+			return RestClientBuilderFactory.createRestClientBuilder(properties.getConnectionConfiguration()).build();
 		}
 
 	}
 
 	@BeforeAll
 	public static void spinUpEs() {
-		container = ElasticsearchContainerUtil.spinUpEs();
+		container = ElasticsearchContainerUtil.spinUpEs(Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).orElse(Version.CURRENT.toString()));
 		HTTP_TEST_PORT = container.getMappedPort(ElasticsearchContainerUtil.ES_PORT);
 	}
 

indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchContainerUtil.java

@@ -19,13 +19,19 @@ public class ElasticsearchContainerUtil {
 	public static final int ES_PORT = 9200;
 
 	public static ElasticsearchContainer spinUpEs() {
+		return spinUpEs(Version.CURRENT.toString());
+	}
+
+	public static ElasticsearchContainer spinUpEs(String esVersion) {
 		log.info("starting Elasticsearch container..");
 		ElasticsearchContainer container = new ElasticsearchContainer(
 				DockerImageName
 						.parse("docker.elastic.co/elasticsearch/elasticsearch")
-						.withTag(Version.CURRENT.toString()));
+						.withTag(esVersion));
 		container.addEnv("discovery.type", "single-node");
 		container.addEnv("ES_JAVA_OPTS", "-Xms1024m -Xmx1024m");
+		container.addEnv("xpack.security.enabled", "false");
+		container.addEnv("xpack.security.http.ssl.enabled", "false");
 		container.setWaitStrategy(new HttpWaitStrategy().forPort(ES_PORT));
 		container.withStartupTimeout(Duration.ofSeconds(60));
 		container.start();

indexer-service/src/test/java/de/cxp/ocs/elasticsearch/ElasticsearchIndexerTest.java

@@ -5,16 +5,15 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
+import java.util.Optional;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.core.List;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.testcontainers.elasticsearch.ElasticsearchContainer;
 
 import de.cxp.ocs.api.indexer.ImportSession;
@@ -29,7 +28,7 @@ public class ElasticsearchIndexerTest {
 
 	@BeforeAll
 	public static void startElasticsearch() {
-		container = ElasticsearchContainerUtil.spinUpEs();
+		container = ElasticsearchContainerUtil.spinUpEs(Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).orElse(Version.CURRENT.toString()));
 	}
 
 	@AfterAll

integration-tests/src/test/java/de/cxp/ocs/ITBulkIndexationWorks.java

@@ -45,6 +45,6 @@ public void testDefaultIndexation() throws Exception {
 	}
 
 	private void flushIndex() throws IOException {
-		getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush/synced"));
+		getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush?wait_if_ongoing=true"));
 	}
 }

integration-tests/src/test/java/de/cxp/ocs/ITPartialUpdates.java

@@ -119,7 +119,7 @@ private SearchResult doSimpleSearch(String query) {
 	}
 
 	private void flushIndex() throws IOException {
-		getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush/synced"));
+		getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush?wait_if_ongoing=true"));
 	}
 
 }

integration-tests/src/test/java/de/cxp/ocs/ITUpdateApi.java

@@ -32,13 +32,13 @@ public class ITUpdateApi {
 	// no indexation, since we want to test if the update API creates the necessary index
 	@AfterEach
 	public void deleteIndexes() {
-		Request deleteRequest = new Request("DELETE", "ocs-*-" + indexName + "*");
+		Request deleteRequest = new Request("DELETE", "ocs-1-" + indexName + "-en");
 		try {
 			Response response = getElasticsearchClient().performRequest(deleteRequest);
 			log.info("deleting index {} responded with {}", indexName, response);
 		}
 		catch (IOException e) {
-			log.error("failed to delete index {} after test", indexName);
+			log.error("failed to delete index {} after test: {}:{}", indexName, e.getClass(), e.getMessage());
 		}
 	}
 
@@ -93,7 +93,7 @@ private SearchResult doSimpleSearch(String query) {
 	}
 
 	private void flushIndex() throws IOException {
-		getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush/synced"));
+		getElasticsearchClient().performRequest(new Request("POST", indexName + "/_flush?wait_if_ongoing=true"));
 	}
 
 }

integration-tests/src/test/java/de/cxp/ocs/OCSStack.java

@@ -3,6 +3,7 @@
 import java.lang.reflect.Method;
 import java.net.URL;
 import java.util.Collections;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -71,8 +72,11 @@ public void beforeAll(ExtensionContext context) throws Exception {
 				}
 			}
 			else {
-				elasticsearch = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + Version.CURRENT.toString());
+				String esVersion = Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).orElse(Version.CURRENT.toString());
+				elasticsearch = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + esVersion);
 				elasticsearch.addEnv("discovery.type", "single-node");
+				elasticsearch.addEnv("xpack.security.enabled", "false");
+				elasticsearch.addEnv("xpack.security.http.ssl.enabled", "false");
 				elasticsearch.setExposedPorts(Collections.singletonList(ES_DEFAULT_PORT));
 				elasticsearch.withNetwork(Network.newNetwork());
 				elasticsearch.withNetworkAliases("elasticsearch");
@@ -105,7 +109,12 @@ private CompletableFuture<String> startIndexerService(CompletableFuture<HttpHost
 		else {
 			indexerService = new GenericContainer<>("commerceexperts/ocs-indexer-service:latest");
 			indexerService.addExposedPort(INDEXER_DEFAULT_PORT);
-			indexerService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,test");
+
+			String v8comp = "";
+			if (isWithEs8Compatibility()) {
+				v8comp = " -Docs.connection-configuration.use-compatibility-mode=true";
+			}
+			indexerService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,test" + v8comp);
 
 			bindFile(indexerService, "indexer.application-test.yml", "application-test.yml");
 
@@ -136,7 +145,11 @@ private CompletableFuture<String> startSearchService(CompletableFuture<HttpHost>
 			searchService.addExposedPort(SEARCH_DEFAULT_PORT);
 			// searchService.setCommand("-Dspring.cloud.config.enabled=false",
 			// "-Dspring.profiles.active=preset");
-			searchService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,trace-searches,test");
+			String v8comp = "";
+			if (isWithEs8Compatibility()) {
+				v8comp = " -Docs.connection-configuration.use-compatibility-mode=true";
+			}
+			searchService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m -Dspring.cloud.config.enabled=false -Dspring.profiles.active=default,preset,trace-searches,test" + v8comp);
 
 			bindFile(searchService, "searcher.application-test.yml", "application-test.yml");
 			bindFile(searchService, "querqy-test-rules.txt", "querqy-test-rules.txt");
@@ -176,7 +189,12 @@ private CompletableFuture<String> startSuggestService(CompletableFuture<HttpHost
 		else {
 			suggestService = new GenericContainer<>("commerceexperts/ocs-suggest-service:latest");
 			suggestService.addExposedPort(SUGGEST_DEFAULT_PORT);
-			suggestService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m");
+
+			String v8comp = "";
+			if (isWithEs8Compatibility()) {
+				v8comp = " -Delasticsearch.useCompatibilityMode=true";
+			}
+			suggestService.addEnv("JAVA_TOOL_OPTIONS", "-Xms265m -Xmx1024m" + v8comp);
 
 			String esAddr;
 			if (elasticsearch != null) {
@@ -202,6 +220,10 @@ private CompletableFuture<String> startSuggestService(CompletableFuture<HttpHost
 		return suggestServiceHost;
 	}
 
+	public static Boolean isWithEs8Compatibility() {
+		return Optional.ofNullable(System.getenv("ES_CONTAINER_VERSION")).map(v -> v.startsWith("8")).orElse(false);
+	}
+
 	public static RestClient getElasticsearchClient() {
 		assert isStarted.get() : "Stack not started yet!";
 		return esRestClient;

ocs-commons/pom.xml

@@ -12,7 +12,7 @@
 	</parent>
 
 	<artifactId>ocs-commons</artifactId>
-	<version>0.34.3</version>
+	<version>0.35.0</version>
 	<packaging>jar</packaging>
 
 	<description>Library that contains common models and utility for the open-commerce-search stack</description>

ocs-commons/src/main/java/de/cxp/ocs/config/ConnectionConfiguration.java

@@ -22,4 +22,10 @@ public class ConnectionConfiguration {
 	 * Example: "elastic:my$ecretPassw0rd"
 	 */
 	private String auth;
+
+	/**
+	 * If set to 'true' the rest-high-level-client version 7 can be used with elasticsearch version 8.
+	 * <a href="https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-compatibility.html">see Compatibility with Elasticsearch 8.x</a>
+	 */
+	private boolean useCompatibilityMode = false;
 }

ocs-commons/src/main/java/de/cxp/ocs/elasticsearch/ElasticSearchBuilder.java

@@ -1,22 +1,27 @@
 package de.cxp.ocs.elasticsearch;
 
-import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.RestHighLevelClientBuilder;
 
 public class ElasticSearchBuilder implements AutoCloseable {
 
 	private       RestHighLevelClient highLevelClient;
-	private final RestClientBuilder   restClientBuilder;
+	private final RestClient          restClient;
+	private final boolean             useCompatibilityMode;
 
-	public ElasticSearchBuilder(RestClientBuilder clientBuilder) {
-		restClientBuilder = clientBuilder;
+	public ElasticSearchBuilder(RestClient restClient, boolean useCompatibilityMode) {
+		this.restClient = restClient;
+		this.useCompatibilityMode = useCompatibilityMode;
 	}
 
 	public RestHighLevelClient getRestHLClient() {
 		if (highLevelClient == null) {
 			synchronized (this) {
 				if (highLevelClient == null) {
-					highLevelClient = new RestHighLevelClient(restClientBuilder);
+					highLevelClient = new RestHighLevelClientBuilder(restClient)
+							.setApiCompatibilityMode(useCompatibilityMode)
+							.build();
 				}
 			}
 		}

ocs-plugin-spi/pom.xml

@@ -10,7 +10,7 @@
 	</parent>
 
 	<artifactId>ocs-plugin-spi</artifactId>
-	<version>1.18.2</version>
+	<version>1.19.0</version>
 	<packaging>jar</packaging>
 
 	<description>Service Provider Interface for Plugins that could be optionally added to the services</description>