compileOnlyConfiguration = GradleUtils.getFiles(
+ final FileCollection runtime = GradleUtils.getFirstLevelModuleDependencyFiles(project, getRuntimeConfiguration(), reallyThirdParty);
+ // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin
+ final FileCollection compileOnly = GradleUtils.getFirstLevelModuleDependencyFiles(
project,
project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME),
reallyThirdParty
- ).getFiles();
- // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin
- if (compileOnlyConfiguration != null) {
- jars.removeAll(compileOnlyConfiguration);
- }
- return jars;
+ );
+ return runtime.minus(compileOnly).getFiles();
}
@TaskAction
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java b/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java
index 3352dda98ef66..49135b60dcdd9 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java
@@ -64,7 +64,7 @@
*
* This task is necessary because the built-in task {@link org.gradle.api.tasks.bundling.Tar} does not preserve symbolic links.
*/
-public class SymbolicLinkPreservingTar extends Tar {
+public abstract class SymbolicLinkPreservingTar extends Tar {
private long lastModifiedTimestamp = 0;
public void setLastModifiedTimestamp(long lastModifiedTimestamp) {
@@ -231,5 +231,4 @@ private long getModTime(final FileCopyDetails details) {
}
}
-
}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java
index c5035f3b082fe..8c4bbe6c2db42 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java
@@ -168,6 +168,8 @@ public void beforeStart() {
firstNode.setting("discovery.seed_hosts", LOCALHOST_ADDRESS_PREFIX + DEFAULT_TRANSPORT_PORT);
cluster.setPreserveDataDir(preserveData);
for (OpenSearchNode node : cluster.getNodes()) {
+ // TODO : remove this - this disables assertions
+ node.jvmArgs(" -da ");
if (node != firstNode) {
node.setHttpPort(String.valueOf(httpPort));
httpPort++;
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java b/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java
index 31e2e5346c751..55b8c0cd6fc86 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java
@@ -39,10 +39,8 @@
import org.gradle.api.UnknownTaskException;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
-import org.gradle.api.artifacts.LenientConfiguration;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
import org.gradle.api.file.FileCollection;
-import org.gradle.api.internal.artifacts.ivyservice.ResolvedFilesCollectingVisitor;
-import org.gradle.api.internal.artifacts.ivyservice.resolveengine.artifact.SelectedArtifactSet;
import org.gradle.api.plugins.JavaBasePlugin;
import org.gradle.api.plugins.JavaPluginExtension;
import org.gradle.api.provider.Provider;
@@ -58,16 +56,15 @@
import org.gradle.plugins.ide.eclipse.model.EclipseModel;
import org.gradle.plugins.ide.idea.model.IdeaModel;
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.MethodHandles;
-import java.lang.invoke.MethodType;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.function.Function;
+import java.util.stream.Collectors;
public abstract class GradleUtils {
@@ -254,21 +251,20 @@ public static String getProjectPathFromTask(String taskPath) {
return lastDelimiterIndex == 0 ? ":" : taskPath.substring(0, lastDelimiterIndex);
}
- public static FileCollection getFiles(Project project, Configuration cfg, Spec spec) {
- final LenientConfiguration configuration = cfg.getResolvedConfiguration().getLenientConfiguration();
- try {
- // Using reflection here to cover the pre 8.7 releases (since those have no such APIs), the
- // ResolverResults.LegacyResolverResults.LegacyVisitedArtifactSet::select(...) is not available
- // on older versions.
- final MethodHandle mh = MethodHandles.lookup()
- .findVirtual(configuration.getClass(), "select", MethodType.methodType(SelectedArtifactSet.class, Spec.class))
- .bindTo(configuration);
-
- final ResolvedFilesCollectingVisitor visitor = new ResolvedFilesCollectingVisitor();
- ((SelectedArtifactSet) mh.invoke(spec)).visitArtifacts(visitor, false);
- return project.files(visitor.getFiles());
- } catch (Throwable ex) {
- return project.files(configuration.getFiles(spec));
- }
+ public static FileCollection getFirstLevelModuleDependencyFiles(
+ Project project,
+ Configuration cfg,
+ Spec spec
+ ) {
+ final FileCollection files = cfg.getIncoming().artifactView(viewConfiguration -> {
+ final Set directDependencies = cfg.getResolvedConfiguration()
+ .getFirstLevelModuleDependencies()
+ .stream()
+ .flatMap(dep -> dep.getModuleArtifacts().stream().map(artifact -> artifact.getId().getComponentIdentifier()))
+ .collect(Collectors.toSet());
+ viewConfiguration.setLenient(true);
+ viewConfiguration.componentFilter(ci -> spec.isSatisfiedBy(ci) && directDependencies.contains(ci));
+ }).getFiles();
+ return project.files(files);
}
}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java
index 9d957a301dde4..db1d5d5c89bbb 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java
@@ -40,6 +40,9 @@
import org.gradle.api.execution.TaskActionListener;
import org.gradle.api.execution.TaskExecutionListener;
import org.gradle.api.tasks.TaskState;
+import org.gradle.process.ExecOperations;
+
+import javax.inject.Inject;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
@@ -81,6 +84,13 @@ static class VagrantSetupCheckerPlugin implements Plugin {
private static final Pattern VAGRANT_VERSION = Pattern.compile("Vagrant (\\d+\\.\\d+\\.\\d+)");
private static final Pattern VIRTUAL_BOX_VERSION = Pattern.compile("(\\d+\\.\\d+)");
+ private final ExecOperations execOperations;
+
+ @Inject
+ public VagrantSetupCheckerPlugin(ExecOperations execOperations) {
+ this.execOperations = execOperations;
+ }
+
@Override
public void apply(Project project) {
if (project != project.getRootProject()) {
@@ -98,7 +108,7 @@ public void apply(Project project) {
void checkVersion(Project project, String tool, Pattern versionRegex, int... minVersion) {
ByteArrayOutputStream pipe = new ByteArrayOutputStream();
- project.exec(spec -> {
+ execOperations.exec(spec -> {
spec.setCommandLine(tool, "--version");
spec.setStandardOutput(pipe);
});
diff --git a/buildSrc/src/test/java/org/opensearch/gradle/precommit/DependencyLicensesTaskTests.java b/buildSrc/src/test/java/org/opensearch/gradle/precommit/DependencyLicensesTaskTests.java
index 28513710470af..d0c3d1e56a4d7 100644
--- a/buildSrc/src/test/java/org/opensearch/gradle/precommit/DependencyLicensesTaskTests.java
+++ b/buildSrc/src/test/java/org/opensearch/gradle/precommit/DependencyLicensesTaskTests.java
@@ -35,7 +35,6 @@
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
-import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.JavaPlugin;
@@ -298,10 +297,10 @@ private Project createProject() {
Project project = ProjectBuilder.builder().build();
project.getPlugins().apply(JavaPlugin.class);
- Configuration compileClasspath = project.getConfigurations().getByName("compileClasspath");
- Configuration someCompileConfiguration = project.getConfigurations().create("someCompileConfiguration");
// Declare a configuration that is going to resolve the compile classpath of the application
- project.getConfigurations().add(compileClasspath.extendsFrom(someCompileConfiguration));
+ project.getConfigurations()
+ .getByName("compileClasspath")
+ .extendsFrom(project.getConfigurations().create("someCompileConfiguration"));
return project;
}
diff --git a/buildSrc/src/test/java/org/opensearch/gradle/precommit/UpdateShasTaskTests.java b/buildSrc/src/test/java/org/opensearch/gradle/precommit/UpdateShasTaskTests.java
index 15d6d6cd4c31c..3ad41460aebf0 100644
--- a/buildSrc/src/test/java/org/opensearch/gradle/precommit/UpdateShasTaskTests.java
+++ b/buildSrc/src/test/java/org/opensearch/gradle/precommit/UpdateShasTaskTests.java
@@ -36,7 +36,6 @@
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
-import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.JavaPlugin;
@@ -123,10 +122,10 @@ private Project createProject() {
Project project = ProjectBuilder.builder().build();
project.getPlugins().apply(JavaPlugin.class);
- Configuration compileClasspath = project.getConfigurations().getByName("compileClasspath");
- Configuration someCompileConfiguration = project.getConfigurations().create("someCompileConfiguration");
// Declare a configuration that is going to resolve the compile classpath of the application
- project.getConfigurations().add(compileClasspath.extendsFrom(someCompileConfiguration));
+ project.getConfigurations()
+ .getByName("compileClasspath")
+ .extendsFrom(project.getConfigurations().create("someCompileConfiguration"));
return project;
}
diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
index 3a44aa603378c..00d5202168800 100644
--- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
+++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
@@ -17,7 +17,7 @@ repositories {
}
dependencies {
- implementation "org.apache.logging.log4j:log4j-core:2.25.1"
+ implementation "org.apache.logging.log4j:log4j-core:2.25.2"
}
["0.0.1", "0.0.2"].forEach { v ->
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 3ac58c3cfc095..8ab641b598eb4 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,2 +1,2 @@
# Please use ../gradle/libs.versions.toml for dependency management
-opensearch = 3.2.0
+opensearch = 3.3.0
diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index 22fb38ded3bde..ed5eedb65e140 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -105,9 +105,6 @@ testingConventions {
thirdPartyAudit {
ignoreMissingClasses(
'org.conscrypt.Conscrypt',
- 'org.slf4j.impl.StaticLoggerBinder',
- 'org.slf4j.impl.StaticMDCBinder',
- 'org.slf4j.impl.StaticMarkerBinder',
//commons-logging optional dependencies
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
diff --git a/client/rest/licenses/bc-fips-2.0.0.jar.sha1 b/client/rest/licenses/bc-fips-2.0.0.jar.sha1
deleted file mode 100644
index 79f0e3e9930bb..0000000000000
--- a/client/rest/licenses/bc-fips-2.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab
\ No newline at end of file
diff --git a/client/rest/licenses/bc-fips-2.1.1.jar.sha1 b/client/rest/licenses/bc-fips-2.1.1.jar.sha1
new file mode 100644
index 0000000000000..831a41da72aa5
--- /dev/null
+++ b/client/rest/licenses/bc-fips-2.1.1.jar.sha1
@@ -0,0 +1 @@
+34c72d0367d41672883283933ebec24843570bf5
\ No newline at end of file
diff --git a/client/rest/licenses/bctls-fips-2.0.20.jar.sha1 b/client/rest/licenses/bctls-fips-2.0.20.jar.sha1
deleted file mode 100644
index 66cd82b49b537..0000000000000
--- a/client/rest/licenses/bctls-fips-2.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1138f7896e0d1bb0d924bc868ed2dfda4f69470e
\ No newline at end of file
diff --git a/client/rest/licenses/bctls-fips-2.1.20.jar.sha1 b/client/rest/licenses/bctls-fips-2.1.20.jar.sha1
new file mode 100644
index 0000000000000..7266ec5abf10a
--- /dev/null
+++ b/client/rest/licenses/bctls-fips-2.1.20.jar.sha1
@@ -0,0 +1 @@
+9c0632a6c5ca09a86434cf5e02e72c221e1c930f
\ No newline at end of file
diff --git a/client/rest/licenses/bcutil-fips-2.0.3.jar.sha1 b/client/rest/licenses/bcutil-fips-2.0.3.jar.sha1
deleted file mode 100644
index d553536576656..0000000000000
--- a/client/rest/licenses/bcutil-fips-2.0.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a1857cd639295b10cc90e6d31ecbc523cdafcc19
\ No newline at end of file
diff --git a/client/rest/licenses/bcutil-fips-2.1.4.jar.sha1 b/client/rest/licenses/bcutil-fips-2.1.4.jar.sha1
new file mode 100644
index 0000000000000..73b19722430fb
--- /dev/null
+++ b/client/rest/licenses/bcutil-fips-2.1.4.jar.sha1
@@ -0,0 +1 @@
+1d37b7a28560684f5b8e4fd65478c9130d4015d0
\ No newline at end of file
diff --git a/client/rest/licenses/commons-codec-1.16.1.jar.sha1 b/client/rest/licenses/commons-codec-1.16.1.jar.sha1
deleted file mode 100644
index 6b8803089c6d7..0000000000000
--- a/client/rest/licenses/commons-codec-1.16.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/client/rest/licenses/commons-codec-1.18.0.jar.sha1 b/client/rest/licenses/commons-codec-1.18.0.jar.sha1
new file mode 100644
index 0000000000000..01a6a8f446302
--- /dev/null
+++ b/client/rest/licenses/commons-codec-1.18.0.jar.sha1
@@ -0,0 +1 @@
+ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f
\ No newline at end of file
diff --git a/client/rest/licenses/slf4j-api-1.7.36.jar.sha1 b/client/rest/licenses/slf4j-api-1.7.36.jar.sha1
deleted file mode 100644
index 77b9917528382..0000000000000
--- a/client/rest/licenses/slf4j-api-1.7.36.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c62681a2f655b49963a5983b8b0950a6120ae14
\ No newline at end of file
diff --git a/client/rest/licenses/slf4j-api-2.0.17.jar.sha1 b/client/rest/licenses/slf4j-api-2.0.17.jar.sha1
new file mode 100644
index 0000000000000..435f6c13a28b6
--- /dev/null
+++ b/client/rest/licenses/slf4j-api-2.0.17.jar.sha1
@@ -0,0 +1 @@
+d9e58ac9c7779ba3bf8142aff6c830617a7fe60f
\ No newline at end of file
diff --git a/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1 b/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1
deleted file mode 100644
index 6b8803089c6d7..0000000000000
--- a/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/client/sniffer/licenses/commons-codec-1.18.0.jar.sha1 b/client/sniffer/licenses/commons-codec-1.18.0.jar.sha1
new file mode 100644
index 0000000000000..01a6a8f446302
--- /dev/null
+++ b/client/sniffer/licenses/commons-codec-1.18.0.jar.sha1
@@ -0,0 +1 @@
+ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f
\ No newline at end of file
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index cc371a3275570..ecc2d2c5c5766 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -132,8 +132,8 @@ project.ext {
}
void addCopyDockerContextTask(Architecture architecture, DockerBase base) {
- if (base != DockerBase.CENTOS) {
- throw new GradleException("The only allowed docker base image for builds is CENTOS")
+ if (base != DockerBase.ALMALINUX) {
+ throw new GradleException("The only allowed docker base image for builds is ALMALINUX")
}
tasks.register(taskName("copy", architecture, base, "DockerContext"), Sync) {
@@ -181,8 +181,8 @@ opensearch_distributions {
tasks.named("preProcessFixture").configure {
dependsOn opensearch_distributions.docker
// always run the task, otherwise the folders won't be created
- outputs.upToDateWhen {
- false
+ outputs.upToDateWhen {
+ false
}
doLast {
// tests expect to have an empty repo
@@ -208,8 +208,8 @@ tasks.named("check").configure {
}
void addBuildDockerImage(Architecture architecture, DockerBase base) {
- if (base != DockerBase.CENTOS) {
- throw new GradleException("The only allowed docker base image for builds is CENTOS")
+ if (base != DockerBase.ALMALINUX) {
+ throw new GradleException("The only allowed docker base image for builds is ALMALINUX")
}
final TaskProvider buildDockerImageTask =
@@ -232,9 +232,9 @@ void addBuildDockerImage(Architecture architecture, DockerBase base) {
}
for (final Architecture architecture : Architecture.values()) {
- // We only create Docker images for the distribution on CentOS.
+ // We only create Docker images for the distribution on AlmaLinux.
for (final DockerBase base : DockerBase.values()) {
- if (base == DockerBase.CENTOS) {
+ if (base == DockerBase.ALMALINUX) {
addCopyDockerContextTask(architecture, base)
addBuildDockerImage(architecture, base)
}
@@ -257,7 +257,7 @@ subprojects { Project subProject ->
apply plugin: 'distribution'
final Architecture architecture = subProject.name.contains('arm64-') ? Architecture.ARM64 : Architecture.X64
- final DockerBase base = DockerBase.CENTOS
+ final DockerBase base = DockerBase.ALMALINUX
final String arch = architecture == Architecture.ARM64 ? '-arm64' : ''
final String extension = 'docker.tar'
diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle
index a5bea2935b3ea..3426df47780dc 100644
--- a/distribution/docker/docker-build-context/build.gradle
+++ b/distribution/docker/docker-build-context/build.gradle
@@ -19,7 +19,7 @@ tasks.register("buildDockerBuildContext", Tar) {
archiveClassifier = "docker-build-context"
archiveBaseName = "opensearch"
// Non-local builds don't need to specify an architecture.
- with dockerBuildContext(null, DockerBase.CENTOS, false)
+ with dockerBuildContext(null, DockerBase.ALMALINUX, false)
}
tasks.named("assemble").configure { dependsOn "buildDockerBuildContext" }
diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile
index c980217b0b8dc..fc2b66aaf7d53 100644
--- a/distribution/docker/src/docker/Dockerfile
+++ b/distribution/docker/src/docker/Dockerfile
@@ -63,16 +63,12 @@ FROM ${base_image}
ENV OPENSEARCH_CONTAINER true
-RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \\
- sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* && \\
- for iter in {1..10}; do \\
- ${package_manager} update --setopt=tsflags=nodocs -y && \\
- ${package_manager} install --setopt=tsflags=nodocs -y \\
- nc shadow-utils zip unzip && \\
- ${package_manager} clean all && exit_code=0 && break || exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && \\
- sleep 10; \\
- done; \\
- (exit \$exit_code)
+RUN set -e \\
+ && dnf -y update \\
+ && dnf -y install --setopt=tsflags=nodocs \\
+ nmap-ncat shadow-utils zip unzip \\
+ && dnf clean all \\
+ && rm -rf /var/cache/dnf
RUN groupadd -g 1000 opensearch && \\
adduser -u 1000 -g 1000 -G 0 -d /usr/share/opensearch opensearch && \\
diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index 194c683da5ec7..d72ca53401b03 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -63,7 +63,7 @@ import java.util.regex.Pattern
*/
plugins {
- id "com.netflix.nebula.ospackage-base" version "12.0.0"
+ id "com.netflix.nebula.ospackage-base" version "12.1.1"
}
void addProcessFilesTask(String type, boolean jdk) {
diff --git a/distribution/packages/src/common/systemd/opensearch.service b/distribution/packages/src/common/systemd/opensearch.service
index 760fc39723f65..a27608c629410 100644
--- a/distribution/packages/src/common/systemd/opensearch.service
+++ b/distribution/packages/src/common/systemd/opensearch.service
@@ -96,9 +96,8 @@ ProtectControlGroups=yes
LockPersonality=yes
-# System call filtering
# System call filterings which restricts which system calls a process can make
-# @ means allowed
+# @ means predefined sets
# ~ means not allowed
# These syscalls are related to mmap which is needed for OpenSearch Services
SystemCallFilter=seccomp mincore
@@ -149,6 +148,7 @@ ReadOnlyPaths=/proc/self/mountinfo /proc/diskstats
## Allow read access to control group stats
ReadOnlyPaths=/proc/self/cgroup /sys/fs/cgroup/cpu /sys/fs/cgroup/cpu/-
ReadOnlyPaths=/sys/fs/cgroup/cpuacct /sys/fs/cgroup/cpuacct/- /sys/fs/cgroup/memory /sys/fs/cgroup/memory/-
+ReadOnlyPaths=/sys/fs/cgroup/system.slice/-
RestrictNamespaces=true
@@ -177,4 +177,4 @@ ProtectClock=true
[Install]
WantedBy=multi-user.target
-# Built for ${project.name}-${project.version} (${project.name})
\ No newline at end of file
+# Built for ${project.name}-${project.version} (${project.name})
diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index e083f07edabc8..731c934aee8be 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -86,3 +86,21 @@ ${error.file}
21-:-javaagent:agent/opensearch-agent.jar
21-:--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED
+
+# Heap size settings
+# -Xms32g
+# -Xmx32g
+
+# Enable native memory tracking
+-XX:NativeMemoryTracking=summary
+
+# Allow jcmd to attach to the process (required for 'jcmd VM.native_memory' commands)
+-XX:+UnlockDiagnosticVMOptions
+
+# Enabling debug logs for Allocators in Arrow
+-Darrow.memory.debug.allocator=true
+
+# For cases with high memory-mapped file counts, a lower value can improve stability and
+# prevent issues like "leaked" maps or performance degradation. A value of 1 effectively
+# disables the shared Arena pooling and uses a confined Arena for each MMapDirectory
+-Dorg.apache.lucene.store.MMapDirectory.sharedArenaMaxPermits=1
diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
index 29070a59cb5df..1b0fe139fbaab 100644
--- a/distribution/src/config/opensearch.yml
+++ b/distribution/src/config/opensearch.yml
@@ -121,3 +121,15 @@ ${path.logs}
# Once there is no observed impact on performance, this feature flag can be removed.
#
#opensearch.experimental.optimization.datetime_formatter_caching.enabled: false
+#
+#
+# Limits the memory pool for datafusion which it uses for query execution.
+#datafusion.search.memory_pool: 1GB
+#node.attr.remote_store.segment.repository: my-repo-1
+#node.attr.remote_store.translog.repository: my-repo-1
+#node.attr.remote_store.state.repository: my-repo-1
+#cluster.remote_store.state.enabled: true
+#node.attr.remote_store.repository.my-repo-1.type: fs
+#path.repo: /tmp/remote-store-repo
+#node.attr.remote_store.repository.my-repo-1.settings.location: /tmp/remote-store-repo
+
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
index 8beb17bb8bf9a..adad705e8cf7f 100644
--- a/distribution/tools/plugin-cli/build.gradle
+++ b/distribution/tools/plugin-cli/build.gradle
@@ -45,8 +45,10 @@ dependencies {
testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
transitive = false
}
+ api "org.apache.commons:commons-compress:${versions.commonscompress}"
api "commons-io:commons-io:${versions.commonsio}"
- implementation "org.apache.commons:commons-compress:${versions.commonscompress}"
+ api "org.apache.commons:commons-lang3:${versions.commonslang}"
+ api "commons-codec:commons-codec:${versions.commonscodec}"
}
tasks.named("dependencyLicenses").configure {
@@ -60,7 +62,9 @@ test {
}
thirdPartyAudit.ignoreMissingClasses(
+ // org.brotli:dec is an optional dependency of commons-compress
'org.brotli.dec.BrotliInputStream',
+ // org.ow2.asm:asm is an optional dependency of commons-compress
'org.objectweb.asm.AnnotationVisitor',
'org.objectweb.asm.Attribute',
'org.objectweb.asm.ClassReader',
@@ -69,6 +73,7 @@ thirdPartyAudit.ignoreMissingClasses(
'org.objectweb.asm.Label',
'org.objectweb.asm.MethodVisitor',
'org.objectweb.asm.Type',
+ // org.tukaani:xz is an optional dependency of commons-compress
'org.tukaani.xz.DeltaOptions',
'org.tukaani.xz.FilterOptions',
'org.tukaani.xz.LZMA2InputStream',
@@ -78,8 +83,5 @@ thirdPartyAudit.ignoreMissingClasses(
'org.tukaani.xz.MemoryLimitException',
'org.tukaani.xz.UnsupportedOptionsException',
'org.tukaani.xz.XZ',
- 'org.tukaani.xz.XZOutputStream',
- 'org.apache.commons.codec.digest.PureJavaCrc32C',
- 'org.apache.commons.codec.digest.XXHash32',
- 'org.apache.commons.lang3.reflect.FieldUtils'
+ 'org.tukaani.xz.XZOutputStream'
)
diff --git a/distribution/tools/plugin-cli/licenses/bc-fips-2.0.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/bc-fips-2.0.0.jar.sha1
deleted file mode 100644
index 79f0e3e9930bb..0000000000000
--- a/distribution/tools/plugin-cli/licenses/bc-fips-2.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/bc-fips-2.1.1.jar.sha1 b/distribution/tools/plugin-cli/licenses/bc-fips-2.1.1.jar.sha1
new file mode 100644
index 0000000000000..831a41da72aa5
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/bc-fips-2.1.1.jar.sha1
@@ -0,0 +1 @@
+34c72d0367d41672883283933ebec24843570bf5
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1 b/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1
deleted file mode 100644
index 39805c3a32614..0000000000000
--- a/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-19f38a0d8048e87039b1bb6c1ba4d2b104891d04
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/bcpg-fips-2.1.11.jar.sha1 b/distribution/tools/plugin-cli/licenses/bcpg-fips-2.1.11.jar.sha1
new file mode 100644
index 0000000000000..09bc4760767a1
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/bcpg-fips-2.1.11.jar.sha1
@@ -0,0 +1 @@
+727e087a843f3a5a8143e4f3a7518c8c3517df18
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/commons-codec-1.18.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-codec-1.18.0.jar.sha1
new file mode 100644
index 0000000000000..01a6a8f446302
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-codec-1.18.0.jar.sha1
@@ -0,0 +1 @@
+ee45d1cf6ec2cc2b809ff04b4dc7aec858e0df8f
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/licenses/commons-lang-LICENSE.txt b/distribution/tools/plugin-cli/licenses/commons-codec-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure-classic/licenses/commons-lang-LICENSE.txt
rename to distribution/tools/plugin-cli/licenses/commons-codec-LICENSE.txt
diff --git a/distribution/tools/plugin-cli/licenses/commons-codec-NOTICE.txt b/distribution/tools/plugin-cli/licenses/commons-codec-NOTICE.txt
new file mode 100644
index 0000000000000..1da9af50f6008
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-codec-NOTICE.txt
@@ -0,0 +1,17 @@
+Apache Commons Codec
+Copyright 2002-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+===============================================================================
+
+The content of package org.apache.commons.codec.language.bm has been translated
+from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1
deleted file mode 100644
index 912bda85de18a..0000000000000
--- a/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-44331c1130c370e726a2e1a3e6fba6d2558ef04a
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.28.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.28.0.jar.sha1
new file mode 100644
index 0000000000000..5edae62aeeb5d
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-compress-1.28.0.jar.sha1
@@ -0,0 +1 @@
+e482f2c7a88dac3c497e96aa420b6a769f59c8d7
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/commons-lang3-3.18.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-lang3-3.18.0.jar.sha1
new file mode 100644
index 0000000000000..a1a6598bd4f1b
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-lang3-3.18.0.jar.sha1
@@ -0,0 +1 @@
+fb14946f0e39748a6571de0635acbe44e7885491
\ No newline at end of file
diff --git a/plugins/identity-shiro/licenses/commons-lang-LICENSE.txt b/distribution/tools/plugin-cli/licenses/commons-lang3-LICENSE.txt
similarity index 100%
rename from plugins/identity-shiro/licenses/commons-lang-LICENSE.txt
rename to distribution/tools/plugin-cli/licenses/commons-lang3-LICENSE.txt
diff --git a/distribution/tools/plugin-cli/licenses/commons-lang3-NOTICE.txt b/distribution/tools/plugin-cli/licenses/commons-lang3-NOTICE.txt
new file mode 100644
index 0000000000000..13a3140897472
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-lang3-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Commons Lang
+Copyright 2001-2019 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java
index ea76e051d253e..4f69157545a22 100644
--- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java
+++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java
@@ -399,7 +399,7 @@ private String getMavenUrl(Terminal terminal, String[] coordinates, String platf
@SuppressForbidden(reason = "Make HEAD request using URLConnection.connect()")
boolean urlExists(Terminal terminal, String urlString) throws IOException {
terminal.println(VERBOSE, "Checking if url exists: " + urlString);
- URL url = new URL(urlString);
+ URL url = URI.create(urlString).toURL();
assert "https".equals(url.getProtocol()) : "Use of https protocol is required";
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -427,7 +427,7 @@ private List checkMisspelledPlugin(String pluginId) {
@SuppressForbidden(reason = "We use getInputStream to download plugins")
Path downloadZip(Terminal terminal, String urlString, Path tmpDir, boolean isBatch) throws IOException {
terminal.println(VERBOSE, "Retrieving zip from " + urlString);
- URL url = new URL(urlString);
+ URL url = URI.create(urlString).toURL();
Path zip = Files.createTempFile(tmpDir, null, ".zip");
URLConnection urlConnection = url.openConnection();
urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -684,7 +684,7 @@ InputStream getPublicKey() {
*/
// pkg private for tests
URL openUrl(String urlString) throws IOException {
- URL checksumUrl = new URL(urlString);
+ URL checksumUrl = URI.create(urlString).toURL();
HttpURLConnection connection = (HttpURLConnection) checksumUrl.openConnection();
if (connection.getResponseCode() == 404) {
return null;
@@ -960,16 +960,13 @@ private void installConfig(PluginInfo info, Path tmpConfigDir, Path destConfigDi
try (DirectoryStream stream = Files.newDirectoryStream(tmpConfigDir)) {
for (Path srcFile : stream) {
- if (Files.isDirectory(srcFile)) {
- throw new UserException(PLUGIN_MALFORMED, "Directories not allowed in config dir for plugin " + info.getName());
- }
-
Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile));
if (Files.exists(destFile) == false) {
- Files.copy(srcFile, destFile);
- setFileAttributes(destFile, CONFIG_FILES_PERMS);
- if (destConfigDirAttributes != null) {
- setOwnerGroup(destFile, destConfigDirAttributes);
+ if (Files.isDirectory(srcFile)) {
+ copyWithPermissions(srcFile, destFile, CONFIG_DIR_PERMS, destConfigDirAttributes);
+ copyDirectoryRecursively(srcFile, destFile, destConfigDirAttributes);
+ } else {
+ copyWithPermissions(srcFile, destFile, CONFIG_FILES_PERMS, destConfigDirAttributes);
}
}
}
@@ -995,6 +992,42 @@ private static void setFileAttributes(final Path path, final Set permissions,
+ PosixFileAttributes attributes
+ ) throws IOException {
+ Files.copy(srcFile, destFile);
+ setFileAttributes(destFile, permissions);
+ if (attributes != null) {
+ setOwnerGroup(destFile, attributes);
+ }
+ }
+
+ /**
+ * Recursively copies directory contents from source to destination.
+ */
+ private static void copyDirectoryRecursively(Path srcDir, Path destDir, PosixFileAttributes destConfigDirAttributes)
+ throws IOException {
+ try (DirectoryStream stream = Files.newDirectoryStream(srcDir)) {
+ for (Path srcFile : stream) {
+ Path destFile = destDir.resolve(srcDir.relativize(srcFile));
+ if (Files.exists(destFile) == false) {
+ if (Files.isDirectory(srcFile)) {
+ copyWithPermissions(srcFile, destFile, CONFIG_DIR_PERMS, destConfigDirAttributes);
+ copyDirectoryRecursively(srcFile, destFile, destConfigDirAttributes);
+ } else {
+ copyWithPermissions(srcFile, destFile, CONFIG_FILES_PERMS, destConfigDirAttributes);
+ }
+ }
+ }
+ }
+ }
+
private final List pathsToDeleteOnShutdown = new ArrayList<>();
@Override
diff --git a/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java
index 70cccc94a26f9..e2cd36f7bb8b9 100644
--- a/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java
+++ b/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java
@@ -433,8 +433,6 @@ void assertConfigAndBin(String name, Path original, Environment env) throws IOEx
try (DirectoryStream stream = Files.newDirectoryStream(configDir)) {
for (Path file : stream) {
- assertFalse("not a dir", Files.isDirectory(file));
-
if (isPosix) {
PosixFileAttributes attributes = Files.readAttributes(file, PosixFileAttributes.class);
if (user != null) {
@@ -526,7 +524,7 @@ public void testSpaceInUrl() throws Exception {
Path pluginDir = createPluginDir(temp);
String pluginZip = createPluginUrl("fake", pluginDir);
Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
- try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip))) {
+ try (InputStream in = FileSystemUtils.openFileURLStream(URI.create(pluginZip).toURL())) {
Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
}
installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
@@ -536,8 +534,8 @@ public void testSpaceInUrl() throws Exception {
public void testMalformedUrlNotMaven() throws Exception {
Tuple env = createEnv(fs, temp);
// has two colons, so it appears similar to maven coordinates
- MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1()));
- assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> installPlugin("://host:1234", env.v1()));
+ assertThat(e.getMessage(), startsWith("Expected scheme name"));
}
public void testFileNotMaven() throws Exception {
@@ -803,9 +801,14 @@ public void testConfigContainsDir() throws Exception {
Files.createDirectories(dirInConfigDir);
Files.createFile(dirInConfigDir.resolve("myconfig.yml"));
String pluginZip = createPluginUrl("fake", pluginDir);
- UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
- assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in config dir for plugin"));
- assertInstallCleaned(env.v2());
+ installPlugin(pluginZip, env.v1());
+ assertPlugin("fake", pluginDir, env.v2());
+
+ // Verify the directory and file were installed
+ Path installedConfigDir = env.v2().configDir().resolve("fake").resolve("foo");
+ assertTrue(Files.exists(installedConfigDir));
+ assertTrue(Files.isDirectory(installedConfigDir));
+ assertTrue(Files.exists(installedConfigDir.resolve("myconfig.yml")));
}
public void testMissingDescriptor() throws Exception {
diff --git a/gradle/code-coverage.gradle b/gradle/code-coverage.gradle
index fe7a68f0d3483..11a8a1253e1d1 100644
--- a/gradle/code-coverage.gradle
+++ b/gradle/code-coverage.gradle
@@ -52,6 +52,10 @@ allprojects {
executionDataFiles.add("$buildDir/jacoco/javaRestTest.exec")
sourceSetsList.add(sourceSets.javaRestTest)
}
+ if (tasks.findByName('yamlRestTest')) {
+ executionDataFiles.add("$buildDir/jacoco/yamlRestTest.exec")
+ sourceSetsList.add(sourceSets.yamlRestTest)
+ }
if (!executionDataFiles.isEmpty()) {
executionData.setFrom(files(executionDataFiles).filter { it.exists() })
sourceSets(*sourceSetsList)
@@ -59,7 +63,8 @@ allprojects {
onlyIf {
file("$buildDir/jacoco/test.exec").exists() ||
file("$buildDir/jacoco/internalClusterTest.exec").exists() ||
- file("$buildDir/jacoco/javaRestTest.exec").exists()
+ file("$buildDir/jacoco/javaRestTest.exec").exists() ||
+ file("$buildDir/jacoco/yamlRestTest.exec").exists()
}
}
}
@@ -71,9 +76,15 @@ if (System.getProperty("tests.coverage")) {
testCodeCoverageReport(JacocoCoverageReport) {
testSuiteName = "test"
}
+ testCodeCoverageReportInternalClusterTest(JacocoCoverageReport) {
+ testSuiteName = "internalClusterTest"
+ }
testCodeCoverageReportJavaRestTest(JacocoCoverageReport) {
testSuiteName = "javaRestTest"
}
+ testCodeCoverageReportYamlRestTest(JacocoCoverageReport) {
+ testSuiteName = "yamlRestTest"
+ }
}
}
@@ -81,7 +92,9 @@ if (System.getProperty("tests.coverage")) {
project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure {
dependsOn(
tasks.named('testCodeCoverageReport', JacocoReport),
- tasks.named('testCodeCoverageReportJavaRestTest', JacocoReport)
+ tasks.named('testCodeCoverageReportInternalClusterTest', JacocoReport),
+ tasks.named('testCodeCoverageReportJavaRestTest', JacocoReport),
+ tasks.named('testCodeCoverageReportYamlRestTest', JacocoReport)
)
}
}
diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle
index 45d63fd43e875..958afae9dcad7 100644
--- a/gradle/formatting.gradle
+++ b/gradle/formatting.gradle
@@ -81,8 +81,7 @@ allprojects {
'',
'\\#java|\\#org.opensearch|\\#org.hamcrest|\\#'
)
-
- eclipse().withP2Mirrors(Map.of("https://download.eclipse.org/", "https://mirror.umd.edu/eclipse/")).configFile rootProject.file('buildSrc/formatterConfig.xml')
+ eclipse().configFile rootProject.file('buildSrc/formatterConfig.xml')
trimTrailingWhitespace()
endWithNewline()
diff --git a/gradle/ide.gradle b/gradle/ide.gradle
index 50b0ec7e7ad14..548bc2a0aa511 100644
--- a/gradle/ide.gradle
+++ b/gradle/ide.gradle
@@ -12,6 +12,8 @@
import org.opensearch.gradle.info.BuildParams
import org.jetbrains.gradle.ext.Remote
import org.jetbrains.gradle.ext.JUnit
+import groovy.xml.XmlNodePrinter
+import groovy.xml.XmlParser
buildscript {
repositories {
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 2a56f138f499d..3457f04e0f87e 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -1,6 +1,6 @@
[versions]
-opensearch = "3.2.0"
-lucene = "10.2.2"
+opensearch = "3.3.0"
+lucene = "10.3.1"
bundled_jdk_vendor = "adoptium"
bundled_jdk = "24.0.2+12"
@@ -14,31 +14,33 @@ snakeyaml = "2.1"
icu4j = "77.1"
supercsv = "2.4.0"
log4j = "2.21.0"
-slf4j = "1.7.36"
+slf4j = "2.0.17"
asm = "9.7"
jettison = "1.5.4"
woodstox = "6.4.0"
kotlin = "1.7.10"
antlr4 = "4.13.1"
guava = "33.2.1-jre"
-protobuf = "3.25.5"
+gson = "2.13.2"
+opensearchprotobufs = "0.19.0"
+protobuf = "3.25.8"
jakarta_annotation = "1.3.5"
google_http_client = "1.44.1"
google_auth = "1.29.0"
tdigest = "3.3"
hdrhistogram = "2.2.2"
-grpc = "1.68.2"
+grpc = "1.75.0"
json_smart = "2.5.2"
# when updating the JNA version, also update the version in buildSrc/build.gradle
jna = "5.16.0"
-netty = "4.1.121.Final"
+netty = "4.1.125.Final"
joda = "2.12.7"
roaringbitmap = "1.3.0"
# project reactor
-reactor_netty = "1.2.5"
+reactor_netty = "1.2.9"
reactor = "3.7.5"
# client dependencies
@@ -48,12 +50,13 @@ httpclient = "4.5.14"
httpcore = "4.4.16"
httpasyncclient = "4.1.5"
commonslogging = "1.2"
-commonscodec = "1.16.1"
-commonslang = "3.14.0"
-commonscompress = "1.26.1"
+commonscodec = "1.18.0"
+commonslang = "3.18.0"
+commonscompress = "1.28.0"
commonsio = "2.16.0"
+commonscollections4 = "4.5.0"
# plugin dependencies
-aws = "2.30.31"
+aws = "2.32.29"
awscrt = "0.35.0"
reactivestreams = "1.0.4"
hadoop3 = "3.3.6"
@@ -61,11 +64,11 @@ hadoop3 = "3.3.6"
# when updating this version, you need to ensure compatibility with:
# - plugins/ingest-attachment (transitive dependency, check the upstream POM)
# - distribution/tools/plugin-cli
-bouncycastle_jce = "2.0.0"
-bouncycastle_tls = "2.0.20"
-bouncycastle_pkix = "2.0.8"
-bouncycastle_pg = "2.0.11"
-bouncycastle_util = "2.0.3"
+bouncycastle_jce = "2.1.1"
+bouncycastle_tls = "2.1.20"
+bouncycastle_pkix = "2.1.9"
+bouncycastle_pg = "2.1.11"
+bouncycastle_util = "2.1.4"
password4j = "1.8.3"
# test dependencies
@@ -74,7 +77,7 @@ junit = "4.13.2"
hamcrest = "2.1"
mockito = "5.16.1"
objenesis = "3.3"
-bytebuddy = "1.17.5"
+bytebuddy = "1.17.7"
# benchmark dependencies
jmh = "1.35"
@@ -87,8 +90,8 @@ jzlib = "1.1.3"
resteasy = "6.2.4.Final"
# opentelemetry dependencies
-opentelemetry = "1.46.0"
-opentelemetrysemconv = "1.29.0-alpha"
+opentelemetry = "1.53.0"
+opentelemetrysemconv = "1.34.0"
# arrow dependencies
arrow = "18.1.0"
@@ -106,6 +109,7 @@ bouncycastle-tls = { group = "org.bouncycastle", name = "bctls-fips", version.re
bouncycastle-pkix = { group = "org.bouncycastle", name = "bcpkix-fips", version.ref = "bouncycastle_pkix"}
bouncycastle-pg = { group = "org.bouncycastle", name = "bcpg-fips", version.ref = "bouncycastle_pg"}
bouncycastle-util = { group = "org.bouncycastle", name = "bcutil-fips", version.ref = "bouncycastle_util"}
+gson = { group = "com.google.code.gson", name = "gson", version.ref = "gson" }
hamcrest = { group = "org.hamcrest", name = "hamcrest", version.ref = "hamcrest" }
hdrhistogram = { group = "org.hdrhistogram", name = "HdrHistogram", version.ref = "hdrhistogram" }
jackson-annotation = { group = "com.fasterxml.jackson.core", name = "jackson-annotations", version.ref = "jackson" }
diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle
index 5f3ef5c0b7d48..6cc6d9c092a73 100644
--- a/gradle/missing-javadoc.gradle
+++ b/gradle/missing-javadoc.gradle
@@ -160,7 +160,12 @@ configure([
project(":test:fixtures:hdfs-fixture"),
project(":test:fixtures:s3-fixture"),
project(":test:framework"),
- project(":test:logger-usage")
+ project(":test:logger-usage"),
+ project(":libs:opensearch-vectorized-exec-spi"), // TODO
+ project(":plugins:engine-datafusion"), //TODO
+ project(":server"),
+ project(":modules:parquet-data-format"),
+ project(":modules:parquet-data-format:benchmarks")
]) {
project.tasks.withType(MissingJavadocTask) {
isExcluded = true
diff --git a/gradle/run.gradle b/gradle/run.gradle
index ac58d74acd6b0..a2d96d31ad096 100644
--- a/gradle/run.gradle
+++ b/gradle/run.gradle
@@ -28,6 +28,7 @@
* under the License.
*/
import org.opensearch.gradle.testclusters.RunTask
+import org.opensearch.gradle.VersionProperties
apply plugin: 'opensearch.testclusters'
@@ -39,10 +40,75 @@ testClusters {
testDistribution = 'archive'
if (numZones > 1) numberOfZones = numZones
if (numNodes > 1) numberOfNodes = numNodes
+ // S3 repository configuration
+ if (findProperty("enableS3")) {
+ plugin(':plugins:repository-s3')
+ if (findProperty("s3Endpoint")) {
+ setting 's3.client.default.endpoint', findProperty("s3Endpoint")
+ }
+ setting 's3.client.default.region', findProperty("s3Region") ?: 'us-east-1'
+ keystore 's3.client.default.access_key', findProperty("s3AccessKey") ?: System.getenv("AWS_ACCESS_KEY_ID") ?: 'test'
+ keystore 's3.client.default.secret_key', findProperty("s3SecretKey") ?: System.getenv("AWS_SECRET_ACCESS_KEY") ?: 'test'
+
+
+ // Remote store configuration
+ setting 'node.attr.remote_store.segment.repository', 'my-s3-repo'
+ setting 'node.attr.remote_store.translog.repository', 'my-s3-repo'
+ setting 'node.attr.remote_store.state.repository', 'my-s3-repo'
+ setting 'cluster.remote_store.state.enabled', 'true'
+ setting 'node.attr.remote_store.repository.my-s3-repo.type', 's3'
+ setting 'node.attr.remote_store.repository.my-s3-repo.settings.bucket', 'local-opensearch-bucket'
+ setting 'node.attr.remote_store.repository.my-s3-repo.settings.base_path', 'raghraaj-local-1230'
+
+ // SSE-KMS configuration
+ if (findProperty("enableSseKms")) {
+ setting 'node.attr.remote_store.repository.my-s3-repo.settings.server_side_encryption_type', 'aws:kms'
+ setting 'node.attr.remote_store.repository.my-s3-repo.settings.server_side_encryption_kms_key_id', 'arn:aws:kms:us-east-1:389347062219:key/006ef490-5452-4f4d-8da3-a0cb7344ab59'
+ setting 'node.attr.remote_store.repository.my-s3-repo.settings.server_side_encryption_bucket_key_enabled', findProperty("sseBucketKeyEnabled") ?: 'true'
+ setting 'node.attr.remote_store.repository.my-s3-repo.settings.server_side_encryption_encryption_context', '{"identifier":"mustang"}'
+ }
+ }
if (findProperty("installedPlugins")) {
installedPlugins = Eval.me(installedPlugins)
+
+ def resolveMavenPlugin = { coords ->
+ // Add default groupId if not fully qualified (less than 2 colons)
+ String[] parts = coords.split(':')
+ if (parts.length == 2 && parts[0].contains('.')) {
+ throw new IllegalArgumentException("version is required if groupdId is specified '${coords}' Use format: groupId:artifactId:version")
+ }
+ String fullCoords = parts.length < 3 ? 'org.opensearch.plugin:' + coords : coords
+ def config = project.configurations.detachedConfiguration(
+ project.dependencies.create(fullCoords)
+ )
+ config.resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
+ plugin(project.layout.file(project.provider { config.singleFile }))
+ }
+
for (String p : installedPlugins) {
- plugin('plugins:'.concat(p))
+ // check if its a local plugin first
+ if (project.findProject(':plugins:' + p) != null) {
+ plugin('plugins:' + p)
+ } else {
+ // attempt to fetch it from maven
+ project.repositories.mavenLocal()
+ project.repositories {
+ maven {
+ name = 'OpenSearch Snapshots'
+ url = 'https://central.sonatype.com/repository/maven-snapshots/'
+ }
+ }
+ if (p.contains(':')) {
+ // Maven coordinates with version specified
+ String coords = p.contains('@') ? p : (p + '@zip')
+ resolveMavenPlugin(coords)
+ } else {
+ // Not found locally, try Maven with current OS version + 0
+ String version = VersionProperties.getOpenSearch().replace('-SNAPSHOT', '.0-SNAPSHOT')
+ String coords = p + ':' + version + '@zip'
+ resolveMavenPlugin(coords)
+ }
+ }
if (p.equals("arrow-flight-rpc")) {
// Add system properties for Netty configuration
systemProperty 'io.netty.allocator.numDirectArenas', '1'
@@ -52,6 +118,30 @@ testClusters {
}
}
}
+
+ if (findProperty("remotePlugins")) {
+ remotePlugins = Eval.me(remotePlugins)
+ for (String coords : remotePlugins) {
+ if (coords.startsWith('/') || coords.startsWith('file:')) {
+ // Direct file path
+ plugin(project.layout.file(project.provider { new File(coords) }))
+ } else {
+ // Maven coordinates
+ def config = project.configurations.detachedConfiguration(
+ project.dependencies.create(coords + '@zip')
+ )
+ config.resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
+ project.repositories.mavenLocal()
+ project.repositories {
+ maven {
+ name = 'OpenSearch Snapshots'
+ url = 'https://central.sonatype.com/repository/maven-snapshots/'
+ }
+ }
+ plugin(project.layout.file(project.provider { config.singleFile }))
+ }
+ }
+ }
}
}
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 8510f82823ef1..2757f36aeea64 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,17 +1,8 @@
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# The OpenSearch Contributors require contributions made to
-# this file be licensed under the Apache-2.0 license or a
-# compatible open source license.
-#
-# Modifications Copyright OpenSearch Contributors. See
-# GitHub history for details.
-#
-
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-all.zip
+networkTimeout=10000
+validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
-distributionSha256Sum=efe9a3d147d948d7528a9887fa35abcf24ca1a43ad06439996490f77569b02d1
+distributionSha256Sum=ed1a8d686605fd7c23bdf62c7fc7add1c5b23b2bbc3721e661934ef4a4911d7c
diff --git a/libs/agent-sm/agent/licenses/byte-buddy-1.17.5.jar.sha1 b/libs/agent-sm/agent/licenses/byte-buddy-1.17.5.jar.sha1
deleted file mode 100644
index d22afd953f340..0000000000000
--- a/libs/agent-sm/agent/licenses/byte-buddy-1.17.5.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-88450f120903b7e72470462cdbd2b75a3842223c
\ No newline at end of file
diff --git a/libs/agent-sm/agent/licenses/byte-buddy-1.17.7.jar.sha1 b/libs/agent-sm/agent/licenses/byte-buddy-1.17.7.jar.sha1
new file mode 100644
index 0000000000000..ef8db8435e631
--- /dev/null
+++ b/libs/agent-sm/agent/licenses/byte-buddy-1.17.7.jar.sha1
@@ -0,0 +1 @@
+3856bfab61beb23e099a0d6629f2ba8de4b98ace
\ No newline at end of file
diff --git a/libs/common/src/main/java/org/opensearch/common/Numbers.java b/libs/common/src/main/java/org/opensearch/common/Numbers.java
index d5a364a4a934e..6392884dff28b 100644
--- a/libs/common/src/main/java/org/opensearch/common/Numbers.java
+++ b/libs/common/src/main/java/org/opensearch/common/Numbers.java
@@ -83,44 +83,60 @@ public static boolean isValidDouble(double value) {
* stored value cannot be converted to a long that stores the exact same
* value. */
public static long toLongExact(Number n) {
- if (n instanceof Byte || n instanceof Short || n instanceof Integer || n instanceof Long) {
- return n.longValue();
- } else if (n instanceof Float || n instanceof Double) {
- double d = n.doubleValue();
- if (d != Math.round(d)) {
- throw new IllegalArgumentException(n + " is not an integer value");
+ return switch (n) {
+ case Byte b -> b.longValue();
+ case Short s -> s.longValue();
+ case Integer i -> i.longValue();
+ case Long l -> l;
+ case Float f -> {
+ double d = f.doubleValue();
+ if (d != Math.round(d)) {
+ throw new IllegalArgumentException(f + " is not an integer value");
+ }
+ yield f.longValue();
}
- return n.longValue();
- } else if (n instanceof BigDecimal) {
- return ((BigDecimal) n).toBigIntegerExact().longValueExact();
- } else if (n instanceof BigInteger) {
- return ((BigInteger) n).longValueExact();
- } else {
- throw new IllegalArgumentException(
+ case Double d -> {
+ if (d != Math.round(d)) {
+ throw new IllegalArgumentException(d + " is not an integer value");
+ }
+ yield d.longValue();
+ }
+ case BigDecimal bd -> bd.toBigIntegerExact().longValueExact();
+ case BigInteger bi -> bi.longValueExact();
+ default -> throw new IllegalArgumentException(
"Cannot check whether [" + n + "] of class [" + n.getClass().getName() + "] is actually a long"
);
- }
+ };
}
/** Return the {@link BigInteger} that {@code n} stores, or throws an exception if the
* stored value cannot be converted to a {@link BigInteger} that stores the exact same
* value. */
public static BigInteger toBigIntegerExact(Number n) {
- if (n instanceof Byte || n instanceof Short || n instanceof Integer || n instanceof Long) {
- return BigInteger.valueOf(n.longValue());
- } else if (n instanceof Float || n instanceof Double) {
- double d = n.doubleValue();
- if (d != Math.round(d)) {
- throw new IllegalArgumentException(n + " is not an integer value");
+ return switch (n) {
+ case Byte b -> BigInteger.valueOf(b.longValue());
+ case Short s -> BigInteger.valueOf(s.longValue());
+ case Integer i -> BigInteger.valueOf(i.longValue());
+ case Long l -> BigInteger.valueOf(l.longValue());
+ case Float f -> {
+ double d = f.doubleValue();
+ if (d != Math.round(d)) {
+ throw new IllegalArgumentException(f + " is not an integer value");
+ }
+ yield BigInteger.valueOf(f.longValue());
}
- return BigInteger.valueOf(n.longValue());
- } else if (n instanceof BigDecimal) {
- return ((BigDecimal) n).toBigIntegerExact();
- } else if (n instanceof BigInteger) {
- return ((BigInteger) n);
- } else {
- throw new IllegalArgumentException("Cannot convert [" + n + "] of class [" + n.getClass().getName() + "] to a BigInteger");
- }
+ case Double d -> {
+ if (d != Math.round(d)) {
+ throw new IllegalArgumentException(d + " is not an integer value");
+ }
+ yield BigInteger.valueOf(d.longValue());
+ }
+ case BigDecimal bd -> bd.toBigIntegerExact();
+ case BigInteger bi -> bi;
+ default -> throw new IllegalArgumentException(
+ "Cannot convert [" + n + "] of class [" + n.getClass().getName() + "] to a BigInteger"
+ );
+ };
}
/** Return the unsigned long (as {@link BigInteger}) that {@code n} stores, or throws an exception if the
diff --git a/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java b/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java
index 94ec0db3a9712..5f419ce621e24 100644
--- a/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java
+++ b/libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java
@@ -85,20 +85,20 @@ public boolean process(Set annotations, RoundEnvironment
Set.of(PublicApi.class, ExperimentalApi.class, DeprecatedApi.class)
);
- for (var element : elements) {
- validate(element);
-
- if (!checkPackage(element)) {
- continue;
- }
-
- // Skip all not-public elements
- checkPublicVisibility(null, element);
-
- if (element instanceof TypeElement) {
- process((TypeElement) element);
- }
- }
+// for (var element : elements) {
+// validate(element);
+//
+// if (!checkPackage(element)) {
+// continue;
+// }
+//
+// // Skip all not-public elements
+// checkPublicVisibility(null, element);
+//
+// if (element instanceof TypeElement) {
+// process((TypeElement) element);
+// }
+// }
return false;
}
diff --git a/libs/common/src/main/java/org/opensearch/common/recycler/Recycler.java b/libs/common/src/main/java/org/opensearch/common/recycler/Recycler.java
index 0b0c98772a77c..50533bd61faeb 100644
--- a/libs/common/src/main/java/org/opensearch/common/recycler/Recycler.java
+++ b/libs/common/src/main/java/org/opensearch/common/recycler/Recycler.java
@@ -32,6 +32,7 @@
package org.opensearch.common.recycler;
+import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.common.lease.Releasable;
/**
@@ -40,6 +41,7 @@
*
* @opensearch.internal
*/
+@ExperimentalApi
public interface Recycler {
/**
@@ -73,6 +75,7 @@ interface C {
*
* @opensearch.internal
*/
+ @ExperimentalApi
interface V extends Releasable {
/** Reference to the value. */
diff --git a/libs/core/licenses/lucene-core-10.2.2.jar.sha1 b/libs/core/licenses/lucene-core-10.2.2.jar.sha1
deleted file mode 100644
index 74d3bb5b1cbc8..0000000000000
--- a/libs/core/licenses/lucene-core-10.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-336a9c4b24e5704bd5fd71af794cce80f479a3ae
\ No newline at end of file
diff --git a/libs/core/licenses/lucene-core-10.3.1.jar.sha1 b/libs/core/licenses/lucene-core-10.3.1.jar.sha1
new file mode 100644
index 0000000000000..2f8f5071a7a7b
--- /dev/null
+++ b/libs/core/licenses/lucene-core-10.3.1.jar.sha1
@@ -0,0 +1 @@
+b0ea7e448e7377bd71892d818635cf9546299f4a
\ No newline at end of file
diff --git a/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java b/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java
index 5d5eeb41118c8..765cd1cbdc960 100644
--- a/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java
+++ b/libs/core/src/main/java/org/opensearch/ExceptionsHelper.java
@@ -76,57 +76,76 @@
public final class ExceptionsHelper {
private static final Logger logger = LogManager.getLogger(ExceptionsHelper.class);
+ /**
+ * Shared error message constants for consistent error handling across HTTP and gRPC protocols.
+ * These constants ensure that both REST API and gRPC API return identical error messages
+ * for the same types of exceptions.
+ */
+ public static final class ErrorMessages {
+ /** Error message for invalid argument exceptions */
+ public static final String INVALID_ARGUMENT = "Invalid argument";
+
+ /** Error message for JSON parsing failures */
+ public static final String JSON_PARSE_FAILED = "Failed to parse JSON";
+
+ /** Error message for rate limiting / rejected execution */
+ public static final String TOO_MANY_REQUESTS = "Too many requests";
+
+ /** Error message for JSON type coercion failures */
+ public static final String JSON_COERCION_FAILED = "Incompatible JSON value";
+
+ /** Error message for content format issues */
+ public static final String INVALID_CONTENT_FORMAT = "Invalid content format";
+
+ /** Error message for compression format issues */
+ public static final String INVALID_COMPRESSION_FORMAT = "Invalid compression format";
+
+ /** Generic fallback error message for unknown exceptions */
+ public static final String INTERNAL_FAILURE = "Internal failure";
+
+ private ErrorMessages() {
+ // Utility class, no instances
+ }
+ }
+
// utility class: no ctor
private ExceptionsHelper() {}
public static RuntimeException convertToRuntime(Exception e) {
- if (e instanceof RuntimeException) {
- return (RuntimeException) e;
- }
- return new OpenSearchException(e);
+ return switch (e) {
+ case RuntimeException re -> re;
+ default -> new OpenSearchException(e);
+ };
}
public static OpenSearchException convertToOpenSearchException(Exception e) {
- if (e instanceof OpenSearchException) {
- return (OpenSearchException) e;
- }
- return new OpenSearchException(e);
+ return switch (e) {
+ case OpenSearchException oe -> oe;
+ default -> new OpenSearchException(e);
+ };
}
public static RestStatus status(Throwable t) {
- if (t != null) {
- if (t instanceof OpenSearchException) {
- return ((OpenSearchException) t).status();
- } else if (t instanceof IllegalArgumentException) {
- return RestStatus.BAD_REQUEST;
- } else if (t instanceof InputCoercionException) {
- return RestStatus.BAD_REQUEST;
- } else if (t instanceof JsonParseException) {
- return RestStatus.BAD_REQUEST;
- } else if (t instanceof OpenSearchRejectedExecutionException) {
- return RestStatus.TOO_MANY_REQUESTS;
- } else if (t instanceof NotXContentException) {
- return RestStatus.BAD_REQUEST;
- }
- }
- return RestStatus.INTERNAL_SERVER_ERROR;
+ return switch (t) {
+ case OpenSearchException ose -> ose.status();
+ case IllegalArgumentException ignored -> RestStatus.BAD_REQUEST;
+ case InputCoercionException ignored -> RestStatus.BAD_REQUEST;
+ case JsonParseException ignored -> RestStatus.BAD_REQUEST;
+ case NotXContentException ignored -> RestStatus.BAD_REQUEST;
+ case OpenSearchRejectedExecutionException ignored -> RestStatus.TOO_MANY_REQUESTS;
+ case null, default -> RestStatus.INTERNAL_SERVER_ERROR;
+ };
}
public static String summaryMessage(Throwable t) {
- if (t != null) {
- if (t instanceof OpenSearchException) {
- return getExceptionSimpleClassName(t) + "[" + t.getMessage() + "]";
- } else if (t instanceof IllegalArgumentException) {
- return "Invalid argument";
- } else if (t instanceof InputCoercionException) {
- return "Incompatible JSON value";
- } else if (t instanceof JsonParseException) {
- return "Failed to parse JSON";
- } else if (t instanceof OpenSearchRejectedExecutionException) {
- return "Too many requests";
- }
- }
- return "Internal failure";
+ return switch (t) {
+ case OpenSearchException ose -> getExceptionSimpleClassName(t) + "[" + ose.getMessage() + "]";
+ case IllegalArgumentException ignored -> ErrorMessages.INVALID_ARGUMENT;
+ case InputCoercionException ignored -> ErrorMessages.JSON_COERCION_FAILED;
+ case JsonParseException ignored -> ErrorMessages.JSON_PARSE_FAILED;
+ case OpenSearchRejectedExecutionException ignored -> ErrorMessages.TOO_MANY_REQUESTS;
+ case null, default -> "Internal failure";
+ };
}
public static Throwable unwrapCause(Throwable t) {
@@ -149,6 +168,25 @@ public static Throwable unwrapCause(Throwable t) {
return result;
}
+ /**
+ * Unwraps exception causes up to 10 levels looking for the first OpenSearchException.
+ * This method is used by both HTTP and gRPC error handling to ensure consistent exception
+ * unwrapping behavior across protocols.
+ *
+ * @param e The exception to unwrap
+ * @return The first OpenSearchException found in the cause chain, or the original exception if none found
+ */
+ public static Throwable unwrapToOpenSearchException(Throwable e) {
+ Throwable t = e;
+ for (int counter = 0; counter < 10 && t != null; counter++) {
+ if (t instanceof OpenSearchException) {
+ break;
+ }
+ t = t.getCause();
+ }
+ return t != null ? t : e;
+ }
+
/**
* @deprecated Don't swallow exceptions, allow them to propagate.
*/
diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchException.java b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
index 8f1f5c929d865..8fb343630338b 100644
--- a/libs/core/src/main/java/org/opensearch/OpenSearchException.java
+++ b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
@@ -628,14 +628,8 @@ public static void generateFailureXContent(XContentBuilder builder, ToXContent.P
// Render the exception with a simple message
if (detailed == false) {
- Throwable t = e;
- for (int counter = 0; counter < 10 && t != null; counter++) {
- if (t instanceof OpenSearchException) {
- break;
- }
- t = t.getCause();
- }
- builder.field(ERROR, ExceptionsHelper.summaryMessage(t != null ? t : e));
+ Throwable unwrapped = ExceptionsHelper.unwrapToOpenSearchException(e);
+ builder.field(ERROR, ExceptionsHelper.summaryMessage(unwrapped));
return;
}
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index bfe03a23141d9..6b095a1c29789 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -117,10 +117,12 @@ public class Version implements Comparable, ToXContentFragment {
public static final Version V_2_19_1 = new Version(2190199, org.apache.lucene.util.Version.LUCENE_9_12_1);
public static final Version V_2_19_2 = new Version(2190299, org.apache.lucene.util.Version.LUCENE_9_12_1);
public static final Version V_2_19_3 = new Version(2190399, org.apache.lucene.util.Version.LUCENE_9_12_2);
+ public static final Version V_2_19_4 = new Version(2190499, org.apache.lucene.util.Version.LUCENE_9_12_3);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0);
public static final Version V_3_1_0 = new Version(3010099, org.apache.lucene.util.Version.LUCENE_10_2_1);
public static final Version V_3_2_0 = new Version(3020099, org.apache.lucene.util.Version.LUCENE_10_2_2);
- public static final Version CURRENT = V_3_2_0;
+ public static final Version V_3_3_0 = new Version(3030099, org.apache.lucene.util.Version.LUCENE_10_3_1);
+ public static final Version CURRENT = V_3_3_0;
public static Version fromId(int id) {
final Version known = LegacyESVersion.idToVersion.get(id);
diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
index 6b60e7448cd03..a28d2d930d30a 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -63,11 +63,10 @@ public interface BytesReference extends Comparable, ToXContentFr
static BytesReference bytes(XContentBuilder xContentBuilder) {
xContentBuilder.close();
OutputStream stream = xContentBuilder.getOutputStream();
- if (stream instanceof ByteArrayOutputStream) {
- return new BytesArray(((ByteArrayOutputStream) stream).toByteArray());
- } else {
- return ((BytesStream) stream).bytes();
- }
+ return switch (stream) {
+ case ByteArrayOutputStream baos -> new BytesArray(baos.toByteArray());
+ default -> ((BytesStream) stream).bytes();
+ };
}
/**
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java
index 6498b618b28c3..d27eb7197213f 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java
@@ -789,6 +789,8 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep
o.writeByte((byte) 27);
o.writeSemverRange((SemverRange) v);
});
+ // Have registered ScriptedAvg class with byte 28 in Streamables.java, so that we do not need the implementation reside in the
+ // server module
WRITERS = Collections.unmodifiableMap(writers);
}
diff --git a/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java b/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java
index c7b9bee3cbf4d..e123aafd399be 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java
@@ -169,24 +169,16 @@ private static void deeplyAppendParameter(StringBuilder sbuf, Object o, Set booleanArrayAppend(sbuf, boolArr);
+ case byte[] byteArr -> byteArrayAppend(sbuf, byteArr);
+ case char[] charArr -> charArrayAppend(sbuf, charArr);
+ case short[] shortArr -> shortArrayAppend(sbuf, shortArr);
+ case int[] intArr -> intArrayAppend(sbuf, intArr);
+ case long[] longArr -> longArrayAppend(sbuf, longArr);
+ case float[] floatArr -> floatArrayAppend(sbuf, floatArr);
+ case double[] doubleArr -> doubleArrayAppend(sbuf, doubleArr);
+ default -> objectArrayAppend(sbuf, (Object[]) o, seen);
}
}
}
diff --git a/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java b/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java
index 8b29378dfde12..08f5f120f879d 100644
--- a/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java
+++ b/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java
@@ -40,6 +40,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
@@ -132,21 +133,21 @@ public void testIsHidden() {
}
public void testOpenFileURLStream() throws IOException {
- URL urlWithWrongProtocol = new URL("http://www.google.com");
+ URL urlWithWrongProtocol = URI.create("http://www.google.com").toURL();
try (InputStream is = FileSystemUtils.openFileURLStream(urlWithWrongProtocol)) {
fail("Should throw IllegalArgumentException due to invalid protocol");
} catch (IllegalArgumentException e) {
assertEquals("Invalid protocol [http], must be [file] or [jar]", e.getMessage());
}
- URL urlWithHost = new URL("file", "localhost", txtFile.toString());
+ URL urlWithHost = URI.create("file://localhost/" + txtFile.toString()).toURL();
try (InputStream is = FileSystemUtils.openFileURLStream(urlWithHost)) {
fail("Should throw IllegalArgumentException due to host");
} catch (IllegalArgumentException e) {
assertEquals("URL cannot have host. Found: [localhost]", e.getMessage());
}
- URL urlWithPort = new URL("file", "", 80, txtFile.toString());
+ URL urlWithPort = URI.create("file://:80/" + txtFile.toString()).toURL();
try (InputStream is = FileSystemUtils.openFileURLStream(urlWithPort)) {
fail("Should throw IllegalArgumentException due to port");
} catch (IllegalArgumentException e) {
diff --git a/libs/grok/src/main/java/org/opensearch/grok/Grok.java b/libs/grok/src/main/java/org/opensearch/grok/Grok.java
index aa5b1a936b99d..02e5f5bd54ff2 100644
--- a/libs/grok/src/main/java/org/opensearch/grok/Grok.java
+++ b/libs/grok/src/main/java/org/opensearch/grok/Grok.java
@@ -99,17 +99,28 @@ public final class Grok {
private final Regex compiledExpression;
private final MatcherWatchdog matcherWatchdog;
private final List captureConfig;
+ private final boolean captureAllMatches;
public Grok(Map patternBank, String grokPattern, Consumer logCallBack) {
- this(patternBank, grokPattern, true, MatcherWatchdog.noop(), logCallBack);
+ this(patternBank, grokPattern, true, MatcherWatchdog.noop(), logCallBack, false);
}
public Grok(Map patternBank, String grokPattern, MatcherWatchdog matcherWatchdog, Consumer logCallBack) {
- this(patternBank, grokPattern, true, matcherWatchdog, logCallBack);
+ this(patternBank, grokPattern, true, matcherWatchdog, logCallBack, false);
+ }
+
+ public Grok(
+ Map patternBank,
+ String grokPattern,
+ MatcherWatchdog matcherWatchdog,
+ Consumer logCallBack,
+ boolean captureAllMatches
+ ) {
+ this(patternBank, grokPattern, true, matcherWatchdog, logCallBack, captureAllMatches);
}
Grok(Map patternBank, String grokPattern, boolean namedCaptures, Consumer logCallBack) {
- this(patternBank, grokPattern, namedCaptures, MatcherWatchdog.noop(), logCallBack);
+ this(patternBank, grokPattern, namedCaptures, MatcherWatchdog.noop(), logCallBack, false);
}
private Grok(
@@ -117,11 +128,13 @@ private Grok(
String grokPattern,
boolean namedCaptures,
MatcherWatchdog matcherWatchdog,
- Consumer logCallBack
+ Consumer logCallBack,
+ boolean captureAllMatches
) {
this.patternBank = patternBank;
this.namedCaptures = namedCaptures;
this.matcherWatchdog = matcherWatchdog;
+ this.captureAllMatches = captureAllMatches;
validatePatternBank();
@@ -395,7 +408,7 @@ public boolean match(byte[] utf8Bytes, int offset, int length, GrokCaptureExtrac
if (result == Matcher.FAILED) {
return false;
}
- extracter.extract(utf8Bytes, offset, matcher.getEagerRegion());
+ extracter.extract(utf8Bytes, offset, matcher.getEagerRegion(), captureAllMatches);
return true;
}
diff --git a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java
index b6d881de4fac1..9e052584cef63 100644
--- a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java
+++ b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureExtracter.java
@@ -57,14 +57,29 @@ static class MapExtracter extends GrokCaptureExtracter {
result = captureConfig.isEmpty() ? emptyMap() : new HashMap<>();
fieldExtracters = new ArrayList<>(captureConfig.size());
for (GrokCaptureConfig config : captureConfig) {
- fieldExtracters.add(config.objectExtracter(v -> result.put(config.name(), v)));
+ fieldExtracters.add(config.objectExtracter(v -> {
+ String fieldName = config.name();
+ Object existing = result.get(fieldName);
+ if (existing == null) {
+ result.put(fieldName, v);
+ } else if (existing instanceof List) {
+ @SuppressWarnings("unchecked")
+ List
+