Merge - OpenSearch main

.github/workflows/detect-breaking-change.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         cache-disabled: true
         arguments: japicmp
-        gradle-version: 9.1.0
+        gradle-version: 9.4.1
         build-root-directory: server
     - if: failure()
       run: cat server/build/reports/java-compatibility/report.txt

.github/workflows/sandbox-check.yml

@@ -0,0 +1,43 @@
+name: Sandbox Check
+on:
+  push:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  sandbox-check:
+    if: github.repository == 'opensearch-project/OpenSearch'
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v6
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - name: Set up JDK 25
+        uses: actions/setup-java@v5
+        with:
+          java-version: 25
+          distribution: temurin
+          cache: gradle
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Run sandbox check
+        run: ./gradlew check -p sandbox -Dsandbox.enabled=true
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sandbox-test-results
+          path: sandbox/**/build/reports/tests/
+          retention-days: 7

.gitignore

@@ -71,3 +71,4 @@ testfixtures_shared/
 doc-tools/missing-doclet/bin/
 **/Cargo.lock
 /sandbox/plugins/analytics-backend-datafusion/target/
+/sandbox/libs/dataformat-native/rust/target

CONTRIBUTING.md

@@ -138,7 +138,7 @@ We have a lot of mechanisms to help expedite towards an accepted PR. Here are so
     - `@opensearch.internal`: Marks internal classes subject to rapid changes.
     - `@opensearch.api`: Marks public-facing API classes with backward compatibility guarantees.
     - `@opensearch.experimental`: Indicates rapidly changing [experimental code](./DEVELOPER_GUIDE.md#experimental-development).
-5. *Employ sandbox for significant core changes*: Any new features or enhancements that make changes to core classes (e.g., search phases, codecs, or specialized lucene APIs) are more likely to. be merged if they are sandboxed. This can only be enabled on the java CLI (`-Dsandbox.enabled=true`).
+5. *Employ sandbox for significant core changes*: Any new features or enhancements that make changes to core classes (e.g., search phases, codecs, or specialized lucene APIs) are more likely to be merged if they are sandboxed. Sandbox is disabled by default and can be enabled on the Java CLI with `-Dsandbox.enabled=true`.
 6. *Micro-benchmark critical path*: This is a lesser known mechanism, but if you have critical path changes you're afraid will impact performance (the changes touch the garbage collector, heap, direct memory, or CPU) then including a [microbenchmark](https://github.com/opensearch-project/OpenSearch/tree/main/benchmarks) with your PR (and jfr or flamegraph results in the description) is a *GREAT IDEA* and will help expedite the review process.
 7. *Test rigorously*: Ensure thorough testing ([OpenSearchTestCase](./test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java) for unit tests, [OpenSearchIntegTestCase](./test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java) for integration & cluster tests, [OpenSearchRestTestCase](./test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java) for testing REST endpoint interfaces, and yaml tests with [ClientYamlTestSuiteIT](./rest-api-spec/src/yamlRestTest/java/org/opensearch/test/rest/ClientYamlTestSuiteIT.java) for REST integration tests)
 

DEVELOPER_GUIDE.md

@@ -313,11 +313,11 @@ Another example is the `discovery-gce` plugin. It is *vital* to folks running in
 
 ### `sandbox`
 
-This is where the community can add experimental features in to OpenSearch. There are three directories inside the sandbox - `libs`, `modules` and `plugins` - which mirror the subdirectories in the project root and have the same guidelines for deciding on where a new feature goes. The artifacts from `libs` and `modules` will be automatically included in the **snapshot** distributions. Once a certain feature is deemed worthy to be included in the OpenSearch release, it will be promoted to the corresponding subdirectory in the project root. **Note**: The sandbox code do not have any other guarantees such as backwards compatibility or long term support and can be removed at any time.
+This is where the community can add experimental features in to OpenSearch. There are three directories inside the sandbox - `libs`, `modules` and `plugins` - which mirror the subdirectories in the project root and have the same guidelines for deciding on where a new feature goes. The artifacts from `libs` and `modules` can be included in the **snapshot** distributions when sandbox is enabled. Once a certain feature is deemed worthy to be included in the OpenSearch release, it will be promoted to the corresponding subdirectory in the project root. **Note**: The sandbox code do not have any other guarantees such as backwards compatibility or long term support and can be removed at any time.
 
-To exclude the modules from snapshot distributions, use the `sandbox.enabled` system property.
+To include sandbox modules in snapshot distributions, use the `sandbox.enabled` system property.
 
-    ./gradlew assemble -Dsandbox.enabled=false
+    ./gradlew assemble -Dsandbox.enabled=true
 
 ### `qa`
 

buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java

@@ -34,6 +34,7 @@
 import org.apache.tools.ant.taskdefs.condition.Os;
 import org.opensearch.gradle.Version;
 import org.opensearch.gradle.info.BuildParams;
+import org.opensearch.gradle.util.ExecutableUtils;
 import org.gradle.api.GradleException;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
@@ -50,7 +51,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -65,24 +65,20 @@
  */
 public abstract class DockerSupportService implements BuildService<DockerSupportService.Parameters> {
 
-    private static Logger LOGGER = Logging.getLogger(DockerSupportService.class);
-    // Defines the possible locations of the Docker CLI. These will be searched in order.
-    private static String[] DOCKER_BINARIES_UNIX = { "/usr/bin/docker", "/usr/local/bin/docker" };
+    private static final Logger LOGGER = Logging.getLogger(DockerSupportService.class);
 
-    private static String[] DOCKER_BINARIES_WINDOWS = {
-        System.getenv("PROGRAMFILES") + "\\Docker\\Docker\\resources\\bin\\docker.exe",
-        System.getenv("SystemRoot") + "\\System32\\docker.exe" /* Github Actions */ };
+    private static final String DOCKER_FILENAME = Os.isFamily(Os.FAMILY_WINDOWS) ? "docker.exe" : "docker";
 
-    private static String[] DOCKER_BINARIES = Os.isFamily(Os.FAMILY_WINDOWS) ? DOCKER_BINARIES_WINDOWS : DOCKER_BINARIES_UNIX;
+    private static final String DOCKER_COMPOSE_FILENAME = Os.isFamily(Os.FAMILY_WINDOWS) ? "docker-compose.exe" : "docker-compose";
 
-    private static String[] DOCKER_COMPOSE_BINARIES_UNIX = { "/usr/local/bin/docker-compose", "/usr/bin/docker-compose" };
+    private static final String[] DEFAULT_PATH_UNIX = { "/usr/bin", "/usr/local/bin" };
 
-    private static String[] DOCKER_COMPOSE_BINARIES_WINDOWS = {
-        System.getenv("PROGRAMFILES") + "\\Docker\\Docker\\resources\\bin\\docker-compose.exe" };
+    private static final String[] DEFAULT_PATH_WINDOWS = {
+        System.getenv("PROGRAMFILES") + "\\Docker\\Docker\\resources\\bin",
+        System.getenv("SystemRoot") + "\\System32" /* Github Actions */
+    };
 
-    private static String[] DOCKER_COMPOSE_BINARIES = Os.isFamily(Os.FAMILY_WINDOWS)
-        ? DOCKER_COMPOSE_BINARIES_WINDOWS
-        : DOCKER_COMPOSE_BINARIES_UNIX;
+    private static final String[] DEFAULT_PATH = Os.isFamily(Os.FAMILY_WINDOWS) ? DEFAULT_PATH_WINDOWS : DEFAULT_PATH_UNIX;
 
     private static final Version MINIMUM_DOCKER_VERSION = Version.fromString("17.05.0");
 
@@ -177,10 +173,11 @@ void failIfDockerUnavailable(List<String> tasks) {
 
         // No Docker binary was located
         if (availability.path == null) {
+            final String[] dockerPaths = ExecutableUtils.mergePaths(DEFAULT_PATH, ExecutableUtils.getPathEnv());
             final String message = String.format(
                 Locale.ROOT,
                 "Docker (checked [%s]) is required to run the following task%s: \n%s",
-                String.join(", ", DOCKER_BINARIES),
+                String.join(", ", dockerPaths),
                 tasks.size() > 1 ? "s" : "",
                 String.join("\n", tasks)
             );
@@ -290,15 +287,14 @@ static Map<String, String> parseOsRelease(final List<String> osReleaseLines) {
     }
 
     /**
-     * Searches the entries in {@link #DOCKER_BINARIES} for the Docker CLI. This method does
+     * Searches for the Docker CLI in the system PATH and default locations. This method does
      * not check whether the Docker installation appears usable, see {@link #getDockerAvailability()}
      * instead.
      *
      * @return the path to a CLI, if available.
      */
     private Optional<String> getDockerPath() {
-        // Check if the Docker binary exists
-        return Arrays.asList(DOCKER_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
+        return ExecutableUtils.findExecutableInKnownPaths(DOCKER_FILENAME, DEFAULT_PATH);
     }
 
     private void throwDockerRequiredException(final String message) {
@@ -403,47 +399,70 @@ public boolean isDockerComposeAvailable() {
     /**
      * Marker interface for Docker Compose availability
      */
-    private interface DockerComposeAvailability {
+    public interface DockerComposeAvailability {
         /**
          * Detects Docker Compose V1/V2 availability
          */
         private static Optional<DockerComposeAvailability> detect(ExecOperations execOperations, String dockerPath) {
             Optional<String> composePath = getDockerComposePath();
             if (composePath.isPresent()) {
                 if (runCommand(execOperations, composePath.get(), "version").isSuccess()) {
-                    return Optional.of(new DockerComposeV1Availability());
+                    return Optional.of(new DockerComposeV1Availability(composePath.get()));
                 }
             }
 
             if (runCommand(execOperations, dockerPath, "compose", "version").isSuccess()) {
-                return Optional.of(new DockerComposeV2Availability());
+                return Optional.of(new DockerComposeV2Availability(dockerPath));
             }
 
             return Optional.empty();
         }
 
         /**
-         * Searches the entries in {@link #DOCKER_COMPOSE_BINARIES} for the Docker Compose CLI. This method does
+         * Searches the entries in env variable PATH with fallback to {@link #DEFAULT_PATH} for the Docker Compose CLI. This method does
          * not check whether the installation appears usable, see {@link #getDockerAvailability()} instead.
          *
          * @return the path to a CLI, if available.
          */
         private static Optional<String> getDockerComposePath() {
-            // Check if the Docker binary exists
-            return Arrays.asList(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
+            return ExecutableUtils.findExecutableInKnownPaths(DOCKER_COMPOSE_FILENAME, DEFAULT_PATH);
         }
 
+        /**
+         * The path to the Docker CLI, or null
+         */
+        public String getPath();
     }
 
     /**
      * Docker Compose V1 availability
      */
-    public static class DockerComposeV1Availability implements DockerComposeAvailability {}
+    public static class DockerComposeV1Availability implements DockerComposeAvailability {
+        private final String path;
+
+        DockerComposeV1Availability(String path) {
+            this.path = path;
+        }
+
+        public String getPath() {
+            return this.path;
+        }
+    }
 
     /**
      * Docker Compose V2 availability
      */
-    public static class DockerComposeV2Availability implements DockerComposeAvailability {}
+    public static class DockerComposeV2Availability implements DockerComposeAvailability {
+        private final String path;
+
+        DockerComposeV2Availability(String path) {
+            this.path = path;
+        }
+
+        public String getPath() {
+            return this.path;
+        }
+    }
 
     /**
      * This class models the result of running a command. It captures the exit code, standard output and standard error.

buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -39,10 +39,10 @@
 import com.avast.gradle.dockercompose.tasks.ComposePull;
 import com.avast.gradle.dockercompose.tasks.ComposeUp;
 
-import org.apache.tools.ant.taskdefs.condition.Os;
 import org.opensearch.gradle.SystemPropertyCommandLineArgumentProvider;
 import org.opensearch.gradle.docker.DockerSupportPlugin;
 import org.opensearch.gradle.docker.DockerSupportService;
+import org.opensearch.gradle.docker.DockerSupportService.DockerAvailability;
 import org.opensearch.gradle.docker.DockerSupportService.DockerComposeV2Availability;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.precommit.TestingConventionsTasks;
@@ -68,9 +68,7 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.Optional;
 import java.util.function.BiConsumer;
 
 public class TestFixturesPlugin implements Plugin<Project> {
@@ -79,15 +77,6 @@ public class TestFixturesPlugin implements Plugin<Project> {
     private static final String DOCKER_COMPOSE_THROTTLE = "dockerComposeThrottle";
     static final String DOCKER_COMPOSE_YML = "docker-compose.yml";
 
-    private static String[] DOCKER_COMPOSE_BINARIES_UNIX = { "/usr/local/bin/docker-compose", "/usr/bin/docker-compose" };
-
-    private static String[] DOCKER_COMPOSE_BINARIES_WINDOWS = {
-        System.getenv("PROGRAMFILES") + "\\Docker\\Docker\\resources\\bin\\docker-compose.exe" };
-
-    private static String[] DOCKER_COMPOSE_BINARIES = Os.isFamily(Os.FAMILY_WINDOWS)
-        ? DOCKER_COMPOSE_BINARIES_WINDOWS
-        : DOCKER_COMPOSE_BINARIES_UNIX;
-
     @Inject
     protected FileSystemOperations getFileSystemOperations() {
         throw new UnsupportedOperationException();
@@ -166,12 +155,11 @@ public void execute(Task task) {
             final Integer timeout = ext.has("dockerComposeHttpTimeout") ? (Integer) ext.get("dockerComposeHttpTimeout") : 120;
             composeExtension.getEnvironment().put("COMPOSE_HTTP_TIMEOUT", timeout);
 
-            Optional<String> dockerCompose = Arrays.asList(DOCKER_COMPOSE_BINARIES)
-                .stream()
-                .filter(path -> project.file(path).exists())
-                .findFirst();
+            final DockerAvailability dockerAvailability = dockerSupport.get().getDockerAvailability();
+            if (dockerAvailability.isAvailable && dockerAvailability.isDockerComposeAvailable()) {
+                composeExtension.getExecutable().set(dockerAvailability.dockerComposeAvailability.getPath());
+            }
 
-            composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker");
             composeExtension.getUseDockerComposeV2()
                 .set(dockerSupport.get().getDockerAvailability().dockerComposeAvailability instanceof DockerComposeV2Availability);