Resolve analytics-engine/framework from remote, drop vendored libs/

build.gradle

@@ -81,6 +81,7 @@ plugins {
     id "io.freefair.lombok" version "8.14"
     id 'jacoco'
     id 'com.diffplug.spotless' version '8.1.0'
+    id 'de.undercouch.download' version '5.3.0'
 }
 
 // import versions defined in https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java#L94
@@ -117,6 +118,38 @@ spotless {
     }
 }
 
+// Remote URLs for the analytics-engine + arrow-flight-rpc plugin ZIPs. The OpenSearch CI maven
+// snapshot repo only publishes jar/pom/sources/javadoc, not the plugin distribution zips — so we
+// fetch them from the feature-build artifact URLs directly. Versioned by the OpenSearch core
+// feature build, not by the maven snapshot timestamp.
+ext.analyticsEngineZipUrl = 'https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins/1-analytics-engine-3.7.0.zip'
+ext.analyticsEngineZipFile = "${rootProject.layout.buildDirectory.get().asFile}/distributions/analytics-engine-3.7.0-SNAPSHOT.zip"
+ext.arrowFlightRpcZipUrl = 'https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins/0-arrow-flight-rpc-3.7.0.zip'
+ext.arrowFlightRpcZipFile = "${rootProject.layout.buildDirectory.get().asFile}/distributions/arrow-flight-rpc-3.7.0-SNAPSHOT.zip"
+
+tasks.register('downloadAnalyticsEngineZip', de.undercouch.gradle.tasks.download.Download) {
+    src analyticsEngineZipUrl
+    dest analyticsEngineZipFile
+    overwrite false
+    onlyIfModified true
+    // Skip the network call when the user supplies a local zip via -PanalyticsEngineZip=/path.
+    // (We can't use `project.hasProperty` here because the ext property is always set in the
+    // allprojects block below.)
+    onlyIf {
+        !project.gradle.startParameter.projectProperties.containsKey('analyticsEngineZip')
+    }
+}
+
+tasks.register('downloadArrowFlightRpcZip', de.undercouch.gradle.tasks.download.Download) {
+    src arrowFlightRpcZipUrl
+    dest arrowFlightRpcZipFile
+    overwrite false
+    onlyIfModified true
+    onlyIf {
+        !project.gradle.startParameter.projectProperties.containsKey('arrowFlightRpcZip')
+    }
+}
+
 allprojects {
     version = opensearch_version.tokenize('-')[0] + '.0'
     if (buildVersionQualifier) {
@@ -127,10 +160,13 @@ allprojects {
         version += "-SNAPSHOT"
     }
 
-    // Path to the analytics-engine plugin ZIP. Override with
-    // `-PanalyticsEngineZip=/path/to/zip` if needed.
-    ext.analyticsEngineZip = project.findProperty('analyticsEngineZip') ?:
-            "${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip"
+    // Plugin ZIPs needed by integration test clusters. analytics-engine declares arrow-flight-rpc
+    // as an extendedPlugins parent, so both must be installed (arrow-flight-rpc first). Both
+    // default to files produced by the root :download* tasks; either can be overridden via
+    // `-PanalyticsEngineZip=/path` / `-ParrowFlightRpcZip=/path` for local development against an
+    // unpublished build.
+    ext.analyticsEngineZip = project.findProperty('analyticsEngineZip') ?: rootProject.analyticsEngineZipFile
+    ext.arrowFlightRpcZip = project.findProperty('arrowFlightRpcZip') ?: rootProject.arrowFlightRpcZipFile
 
     plugins.withId('java') {
         java {
@@ -173,7 +209,13 @@ allprojects {
 
 subprojects {
     repositories {
-        mavenLocal()
+        // Skip mavenLocal for org.opensearch.sandbox.* (analytics-framework, analytics-engine):
+        // a locally-published OpenSearch core build can install these with Gradle Module Metadata
+        // declaring different attributes than the published snapshot, which can shadow the remote
+        // and break resolution. Always pull these from the CI snapshot repo.
+        mavenLocal {
+            content { excludeGroup 'org.opensearch.sandbox' }
+        }
         mavenCentral()
         maven {
             url 'https://jitpack.io'
@@ -183,6 +225,33 @@ subprojects {
         maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }
     }
 
+    // Sandbox artifacts (analytics-framework, analytics-engine) publish Gradle Module Metadata
+    // declaring `org.gradle.jvm.version=25` because the sandbox uses the FFM API (finalized in
+    // JDK 22). sql targets JVM 21, which would make Gradle's variant matcher refuse them.
+    // Override the published attribute to 21 for these two modules only — safe because:
+    //   - sql code references only non-FFM interfaces of these jars (compile-time API surface).
+    //   - Production runtime targets JVM 25 nodes, where the actual JDK 25 bytecode runs fine.
+    //   - Rule is scoped to `org.opensearch.sandbox:{analytics-framework,analytics-engine}` —
+    //     no effect on any other dependency.
+    dependencies {
+        components {
+            withModule('org.opensearch.sandbox:analytics-framework') { details ->
+                details.allVariants {
+                    attributes {
+                        attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 21)
+                    }
+                }
+            }
+            withModule('org.opensearch.sandbox:analytics-engine') { details ->
+                details.allVariants {
+                    attributes {
+                        attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 21)
+                    }
+                }
+            }
+        }
+    }
+
     // Publish internal modules as Maven artifacts for external use, such as by opensearch-spark and opensearch-cli.
     def publishedModules = ['api', 'sql', 'ppl', 'core', 'opensearch', 'common', 'protocol', 'datasources', 'legacy']
     if (publishedModules.contains(name)) {

core/build.gradle

@@ -64,11 +64,19 @@ dependencies {
     }
     api 'org.apache.calcite:calcite-linq4j:1.41.0'
     api project(':common')
-    compileOnly files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    // transitive = false: at runtime, sql's classloader delegates to analytics-engine's via
+    // `extendedPlugins` parent-first, so the patched calcite (1.41.0-opensearch-1) it transitively
+    // pulls in must NOT enter sql's compile classpath — sql compiles & bundles vanilla calcite
+    // 1.41.0, and the AE-supplied patched copy wins at runtime via classloader delegation.
+    compileOnly('org.opensearch.sandbox:analytics-framework:3.7.0-SNAPSHOT') {
+        transitive = false
+    }
     // Needed because the analytics-framework's QueryPlanExecutor signature uses
     // org.opensearch.core.action.ActionListener; AnalyticsExecutionEngine references that type.
     compileOnly group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}"
-    testImplementation files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    testImplementation('org.opensearch.sandbox:analytics-framework:3.7.0-SNAPSHOT') {
+        transitive = false
+    }
     testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}"
     implementation "com.github.seancfoley:ipaddress:5.4.2"
     implementation "com.jayway.jsonpath:json-path:2.9.0"

doctest/build.gradle

@@ -15,6 +15,11 @@ plugins {
 
 apply plugin: 'opensearch.testclusters'
 
+// Ensure both plugin zips are downloaded before any task that boots a test cluster.
+tasks.withType(RunTask).configureEach {
+    dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip'
+}
+
 def path = project(':').projectDir
 // temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from
 // plugin module, and will only build ppl in it.
@@ -195,7 +200,9 @@ testClusters {
         }))
         */
         plugin(getJobSchedulerPlugin(jsPlugin, bwcOpenSearchJSDownload))
-        plugin provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) }
+        // arrow-flight-rpc must install before analytics-engine (it's an extendedPlugins parent).
+        plugin provider { (RegularFile) (() -> file("${arrowFlightRpcZip}")) }
+        plugin provider { (RegularFile) (() -> file("${analyticsEngineZip}")) }
         plugin ':opensearch-sql-plugin'
         testDistribution = 'archive'
     }

integ-test/build.gradle

@@ -271,14 +271,30 @@ def getGeoSpatialPlugin() {
 }
 
 def getAnalyticsEnginePlugin() {
-    provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) }
+    provider { (RegularFile) (() -> file("${analyticsEngineZip}")) }
+}
+
+// arrow-flight-rpc is declared as an extendedPlugins dependency by analytics-engine, so it must
+// be installed first or analytics-engine's plugin install fails with `Missing plugin
+// [arrow-flight-rpc], dependency of [analytics-engine]`.
+def getArrowFlightRpcPlugin() {
+    provider { (RegularFile) (() -> file("${arrowFlightRpcZip}")) }
+}
+
+// Ensure both plugin zips are downloaded before any task that boots a test cluster.
+tasks.withType(StandaloneRestIntegTestTask).configureEach {
+    dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip'
+}
+tasks.withType(RestIntegTestTask).configureEach {
+    dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip'
 }
 
 testClusters {
     integTest {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
         plugin(getGeoSpatialPlugin())
+        plugin(getArrowFlightRpcPlugin())
         plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
         setting "plugins.query.datasources.encryption.masterkey", "1234567812345678"
@@ -287,6 +303,7 @@ testClusters {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
         plugin(getGeoSpatialPlugin())
+        plugin(getArrowFlightRpcPlugin())
         plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
         setting "plugins.query.datasources.encryption.masterkey", "1234567812345678"
@@ -295,18 +312,21 @@ testClusters {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
         plugin(getGeoSpatialPlugin())
+        plugin(getArrowFlightRpcPlugin())
         plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }
     integTestWithSecurity {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
+        plugin(getArrowFlightRpcPlugin())
         plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }
     remoteIntegTestWithSecurity {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
+        plugin(getArrowFlightRpcPlugin())
         plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }
@@ -362,6 +382,7 @@ stopPrometheus.mustRunAfter startPrometheus
 
 task integJdbcTest(type: RestIntegTestTask) {
     testClusters.findAll {c -> c.clusterName == "integJdbcTest"}.first().with {
+        plugin(getArrowFlightRpcPlugin())
         plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }

plugin/build.gradle

@@ -1,5 +1,8 @@
 import java.util.concurrent.Callable
 import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin
+import org.opensearch.gradle.test.RestIntegTestTask
+import org.opensearch.gradle.testclusters.RunTask
+import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 /*
  * Copyright OpenSearch Contributors
@@ -162,9 +165,15 @@ dependencies {
     api project(":ppl")
     api project(':api')
     // Bundled: analytics-framework interfaces must resolve even when the plugin is absent.
-    api files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    // transitive = false: at runtime sql loads calcite via AE's classloader (extendedPlugins
+    // parent-first), so sql's classpath must not pull AF's patched-calcite transitive.
+    api('org.opensearch.sandbox:analytics-framework:3.7.0-SNAPSHOT') {
+        transitive = false
+    }
     // Not bundled: classes here (e.g. OpenSearchSchemaBuilder) only load when the plugin is installed.
-    compileOnly files("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.jar")
+    compileOnly('org.opensearch.sandbox:analytics-engine:3.7.0-SNAPSHOT') {
+        transitive = false
+    }
     api project(':legacy')
     api project(':opensearch')
     api project(':prometheus')
@@ -310,6 +319,10 @@ def getJobSchedulerPlugin() {
 
 testClusters.integTest {
     plugin(getJobSchedulerPlugin())
+    // arrow-flight-rpc is declared as an extendedPlugins dependency by analytics-engine, so
+    // it must be installed first or analytics-engine fails the plugin install jar-hell check
+    // with `Missing plugin [arrow-flight-rpc], dependency of [analytics-engine]`.
+    plugin provider { (RegularFile) (() -> file("${arrowFlightRpcZip}")) }
     plugin provider { (RegularFile) (() -> file("${analyticsEngineZip}")) }
     plugin(project.tasks.bundlePlugin.archiveFile)
     testDistribution = "ARCHIVE"
@@ -327,3 +340,14 @@ run {
     useCluster testClusters.integTest
 }
 
+// Ensure both plugin zips are downloaded before any task that boots a test cluster.
+tasks.withType(RunTask).configureEach {
+    dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip'
+}
+tasks.withType(StandaloneRestIntegTestTask).configureEach {
+    dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip'
+}
+tasks.withType(RestIntegTestTask).configureEach {
+    dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip'
+}
+