diff --git a/build.gradle b/build.gradle index 5916afc915d..67941417095 100644 --- a/build.gradle +++ b/build.gradle @@ -81,6 +81,7 @@ plugins { id "io.freefair.lombok" version "8.14" id 'jacoco' id 'com.diffplug.spotless' version '8.1.0' + id 'de.undercouch.download' version '5.3.0' } // import versions defined in https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java#L94 @@ -117,6 +118,38 @@ spotless { } } +// Remote URLs for the analytics-engine + arrow-flight-rpc plugin ZIPs. The OpenSearch CI maven +// snapshot repo only publishes jar/pom/sources/javadoc, not the plugin distribution zips — so we +// fetch them from the feature-build artifact URLs directly. Versioned by the OpenSearch core +// feature build, not by the maven snapshot timestamp. +ext.analyticsEngineZipUrl = 'https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins/1-analytics-engine-3.7.0.zip' +ext.analyticsEngineZipFile = "${rootProject.layout.buildDirectory.get().asFile}/distributions/analytics-engine-3.7.0-SNAPSHOT.zip" +ext.arrowFlightRpcZipUrl = 'https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins/0-arrow-flight-rpc-3.7.0.zip' +ext.arrowFlightRpcZipFile = "${rootProject.layout.buildDirectory.get().asFile}/distributions/arrow-flight-rpc-3.7.0-SNAPSHOT.zip" + +tasks.register('downloadAnalyticsEngineZip', de.undercouch.gradle.tasks.download.Download) { + src analyticsEngineZipUrl + dest analyticsEngineZipFile + overwrite false + onlyIfModified true + // Skip the network call when the user supplies a local zip via -PanalyticsEngineZip=/path. + // (We can't use `project.hasProperty` here because the ext property is always set in the + // allprojects block below.) + onlyIf { + !project.gradle.startParameter.projectProperties.containsKey('analyticsEngineZip') + } +} + +tasks.register('downloadArrowFlightRpcZip', de.undercouch.gradle.tasks.download.Download) { + src arrowFlightRpcZipUrl + dest arrowFlightRpcZipFile + overwrite false + onlyIfModified true + onlyIf { + !project.gradle.startParameter.projectProperties.containsKey('arrowFlightRpcZip') + } +} + allprojects { version = opensearch_version.tokenize('-')[0] + '.0' if (buildVersionQualifier) { @@ -127,10 +160,13 @@ allprojects { version += "-SNAPSHOT" } - // Path to the analytics-engine plugin ZIP. Override with - // `-PanalyticsEngineZip=/path/to/zip` if needed. - ext.analyticsEngineZip = project.findProperty('analyticsEngineZip') ?: - "${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip" + // Plugin ZIPs needed by integration test clusters. analytics-engine declares arrow-flight-rpc + // as an extendedPlugins parent, so both must be installed (arrow-flight-rpc first). Both + // default to files produced by the root :download* tasks; either can be overridden via + // `-PanalyticsEngineZip=/path` / `-ParrowFlightRpcZip=/path` for local development against an + // unpublished build. + ext.analyticsEngineZip = project.findProperty('analyticsEngineZip') ?: rootProject.analyticsEngineZipFile + ext.arrowFlightRpcZip = project.findProperty('arrowFlightRpcZip') ?: rootProject.arrowFlightRpcZipFile plugins.withId('java') { java { @@ -173,7 +209,13 @@ allprojects { subprojects { repositories { - mavenLocal() + // Skip mavenLocal for org.opensearch.sandbox.* (analytics-framework, analytics-engine): + // a locally-published OpenSearch core build can install these with Gradle Module Metadata + // declaring different attributes than the published snapshot, which can shadow the remote + // and break resolution. Always pull these from the CI snapshot repo. + mavenLocal { + content { excludeGroup 'org.opensearch.sandbox' } + } mavenCentral() maven { url 'https://jitpack.io' @@ -183,6 +225,33 @@ subprojects { maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" } } + // Sandbox artifacts (analytics-framework, analytics-engine) publish Gradle Module Metadata + // declaring `org.gradle.jvm.version=25` because the sandbox uses the FFM API (finalized in + // JDK 22). sql targets JVM 21, which would make Gradle's variant matcher refuse them. + // Override the published attribute to 21 for these two modules only — safe because: + // - sql code references only non-FFM interfaces of these jars (compile-time API surface). + // - Production runtime targets JVM 25 nodes, where the actual JDK 25 bytecode runs fine. + // - Rule is scoped to `org.opensearch.sandbox:{analytics-framework,analytics-engine}` — + // no effect on any other dependency. + dependencies { + components { + withModule('org.opensearch.sandbox:analytics-framework') { details -> + details.allVariants { + attributes { + attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 21) + } + } + } + withModule('org.opensearch.sandbox:analytics-engine') { details -> + details.allVariants { + attributes { + attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 21) + } + } + } + } + } + // Publish internal modules as Maven artifacts for external use, such as by opensearch-spark and opensearch-cli. def publishedModules = ['api', 'sql', 'ppl', 'core', 'opensearch', 'common', 'protocol', 'datasources', 'legacy'] if (publishedModules.contains(name)) { diff --git a/core/build.gradle b/core/build.gradle index ea0c419b106..aac348dfd1b 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -64,11 +64,19 @@ dependencies { } api 'org.apache.calcite:calcite-linq4j:1.41.0' api project(':common') - compileOnly files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar") + // transitive = false: at runtime, sql's classloader delegates to analytics-engine's via + // `extendedPlugins` parent-first, so the patched calcite (1.41.0-opensearch-1) it transitively + // pulls in must NOT enter sql's compile classpath — sql compiles & bundles vanilla calcite + // 1.41.0, and the AE-supplied patched copy wins at runtime via classloader delegation. + compileOnly('org.opensearch.sandbox:analytics-framework:3.7.0-SNAPSHOT') { + transitive = false + } // Needed because the analytics-framework's QueryPlanExecutor signature uses // org.opensearch.core.action.ActionListener; AnalyticsExecutionEngine references that type. compileOnly group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" - testImplementation files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar") + testImplementation('org.opensearch.sandbox:analytics-framework:3.7.0-SNAPSHOT') { + transitive = false + } testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" implementation "com.github.seancfoley:ipaddress:5.4.2" implementation "com.jayway.jsonpath:json-path:2.9.0" diff --git a/doctest/build.gradle b/doctest/build.gradle index 934ad263a1d..1db6f4de531 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -15,6 +15,11 @@ plugins { apply plugin: 'opensearch.testclusters' +// Ensure both plugin zips are downloaded before any task that boots a test cluster. +tasks.withType(RunTask).configureEach { + dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip' +} + def path = project(':').projectDir // temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from // plugin module, and will only build ppl in it. @@ -195,7 +200,9 @@ testClusters { })) */ plugin(getJobSchedulerPlugin(jsPlugin, bwcOpenSearchJSDownload)) - plugin provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) } + // arrow-flight-rpc must install before analytics-engine (it's an extendedPlugins parent). + plugin provider { (RegularFile) (() -> file("${arrowFlightRpcZip}")) } + plugin provider { (RegularFile) (() -> file("${analyticsEngineZip}")) } plugin ':opensearch-sql-plugin' testDistribution = 'archive' } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 62e1b9861af..d68cc50764f 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -271,7 +271,22 @@ def getGeoSpatialPlugin() { } def getAnalyticsEnginePlugin() { - provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) } + provider { (RegularFile) (() -> file("${analyticsEngineZip}")) } +} + +// arrow-flight-rpc is declared as an extendedPlugins dependency by analytics-engine, so it must +// be installed first or analytics-engine's plugin install fails with `Missing plugin +// [arrow-flight-rpc], dependency of [analytics-engine]`. +def getArrowFlightRpcPlugin() { + provider { (RegularFile) (() -> file("${arrowFlightRpcZip}")) } +} + +// Ensure both plugin zips are downloaded before any task that boots a test cluster. +tasks.withType(StandaloneRestIntegTestTask).configureEach { + dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip' +} +tasks.withType(RestIntegTestTask).configureEach { + dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip' } testClusters { @@ -279,6 +294,7 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin(getGeoSpatialPlugin()) + plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" @@ -287,6 +303,7 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin(getGeoSpatialPlugin()) + plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" @@ -295,18 +312,21 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin(getGeoSpatialPlugin()) + plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } integTestWithSecurity { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) + plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } remoteIntegTestWithSecurity { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) + plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } @@ -362,6 +382,7 @@ stopPrometheus.mustRunAfter startPrometheus task integJdbcTest(type: RestIntegTestTask) { testClusters.findAll {c -> c.clusterName == "integJdbcTest"}.first().with { + plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } diff --git a/libs/analytics-engine-3.7.0-SNAPSHOT.jar b/libs/analytics-engine-3.7.0-SNAPSHOT.jar deleted file mode 100644 index 448f6a421bf..00000000000 Binary files a/libs/analytics-engine-3.7.0-SNAPSHOT.jar and /dev/null differ diff --git a/libs/analytics-engine-3.7.0-SNAPSHOT.zip b/libs/analytics-engine-3.7.0-SNAPSHOT.zip deleted file mode 100644 index 601225ea102..00000000000 Binary files a/libs/analytics-engine-3.7.0-SNAPSHOT.zip and /dev/null differ diff --git a/libs/analytics-framework-3.7.0-SNAPSHOT.jar b/libs/analytics-framework-3.7.0-SNAPSHOT.jar deleted file mode 100644 index a5cfcc32946..00000000000 Binary files a/libs/analytics-framework-3.7.0-SNAPSHOT.jar and /dev/null differ diff --git a/plugin/build.gradle b/plugin/build.gradle index 14b9fadae18..6be054a4a05 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -1,5 +1,8 @@ import java.util.concurrent.Callable import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin +import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.RunTask +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask /* * Copyright OpenSearch Contributors @@ -162,9 +165,15 @@ dependencies { api project(":ppl") api project(':api') // Bundled: analytics-framework interfaces must resolve even when the plugin is absent. - api files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar") + // transitive = false: at runtime sql loads calcite via AE's classloader (extendedPlugins + // parent-first), so sql's classpath must not pull AF's patched-calcite transitive. + api('org.opensearch.sandbox:analytics-framework:3.7.0-SNAPSHOT') { + transitive = false + } // Not bundled: classes here (e.g. OpenSearchSchemaBuilder) only load when the plugin is installed. - compileOnly files("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.jar") + compileOnly('org.opensearch.sandbox:analytics-engine:3.7.0-SNAPSHOT') { + transitive = false + } api project(':legacy') api project(':opensearch') api project(':prometheus') @@ -310,6 +319,10 @@ def getJobSchedulerPlugin() { testClusters.integTest { plugin(getJobSchedulerPlugin()) + // arrow-flight-rpc is declared as an extendedPlugins dependency by analytics-engine, so + // it must be installed first or analytics-engine fails the plugin install jar-hell check + // with `Missing plugin [arrow-flight-rpc], dependency of [analytics-engine]`. + plugin provider { (RegularFile) (() -> file("${arrowFlightRpcZip}")) } plugin provider { (RegularFile) (() -> file("${analyticsEngineZip}")) } plugin(project.tasks.bundlePlugin.archiveFile) testDistribution = "ARCHIVE" @@ -327,3 +340,14 @@ run { useCluster testClusters.integTest } +// Ensure both plugin zips are downloaded before any task that boots a test cluster. +tasks.withType(RunTask).configureEach { + dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip' +} +tasks.withType(StandaloneRestIntegTestTask).configureEach { + dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip' +} +tasks.withType(RestIntegTestTask).configureEach { + dependsOn ':downloadAnalyticsEngineZip', ':downloadArrowFlightRpcZip' +} +