From 12fd7b9ac5677c7a27c5f145ba0fc939993539dd Mon Sep 17 00:00:00 2001 From: petr-pokorny-absa Date: Wed, 15 Apr 2026 15:48:08 +0200 Subject: [PATCH 1/2] support scala 2.13 --- README.md | 16 ++++-- .../org.mockito.plugins.MockMaker | 2 +- .../absa/atum/utils/BuildPropertiesSpec.scala | 2 +- .../ControlMeasureBuilderTest.scala | 4 +- build.sbt | 3 +- .../absa/atum/HdfsInfoIntegrationSuite.scala | 7 +-- project/Dependencies.scala | 51 ++++++++++--------- project/build.properties | 2 +- 8 files changed, 47 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 4892f346..4aaded98 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,14 @@ For project using Scala 2.12 ATUM_VERSION_HERE ``` +For project using Scala 2.13 +```xml + + za.co.absa + atum_2.13 + ATUM_VERSION_HERE + +``` ### Initial info file generation example @@ -305,7 +313,7 @@ Starting with version 3.3.0, there is also persistence support for AWS S3 via AW ```xml za.co.absa - atum-s3-sdk-extension_2.11 + atum-s3-sdk-extension_2.11 ${project.version} ``` @@ -361,13 +369,13 @@ with `3.5.3` and `3.7.0-M15`. ```xml org.json4s - json4s-core_2.11 + json4s-core_2.11 ${json4s.version} provided org.json4s - json4s-jackson_2.11 + json4s-jackson_2.11 ${json4s.version} provided @@ -377,7 +385,7 @@ Then, just include the model library ```xml za.co.absa - atum-model_2.11 + atum-model_2.11 3.5.1 ``` diff --git a/atum/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/atum/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker index 1f0955d4..ca8778f4 100644 --- a/atum/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ b/atum/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -1 +1 @@ -mock-maker-inline +org.mockito.internal.creation.bytebuddy.SubclassByteBuddyMockMaker diff --git a/atum/src/test/scala/za/co/absa/atum/utils/BuildPropertiesSpec.scala b/atum/src/test/scala/za/co/absa/atum/utils/BuildPropertiesSpec.scala index 074aa86b..e1541296 100644 --- a/atum/src/test/scala/za/co/absa/atum/utils/BuildPropertiesSpec.scala +++ b/atum/src/test/scala/za/co/absa/atum/utils/BuildPropertiesSpec.scala @@ -34,6 +34,6 @@ class BuildPropertiesSpec extends AnyFlatSpec { } "Project Name" should "start with atum and scala version" in { - assert(name.matches("""^atum_(2\.11|2\.12)$""")) + assert(name.matches("""^atum_(2\.11|2\.12|2\.13)$""")) } } diff --git a/atum/src/test/scala/za/co/absa/atum/utils/controlmeasure/ControlMeasureBuilderTest.scala b/atum/src/test/scala/za/co/absa/atum/utils/controlmeasure/ControlMeasureBuilderTest.scala index 5751aed9..10a16a77 100644 --- a/atum/src/test/scala/za/co/absa/atum/utils/controlmeasure/ControlMeasureBuilderTest.scala +++ b/atum/src/test/scala/za/co/absa/atum/utils/controlmeasure/ControlMeasureBuilderTest.scala @@ -58,7 +58,7 @@ class ControlMeasureBuilderTest extends AnyFlatSpec with ControlMeasureBaseTestS // prior to stabilization, let's check the actual by-default generated software fields: defaultCm.checkpoints.map(_.software).foreach { swName => swName shouldBe defined - swName.get should fullyMatch regex ("""^atum_(2\.11|2\.12)$""") + swName.get should fullyMatch regex ("""^atum_(2\.11|2\.12|2\.13)$""") } defaultCm.stabilizeTestingControlMeasure shouldBe expectedDefaultControlMeasure @@ -101,7 +101,7 @@ class ControlMeasureBuilderTest extends AnyFlatSpec with ControlMeasureBaseTestS // prior to stabilization, let's check the actual by-default generated software fields: customCm.checkpoints.map(_.software).foreach { swName => swName shouldBe defined - swName.get should fullyMatch regex ("""^atum_(2\.11|2\.12)$""") + swName.get should fullyMatch regex ("""^atum_(2\.11|2\.12|2\.13)$""") } customCm.stabilizeTestingControlMeasure shouldBe expectedCustomControlMeasure diff --git a/build.sbt b/build.sbt index 481c1392..dc08112b 100644 --- a/build.sbt +++ b/build.sbt @@ -22,9 +22,10 @@ import com.github.sbt.jacoco.report.JacocoReportSettings lazy val scala211 = "2.11.12" lazy val scala212 = "2.12.15" +lazy val scala213 = "2.13.14" ThisBuild / scalaVersion := scala211 // default version -ThisBuild / crossScalaVersions := Seq(scala211, scala212) +ThisBuild / crossScalaVersions := Seq(scala211, scala212, scala213) lazy val printSparkScalaVersion = taskKey[Unit]("Print Spark and Scala versions that Atum is being built for.") ThisBuild / printSparkScalaVersion := { diff --git a/examples/src/test/scala/za/co/absa/atum/HdfsInfoIntegrationSuite.scala b/examples/src/test/scala/za/co/absa/atum/HdfsInfoIntegrationSuite.scala index 4efcfeea..94918081 100644 --- a/examples/src/test/scala/za/co/absa/atum/HdfsInfoIntegrationSuite.scala +++ b/examples/src/test/scala/za/co/absa/atum/HdfsInfoIntegrationSuite.scala @@ -22,17 +22,14 @@ import org.apache.log4j.LogManager import org.apache.spark.sql.{DataFrame, SaveMode} import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.Eventually -import org.scalatest.concurrent.PatienceConfiguration.Timeout -import org.scalatest.flatspec.{AnyFlatSpec, AsyncFlatSpec} +import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import org.specs2.matcher.Matchers.concurrentExecutionContext import za.co.absa.atum.model.{Checkpoint, Measurement} import za.co.absa.atum.persistence.ControlMeasuresParser import za.co.absa.atum.utils.SparkTestBase import za.co.absa.atum.AtumImplicits._ -import scala.concurrent.duration.{Duration, DurationInt} -import scala.concurrent.{Await, Future} +import scala.concurrent.duration.DurationInt class HdfsInfoIntegrationSuite extends AnyFlatSpec with SparkTestBase with Matchers with BeforeAndAfterAll with Eventually { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 09d0ddf3..f3a565b8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,20 +20,23 @@ object Dependencies { object Versions { val spark2 = "2.4.8" val spark3 = "3.2.2" + val spark3_213 = "3.5.1" val json4s_spark2 = "3.5.3" val json4s_spark3 = "3.7.0-M11" + val json4s_spark3_213 = json4s_spark3 val jacksonModuleScala_spark2 = "2.12.7" val jacksonModuleScala_spark3 = "2.14.1" + val jacksonModuleScala_spark3_213 = jacksonModuleScala_spark3 val jacksonDatabind_spark2 = "2.12.7.1" // databind has extra extra patch for this version - for Spark2 val jacksonDatabind_spark3 = jacksonModuleScala_spark3 // for Spark3 - latest version is ok + val jacksonDatabind_spark3_213 = jacksonModuleScala_spark3_213 - val absaCommons = "0.0.27" + val absaCommons = "2.0.4" val typesafeConfig = "1.4.1" val mockitoScala = "1.17.12" val scalatest = "3.2.9" - val specs2 = "2.5" val aws = "2.17.85" val apacheCommonsLang3 = "3.12.0" @@ -47,54 +50,55 @@ object Dependencies { scalaVersion match { case _ if scalaVersion.startsWith("2.11") => Versions.spark2 case _ if scalaVersion.startsWith("2.12") => Versions.spark3 - case _ => throw new IllegalArgumentException("Only Scala 2.11 and 2.12 are currently supported.") + case _ if scalaVersion.startsWith("2.13") => Versions.spark3_213 + case _ => throw new IllegalArgumentException("Only Scala 2.11, 2.12 and 2.13 are currently supported.") } } - // general wrapper to simplify s2.11/2.12 version assigning + // general wrapper to simplify s2.11/2.12/2.13 version assigning def moduleByScala(moduleIdWithoutVersion: String => ModuleID) - (scala211Version: String, scala212Version: String) + (scala211Version: String, scala212Version: String, scala213Version: String) (actualScalaVersion: String): ModuleID = { actualScalaVersion match { - case _ if actualScalaVersion.startsWith("2.11") => moduleIdWithoutVersion.apply(scala211Version) - case _ if actualScalaVersion.startsWith("2.12") => moduleIdWithoutVersion.apply(scala212Version) - case _ => throw new IllegalArgumentException("Only Scala 2.11 and 2.12 are currently supported.") + case _ if actualScalaVersion.startsWith("2.11") => moduleIdWithoutVersion(scala211Version) + case _ if actualScalaVersion.startsWith("2.12") => moduleIdWithoutVersion(scala212Version) + case _ if actualScalaVersion.startsWith("2.13") => moduleIdWithoutVersion(scala213Version) + case _ => throw new IllegalArgumentException("Only Scala 2.11, 2.12 and 2.13 are currently supported.") } } - // extended version where to moduleId Fn takes 2 params: module version and scala version (to pass along) def moduleByScalaUsingScalaVersion(moduleIdWithoutVersionNeedsScalaVersion: (String, String) => ModuleID) - (scala211Version: String, scala212Version: String) + (scala211Version: String, scala212Version: String, scala213Version: String) (actualScalaVersion: String): ModuleID = { actualScalaVersion match { - case _ if actualScalaVersion.startsWith("2.11") => moduleIdWithoutVersionNeedsScalaVersion.apply(scala211Version, actualScalaVersion) - case _ if actualScalaVersion.startsWith("2.12") => moduleIdWithoutVersionNeedsScalaVersion.apply(scala212Version, actualScalaVersion) - case _ => throw new IllegalArgumentException("Only Scala 2.11 and 2.12 are currently supported.") + case _ if actualScalaVersion.startsWith("2.11") => moduleIdWithoutVersionNeedsScalaVersion(scala211Version, actualScalaVersion) + case _ if actualScalaVersion.startsWith("2.12") => moduleIdWithoutVersionNeedsScalaVersion(scala212Version, actualScalaVersion) + case _ if actualScalaVersion.startsWith("2.13") => moduleIdWithoutVersionNeedsScalaVersion(scala213Version, actualScalaVersion) + case _ => throw new IllegalArgumentException("Only Scala 2.11, 2.12 and 2.13 are currently supported.") } } - lazy val sparkCore = { def coreWithExcludes(version: String, scalaVersion: String): ModuleID = "org.apache.spark" %% "spark-core" % version % Provided exclude( "com.fasterxml.jackson.core", "jackson-databind" ) exclude( "com.fasterxml.jackson.module", "jackson-module-scala_" + scalaVersion.substring(0, 4) // e.g. 2.11 ) - moduleByScalaUsingScalaVersion(coreWithExcludes)(Versions.spark2, Versions.spark3) _ + moduleByScalaUsingScalaVersion(coreWithExcludes)(Versions.spark2, Versions.spark3, Versions.spark3_213) _ } - lazy val sparkSql = moduleByScala("org.apache.spark" %% "spark-sql" % _ % Provided)(Versions.spark2, Versions.spark3) _ + lazy val sparkSql = moduleByScala("org.apache.spark" %% "spark-sql" % _ % Provided)(Versions.spark2, Versions.spark3, Versions.spark3_213) _ lazy val scalaTest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test - lazy val json4sExt = moduleByScala("org.json4s" %% "json4s-ext" % _)(Versions.json4s_spark2, Versions.json4s_spark3) _ - lazy val json4sCore = moduleByScala("org.json4s" %% "json4s-core" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3) _ - lazy val json4sJackson = moduleByScala("org.json4s" %% "json4s-jackson" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3) _ - lazy val json4sNative = moduleByScala("org.json4s" %% "json4s-native" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3) _ + lazy val json4sExt = moduleByScala("org.json4s" %% "json4s-ext" % _)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ + lazy val json4sCore = moduleByScala("org.json4s" %% "json4s-core" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ + lazy val json4sJackson = moduleByScala("org.json4s" %% "json4s-jackson" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ + lazy val json4sNative = moduleByScala("org.json4s" %% "json4s-native" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ - lazy val jacksonModuleScala = moduleByScala("com.fasterxml.jackson.module" %% "jackson-module-scala" % _)(Versions.jacksonModuleScala_spark2, Versions.jacksonModuleScala_spark3) _ - lazy val jacksonDatabind = moduleByScala("com.fasterxml.jackson.core" % "jackson-databind" % _)(Versions.jacksonDatabind_spark2, Versions.jacksonDatabind_spark3) _ + lazy val jacksonModuleScala = moduleByScala("com.fasterxml.jackson.module" %% "jackson-module-scala" % _)(Versions.jacksonModuleScala_spark2, Versions.jacksonModuleScala_spark3, Versions.jacksonModuleScala_spark3_213) _ + lazy val jacksonDatabind = moduleByScala("com.fasterxml.jackson.core" % "jackson-databind" % _)(Versions.jacksonDatabind_spark2, Versions.jacksonDatabind_spark3, Versions.jacksonDatabind_spark3_213) _ lazy val absaCommons = "za.co.absa.commons" %% "commons" % Versions.absaCommons lazy val commonsConfiguration = "commons-configuration" % "commons-configuration" % Versions.commonsConfiguration @@ -105,8 +109,6 @@ object Dependencies { lazy val mockitoScalaScalatest = "org.mockito" %% "mockito-scala-scalatest" % Versions.mockitoScala % Test lazy val scalaTestProvided = "org.scalatest" %% "scalatest" % Versions.scalatest % Provided - lazy val specs2core = "org.specs2" %% "specs2-core" % Versions.specs2 % Test - lazy val sdkS3 = "software.amazon.awssdk" % "s3" % Versions.aws def rootDependencies(scalaVersion: String): Seq[ModuleID] = Seq( @@ -135,7 +137,6 @@ object Dependencies { ) lazy val examplesDependencies: Seq[ModuleID] = Seq( - specs2core, scalaTestProvided ) diff --git a/project/build.properties b/project/build.properties index 5afecec9..683dc3d5 100644 --- a/project/build.properties +++ b/project/build.properties @@ -13,4 +13,4 @@ # limitations under the License. # -sbt.version = 1.5.5 +sbt.version = 1.9.9 From 721c0ab525669c1fb5e977aa5d2de2599c6bcc86 Mon Sep 17 00:00:00 2001 From: petr-pokorny-absa Date: Wed, 15 Apr 2026 16:01:19 +0200 Subject: [PATCH 2/2] use spark 3.2.2 from scala 2.13, update gh actions --- .github/workflows/build-sbt.yml | 4 +++- .github/workflows/jacoco_check.yml | 5 +++++ project/Dependencies.scala | 22 +++++++++------------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-sbt.yml b/.github/workflows/build-sbt.yml index 48d2a871..de30bb74 100644 --- a/.github/workflows/build-sbt.yml +++ b/.github/workflows/build-sbt.yml @@ -27,13 +27,15 @@ jobs: strategy: fail-fast: false matrix: - scala: [ 2.11.12, 2.12.15 ] + scala: [ 2.11.12, 2.12.15, 2.13.14 ] spark: [ 2.4.8, 3.2.2 ] exclude: - scala: 2.11.12 spark: 3.2.2 - scala: 2.12.15 spark: 2.4.8 + - scala: 2.13.14 + spark: 2.4.8 name: SBT Spark ${{matrix.spark}} on Scala ${{matrix.scala}} steps: - name: Checkout code diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index d1e0909c..f5babf2e 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -37,6 +37,11 @@ jobs: spark: 3.2.2 overall: 0.0 changed: 80.0 + - scala: 2.13.14 + scala_short: 2.13 + spark: 3.2.2 + overall: 0.0 + changed: 80.0 steps: - name: Checkout code uses: actions/checkout@v2 diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f3a565b8..11f68d7b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,18 +20,14 @@ object Dependencies { object Versions { val spark2 = "2.4.8" val spark3 = "3.2.2" - val spark3_213 = "3.5.1" val json4s_spark2 = "3.5.3" val json4s_spark3 = "3.7.0-M11" - val json4s_spark3_213 = json4s_spark3 val jacksonModuleScala_spark2 = "2.12.7" val jacksonModuleScala_spark3 = "2.14.1" - val jacksonModuleScala_spark3_213 = jacksonModuleScala_spark3 val jacksonDatabind_spark2 = "2.12.7.1" // databind has extra extra patch for this version - for Spark2 val jacksonDatabind_spark3 = jacksonModuleScala_spark3 // for Spark3 - latest version is ok - val jacksonDatabind_spark3_213 = jacksonModuleScala_spark3_213 val absaCommons = "2.0.4" val typesafeConfig = "1.4.1" @@ -50,7 +46,7 @@ object Dependencies { scalaVersion match { case _ if scalaVersion.startsWith("2.11") => Versions.spark2 case _ if scalaVersion.startsWith("2.12") => Versions.spark3 - case _ if scalaVersion.startsWith("2.13") => Versions.spark3_213 + case _ if scalaVersion.startsWith("2.13") => Versions.spark3 case _ => throw new IllegalArgumentException("Only Scala 2.11, 2.12 and 2.13 are currently supported.") } } @@ -85,20 +81,20 @@ object Dependencies { ) exclude( "com.fasterxml.jackson.module", "jackson-module-scala_" + scalaVersion.substring(0, 4) // e.g. 2.11 ) - moduleByScalaUsingScalaVersion(coreWithExcludes)(Versions.spark2, Versions.spark3, Versions.spark3_213) _ + moduleByScalaUsingScalaVersion(coreWithExcludes)(Versions.spark2, Versions.spark3, Versions.spark3) _ } - lazy val sparkSql = moduleByScala("org.apache.spark" %% "spark-sql" % _ % Provided)(Versions.spark2, Versions.spark3, Versions.spark3_213) _ + lazy val sparkSql = moduleByScala("org.apache.spark" %% "spark-sql" % _ % Provided)(Versions.spark2, Versions.spark3, Versions.spark3) _ lazy val scalaTest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test - lazy val json4sExt = moduleByScala("org.json4s" %% "json4s-ext" % _)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ - lazy val json4sCore = moduleByScala("org.json4s" %% "json4s-core" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ - lazy val json4sJackson = moduleByScala("org.json4s" %% "json4s-jackson" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ - lazy val json4sNative = moduleByScala("org.json4s" %% "json4s-native" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3_213) _ + lazy val json4sExt = moduleByScala("org.json4s" %% "json4s-ext" % _)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3) _ + lazy val json4sCore = moduleByScala("org.json4s" %% "json4s-core" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3) _ + lazy val json4sJackson = moduleByScala("org.json4s" %% "json4s-jackson" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3) _ + lazy val json4sNative = moduleByScala("org.json4s" %% "json4s-native" % _ % Provided)(Versions.json4s_spark2, Versions.json4s_spark3, Versions.json4s_spark3) _ - lazy val jacksonModuleScala = moduleByScala("com.fasterxml.jackson.module" %% "jackson-module-scala" % _)(Versions.jacksonModuleScala_spark2, Versions.jacksonModuleScala_spark3, Versions.jacksonModuleScala_spark3_213) _ - lazy val jacksonDatabind = moduleByScala("com.fasterxml.jackson.core" % "jackson-databind" % _)(Versions.jacksonDatabind_spark2, Versions.jacksonDatabind_spark3, Versions.jacksonDatabind_spark3_213) _ + lazy val jacksonModuleScala = moduleByScala("com.fasterxml.jackson.module" %% "jackson-module-scala" % _)(Versions.jacksonModuleScala_spark2, Versions.jacksonModuleScala_spark3, Versions.jacksonModuleScala_spark3) _ + lazy val jacksonDatabind = moduleByScala("com.fasterxml.jackson.core" % "jackson-databind" % _)(Versions.jacksonDatabind_spark2, Versions.jacksonDatabind_spark3, Versions.jacksonDatabind_spark3) _ lazy val absaCommons = "za.co.absa.commons" %% "commons" % Versions.absaCommons lazy val commonsConfiguration = "commons-configuration" % "commons-configuration" % Versions.commonsConfiguration