From fe8e3f192a3c81a436410d5f55ccc4b7994350df Mon Sep 17 00:00:00 2001 From: "j.zielinski" Date: Sun, 5 Jul 2026 00:33:13 +0200 Subject: [PATCH] move reusable test-util packages to stream-loaders-test-util --- build.sbt | 44 ++++++++++++++----- .../com/adform/streamloader/Loader.scala | 0 .../behaviors/BasicLoaderBehaviors.scala | 0 .../behaviors/KafkaRestartBehaviors.scala | 0 .../behaviors/RebalanceBehaviors.scala | 0 .../streamloader/fixtures/ClickHouse.scala | 0 .../adform/streamloader/fixtures/Docker.scala | 0 .../adform/streamloader/fixtures/Hdfs.scala | 0 .../adform/streamloader/fixtures/Kafka.scala | 0 .../streamloader/fixtures/Loaders.scala | 0 .../com/adform/streamloader/fixtures/S3.scala | 0 .../streamloader/fixtures/Vertica.scala | 0 .../streamloader/model/StorageMessage.scala | 0 .../storage/JdbcStorageBackend.scala | 0 .../streamloader/storage/StorageBackend.scala | 0 15 files changed, 33 insertions(+), 11 deletions(-) rename {stream-loader-tests => stream-loader-tests-util}/src/main/scala/com/adform/streamloader/Loader.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/fixtures/ClickHouse.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/fixtures/Docker.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/fixtures/Hdfs.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/fixtures/Kafka.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/fixtures/Loaders.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/fixtures/S3.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/fixtures/Vertica.scala (100%) rename {stream-loader-tests => stream-loader-tests-util}/src/main/scala/com/adform/streamloader/model/StorageMessage.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala (100%) rename {stream-loader-tests/src/test => stream-loader-tests-util/src/main}/scala/com/adform/streamloader/storage/StorageBackend.scala (100%) diff --git a/build.sbt b/build.sbt index 8252ba7a..7beba869 100644 --- a/build.sbt +++ b/build.sbt @@ -137,8 +137,40 @@ lazy val dockerImage = settingKey[String]("Full docker image name") val IntegrationTest = config("it").extend(Test) +lazy val `stream-loader-tests-util` = project + .in(file("stream-loader-tests-util")) + .dependsOn(`stream-loader-clickhouse`) + .dependsOn(`stream-loader-hadoop`) + .dependsOn(`stream-loader-iceberg`) + .dependsOn(`stream-loader-s3`) + .dependsOn(`stream-loader-vertica`) + .enablePlugins(BuildInfoPlugin) + .settings(commonSettings) + .settings( + libraryDependencies ++= Seq( + "com.typesafe" % "config" % "1.4.5", + "com.vertica.jdbc" % "vertica-jdbc" % verticaVersion, + "org.log4s" %% "log4s" % "1.10.0", + "org.scalatest" %% "scalatest" % scalaTestVersion, + "org.scalacheck" %% "scalacheck" % scalaCheckVersion, + "org.mandas" % "docker-client" % "9.0.4", + "com.zaxxer" % "HikariCP" % "7.0.2", + "org.duckdb" % "duckdb_jdbc" % duckdbVersion + ), + buildInfoPackage := s"${organization.value}.streamloader", + buildInfoKeys := Seq[BuildInfoKey]( + name, + version, + scalaVersion, + git.gitHeadCommit, + "dockerImage" -> s"adform/stream-loader-tests:${version.value}", + "duckdbVersion" -> duckdbVersion + ) + ) + lazy val `stream-loader-tests` = project .in(file("stream-loader-tests")) + .dependsOn(`stream-loader-tests-util`) .dependsOn(`stream-loader-clickhouse`) .dependsOn(`stream-loader-hadoop`) .dependsOn(`stream-loader-iceberg`) @@ -146,7 +178,6 @@ lazy val `stream-loader-tests` = project .dependsOn(`stream-loader-vertica`) .enablePlugins(PackPlugin) .enablePlugins(DockerPlugin) - .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings(commonSettings) .settings( @@ -171,16 +202,6 @@ lazy val `stream-loader-tests` = project publish := {}, publishLocal := {}, publish / skip := true, - buildInfoPackage := s"${organization.value}.streamloader", - buildInfoKeys := Seq[BuildInfoKey]( - name, - version, - scalaVersion, - sbtVersion, - git.gitHeadCommit, - dockerImage, - "duckdbVersion" -> duckdbVersion - ), packAndSplitJars := { val scalaMajorVersion = scalaVersion.value.split('.').take(2).mkString(".") val mainJar = s"${name.value}_$scalaMajorVersion-${version.value}.jar" @@ -330,5 +351,6 @@ lazy val `stream-loader` = project `stream-loader-iceberg`, `stream-loader-s3`, `stream-loader-vertica`, + `stream-loader-tests-util`, `stream-loader-tests` ) diff --git a/stream-loader-tests/src/main/scala/com/adform/streamloader/Loader.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/Loader.scala similarity index 100% rename from stream-loader-tests/src/main/scala/com/adform/streamloader/Loader.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/Loader.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/ClickHouse.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/ClickHouse.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/ClickHouse.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/ClickHouse.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Docker.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Docker.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Docker.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Docker.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Hdfs.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Hdfs.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Hdfs.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Hdfs.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Kafka.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Kafka.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Kafka.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Kafka.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Loaders.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Loaders.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Loaders.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Loaders.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/S3.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/S3.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/S3.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/S3.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Vertica.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Vertica.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Vertica.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/fixtures/Vertica.scala diff --git a/stream-loader-tests/src/main/scala/com/adform/streamloader/model/StorageMessage.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/model/StorageMessage.scala similarity index 100% rename from stream-loader-tests/src/main/scala/com/adform/streamloader/model/StorageMessage.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/model/StorageMessage.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala diff --git a/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/StorageBackend.scala b/stream-loader-tests-util/src/main/scala/com/adform/streamloader/storage/StorageBackend.scala similarity index 100% rename from stream-loader-tests/src/test/scala/com/adform/streamloader/storage/StorageBackend.scala rename to stream-loader-tests-util/src/main/scala/com/adform/streamloader/storage/StorageBackend.scala