From 86c9a6827f0dc814a65d0171f468d784416d15e8 Mon Sep 17 00:00:00 2001 From: Ivan Lapa Date: Mon, 4 May 2026 16:32:44 +0300 Subject: [PATCH 1/6] ADH-8212: Update dependencies: Add Hadoop Yarn dependencies and configure default activation for Hadoop 3.3 profile --- pom.xml | 47 ++++++++++++++++++++++++++++++++++++ smart-hadoop-support/pom.xml | 3 +++ 2 files changed, 50 insertions(+) diff --git a/pom.xml b/pom.xml index 805530a8a0..82d9cd2900 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ 1.20.4 1.68 3.7.1 + com.google.protobuf:protoc:${protobuf.version} 2.5.3 2.5.32 1.2.1 @@ -150,6 +151,7 @@ false + @@ -480,6 +482,51 @@ + + org.apache.hadoop + hadoop-registry + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-registry + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-server-applicationhistoryservice + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-server-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-server-web-proxy + ${hadoop.version} + org.aspectj diff --git a/smart-hadoop-support/pom.xml b/smart-hadoop-support/pom.xml index b16ae2b5b8..c093b35d10 100644 --- a/smart-hadoop-support/pom.xml +++ b/smart-hadoop-support/pom.xml @@ -42,6 +42,9 @@ hadoop-3.3 + + true + smart-hadoop-3 smart-hadoop-3.3 From 6778dc2f413bb57f576365afedf36d114b636dba Mon Sep 17 00:00:00 2001 From: Ivan Lapa Date: Mon, 4 May 2026 16:32:55 +0300 Subject: [PATCH 2/6] ADH-8213: Add macOS Apple Silicon-specific profile for protoc configuration --- smart-common/pom.xml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/smart-common/pom.xml b/smart-common/pom.xml index 1e136d2152..7f906daf8c 100644 --- a/smart-common/pom.xml +++ b/smart-common/pom.xml @@ -148,7 +148,7 @@ run - com.google.protobuf:protoc:${protobuf.version} + ${protoc.artifact} ${protobuf.version} ${basedir}/src/main/proto @@ -260,4 +260,19 @@ + + + + apple-silicon-protoc + + + mac + aarch64 + + + + com.google.protobuf:protoc:${protobuf.version}:exe:osx-x86_64 + + + From 1bcb08a496a3de8a9a5acc706a03b472bb644dfa Mon Sep 17 00:00:00 2001 From: Ivan Lapa Date: Tue, 5 May 2026 13:02:35 +0300 Subject: [PATCH 3/6] ADH-8852: Update dependencies: Testcontainers, Docker Java, and Commons Lang; catch Throwable in SmartServer for broader error handling --- pom.xml | 23 +++++++++++++++++-- .../org/smartdata/server/SmartServer.java | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 82d9cd2900..03e6e7fc3b 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,8 @@ 2.11.12 2.11 2.28.2 - 1.20.4 + 1.21.4 + 3.7.1 1.68 3.7.1 com.google.protobuf:protoc:${protobuf.version} @@ -110,7 +111,7 @@ 2.7.18 1.0.1.RELEASE 6.6.0 - 3.12.0 + 3.18.0 1.5.5.Final 0.2.0 1.9.22 @@ -651,6 +652,24 @@ ${testcontainers.version} test + + com.github.docker-java + docker-java-api + ${docker-java.version} + test + + + com.github.docker-java + docker-java-transport + ${docker-java.version} + test + + + com.github.docker-java + docker-java-transport-zerodep + ${docker-java.version} + test + junit junit diff --git a/smart-server/src/main/java/org/smartdata/server/SmartServer.java b/smart-server/src/main/java/org/smartdata/server/SmartServer.java index 44eddf1e44..a65b188e81 100644 --- a/smart-server/src/main/java/org/smartdata/server/SmartServer.java +++ b/smart-server/src/main/java/org/smartdata/server/SmartServer.java @@ -372,7 +372,7 @@ public void run() { Thread.sleep(1000); } } - } catch (Exception e) { + } catch (Throwable e) { LOG.error("Failed to create SmartServer", e); errorCode = 1; } finally { From a45a9e3a2e93138d9e1a851877a0fab44ee7f1d0 Mon Sep 17 00:00:00 2001 From: Ivan Lapa Date: Tue, 5 May 2026 13:16:08 +0300 Subject: [PATCH 4/6] Revert "ADH-8212: Update dependencies: Add Hadoop Yarn dependencies and configure default activation for Hadoop 3.3 profile" This reverts commit 86c9a6827f0dc814a65d0171f468d784416d15e8. --- pom.xml | 47 ------------------------------------ smart-hadoop-support/pom.xml | 3 --- 2 files changed, 50 deletions(-) diff --git a/pom.xml b/pom.xml index 03e6e7fc3b..405636922e 100644 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,6 @@ 3.7.1 1.68 3.7.1 - com.google.protobuf:protoc:${protobuf.version} 2.5.3 2.5.32 1.2.1 @@ -152,7 +151,6 @@ false - @@ -483,51 +481,6 @@ - - org.apache.hadoop - hadoop-registry - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-registry - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-common - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-client - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-server-applicationhistoryservice - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-server-common - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-server-resourcemanager - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-server-web-proxy - ${hadoop.version} - org.aspectj diff --git a/smart-hadoop-support/pom.xml b/smart-hadoop-support/pom.xml index c093b35d10..b16ae2b5b8 100644 --- a/smart-hadoop-support/pom.xml +++ b/smart-hadoop-support/pom.xml @@ -42,9 +42,6 @@ hadoop-3.3 - - true - smart-hadoop-3 smart-hadoop-3.3 From 01bc8c1f09827dce7611dd0a23a3f95ed95e8474 Mon Sep 17 00:00:00 2001 From: Ivan Lapa Date: Tue, 5 May 2026 13:57:01 +0300 Subject: [PATCH 5/6] ADH-8852: Add wait_for_file function, enhance service dependencies in docker-compose, and update runtime configurations --- pom.xml | 1 + supports/tools/docker/multihost/common.sh | 24 +++++++- .../datanode/hadoop-datanode-entrypoint.sh | 14 ++++- .../docker/multihost/docker-compose.yaml | 57 +++++++++++++++---- .../namenode/hadoop-namenode-entrypoint.sh | 7 ++- .../multihost/ssm/ssm-server-entrypoint.sh | 19 ++++++- 6 files changed, 104 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 405636922e..8058133481 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,7 @@ 3.7.1 1.68 3.7.1 + com.google.protobuf:protoc:${protobuf.version} 2.5.3 2.5.32 1.2.1 diff --git a/supports/tools/docker/multihost/common.sh b/supports/tools/docker/multihost/common.sh index c967002991..0f236ba2c1 100755 --- a/supports/tools/docker/multihost/common.sh +++ b/supports/tools/docker/multihost/common.sh @@ -30,6 +30,28 @@ function wait_for_it() echo "[$i/$max_try] $service:${port} is available." } +function wait_for_file() +{ + local path=$1 + local retry_seconds=5 + local max_try=100 + let i=1 + + until [ -f "$path" ]; do + echo "[$i/$max_try] check for ${path}..." + echo "[$i/$max_try] ${path} is not available yet" + if (( $i == $max_try )); then + echo "[$i/$max_try] ${path} is still not available; giving up after ${max_try} tries. :/" + exit 1 + fi + + echo "[$i/$max_try] try in ${retry_seconds}s once again ..." + let "i++" + sleep $retry_seconds + done + echo "[$i/$max_try] $path is available." +} + function addProperty() { local path=$1 local name=$2 @@ -63,4 +85,4 @@ function configure() { echo " - Setting $name=$value" addProperty $path $name "$value" done -} \ No newline at end of file +} diff --git a/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh b/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh index daa24fa3f7..e04b9dbc3f 100755 --- a/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh +++ b/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh @@ -2,12 +2,23 @@ . ./common.sh +SSM_RUNTIME_CONF_DIR=/tmp/ssm-conf +rm -rf "$SSM_RUNTIME_CONF_DIR" +mkdir -p "$SSM_RUNTIME_CONF_DIR" +cp -R /opt/ssm/conf/. "$SSM_RUNTIME_CONF_DIR"/ +cp /opt/ssm/smart-site.xml "$SSM_RUNTIME_CONF_DIR"/smart-site.xml + cp /etc/ssm/shared/id_rsa /root/.ssh/id_rsa cp /etc/ssm/shared/id_rsa.pub /root/.ssh/id_rsa.pub cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys service ssh start ssh-keyscan ssm-server.demo >> /root/.ssh/known_hosts echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc +echo "export SMART_CONF_DIR=${SSM_RUNTIME_CONF_DIR}/" >> /root/.bashrc + +wait_for_file /etc/secrets/datanode.keytab +wait_for_file /etc/secrets/agent.keytab +wait_for_file /etc/secrets/http.keytab datadir=`echo $HDFS_CONF_dfs_datanode_data_dir | perl -pe 's#file://##'` if [ ! -d $datadir ]; then @@ -18,7 +29,8 @@ fi chmod +r /etc/secrets/*.keytab moveHadoopConfFiles /etc/conf ${HADOOP_CONF_DIR} +configure "$HADOOP_CONF_DIR"/hdfs-site.xml hdfs HDFS_CONF $HADOOP_HOME/bin/hdfs --debug --config $HADOOP_CONF_DIR datanode -tail -f /dev/null \ No newline at end of file +tail -f /dev/null diff --git a/supports/tools/docker/multihost/docker-compose.yaml b/supports/tools/docker/multihost/docker-compose.yaml index 6da25745b2..5b3a6c50d3 100644 --- a/supports/tools/docker/multihost/docker-compose.yaml +++ b/supports/tools/docker/multihost/docker-compose.yaml @@ -31,7 +31,8 @@ services: networks: - demo depends_on: - - hadoop-datanode + kdc-server: + condition: service_healthy hadoop-datanode: image: hub.adsw.io/ssm/ssm-hadoop-base:${HADOOP_VERSION:-3.4.3} @@ -48,7 +49,7 @@ services: - ./kerberos/krb5.conf:/etc/krb5.conf - ./conf:/etc/conf - ./conf:/opt/ssm/conf - - ./ssm-conf/smart-site-agent.xml:/opt/ssm/conf/smart-site.xml + - ./ssm-conf/smart-site-agent.xml:/opt/ssm/smart-site.xml - ./datanode/hadoop-datanode-entrypoint.sh:/hadoop-datanode-entrypoint.sh entrypoint: ["/hadoop-datanode-entrypoint.sh"] ports: @@ -70,7 +71,12 @@ services: networks: - demo depends_on: - - ssm-server + hadoop-namenode: + condition: service_healthy + kdc-server: + condition: service_healthy + ssm-server: + condition: service_started ssm-server: image: hub.adsw.io/ssm/ssm-server:${SSM_VERSION:-2.3.0-SNAPSHOT} @@ -82,7 +88,7 @@ services: - secrets:/etc/secrets - ./kerberos/krb5.conf:/etc/krb5.conf - ./conf:/opt/ssm/conf - - ./ssm-conf/smart-site-master.xml:/opt/ssm/conf/smart-site.xml + - ./ssm-conf/smart-site-master.xml:/opt/ssm/smart-site.xml - ./common.sh:/common.sh - ./ssm/ssm-server-entrypoint.sh:/ssm-server-entrypoint.sh entrypoint: ["/ssm-server-entrypoint.sh"] @@ -96,15 +102,23 @@ services: environment: SSM_DEBUG_OPT: ${SSM_DEBUG_OPT} healthcheck: - test: ["CMD-SHELL", "curl http://ssm-server.demo:8081 || exit 1"] + test: ["CMD-SHELL", "curl -fsS http://ssm-server.demo:8081/actuator/health || exit 1"] interval: 30s timeout: 15s retries: 5 networks: - demo depends_on: - - ssm-metastore-db - - kdc-server + ssm-metastore-db: + condition: service_healthy + kdc-server: + condition: service_healthy + samba: + condition: service_healthy + hadoop-namenode: + condition: service_healthy + hive-metastore: + condition: service_healthy ssm-metastore-db: image: "docker.io/library/postgres:14.0" @@ -173,6 +187,14 @@ services: image: apache/hive:4.0.1 container_name: hive-metastore hostname: hive-metastore.demo + restart: unless-stopped + depends_on: + ssm-metastore-db: + condition: service_healthy + kdc-server: + condition: service_healthy + hadoop-namenode: + condition: service_healthy environment: SERVICE_NAME: metastore DB_DRIVER: postgres @@ -190,6 +212,8 @@ services: - ./kerberos/krb5.conf:/etc/krb5.conf - ./hive/hive-site.xml:/opt/hive/conf/hive-site.xml - ./hive/core-site.xml:/opt/hive/conf/core-site.xml + - ./conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml + - ./conf/yarn-site.xml:/opt/hive/conf/yarn-site.xml ports: - "9083:9083" healthcheck: @@ -198,15 +222,22 @@ services: timeout: 15s retries: 5 networks: - - demo + demo: + aliases: + - hive-metastore.demo hive-server2: image: apache/hive:4.0.1 container_name: hive-server2 hostname: hive-server2.demo + restart: unless-stopped depends_on: - - hive-metastore - - hadoop-namenode + hive-metastore: + condition: service_healthy + hadoop-namenode: + condition: service_healthy + kdc-server: + condition: service_healthy environment: SERVICE_NAME: hiveserver2 IS_RESUME: true @@ -214,15 +245,17 @@ services: -Dhive.metastore.uris=thrift://hive-metastore.demo:9083 -Dhive.metastore.warehouse.dir=hdfs://hadoop-namenode.demo:8020/user/hive/warehouse healthcheck: - test: [ "CMD", "sh", "-c", "beeline -u 'jdbc:hive2://localhost:10000/'" ] + test: [ "CMD", "sh", "-c", "/opt/hive/bin/beeline -u 'jdbc:hive2://localhost:10000/default' -e 'show databases;'" ] interval: 30s timeout: 10s - retries: 3 + retries: 5 volumes: - secrets:/etc/secrets - ./kerberos/krb5.conf:/etc/krb5.conf - ./hive/hive-site.xml:/opt/hive/conf/hive-site.xml - ./hive/core-site.xml:/opt/hive/conf/core-site.xml + - ./conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml + - ./conf/yarn-site.xml:/opt/hive/conf/yarn-site.xml ports: - "10000:10000" - "10002:10002" diff --git a/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh b/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh index 4b11fc6359..5acffdab35 100755 --- a/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh +++ b/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh @@ -6,6 +6,10 @@ service ssh start ssh-keyscan ssm-server.demo >> /root/.ssh/known_hosts echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc +wait_for_file /etc/secrets/namenode.keytab +wait_for_file /etc/secrets/yarn.keytab +chmod +r /etc/secrets/*.keytab + namedir=`echo $HDFS_CONF_dfs_namenode_name_dir | perl -pe 's#file://##'` if [ ! -d $namedir ]; then echo "Namenode name directory not found: $namedir" @@ -18,6 +22,7 @@ if [ -z "$CLUSTER_NAME" ]; then fi moveHadoopConfFiles /etc/conf ${HADOOP_CONF_DIR} +configure "$HADOOP_CONF_DIR"/hdfs-site.xml hdfs HDFS_CONF # HDFS addProperty "$HADOOP_CONF_DIR"/hdfs-site.xml dfs.namenode.rpc-bind-host 0.0.0.0 @@ -61,4 +66,4 @@ echo "----------------------" $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager & wait_for_it $(hostname -f):8088 -tail -f /dev/null \ No newline at end of file +tail -f /dev/null diff --git a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh index 61d9ab116d..5ba8d160db 100755 --- a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh +++ b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh @@ -2,13 +2,19 @@ . ./common.sh +SSM_RUNTIME_CONF_DIR=/tmp/ssm-conf +rm -rf "$SSM_RUNTIME_CONF_DIR" +mkdir -p "$SSM_RUNTIME_CONF_DIR" +cp -R "$SSM_HOME"/conf/. "$SSM_RUNTIME_CONF_DIR"/ +cp "$SSM_HOME"/smart-site.xml "$SSM_RUNTIME_CONF_DIR"/smart-site.xml + cp /root/.ssh/id_rsa /tmp/shared/id_rsa cp /root/.ssh/id_rsa.pub /tmp/shared/id_rsa.pub service ssh start ssh-keyscan "$HOSTNAME" >> /root/.ssh/known_hosts echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc echo "export SMART_HOME=${SSM_HOME}" >> /root/.bashrc -echo "export SMART_CONF_DIR=${SSM_HOME}/conf/" >> /root/.bashrc +echo "export SMART_CONF_DIR=${SSM_RUNTIME_CONF_DIR}/" >> /root/.bashrc # Starting Smart Storage Manager cd $SSM_HOME || exit @@ -17,8 +23,15 @@ echo "---------------------------" echo "Starting SSM server and agents" echo "---------------------------" -source bin/start-ssm.sh ${SSM_DEBUG_OPT} --config ${SSM_HOME}/conf/ & +wait_for_file /etc/secrets/ssm.keytab +wait_for_file /etc/secrets/http.keytab +wait_for_it ssm-metastore-db.demo:5432 +wait_for_it samba:389 +wait_for_it hadoop-namenode.demo:8020 +wait_for_it hadoop-datanode.demo:22 + +source bin/start-ssm.sh ${SSM_DEBUG_OPT} --config ${SSM_RUNTIME_CONF_DIR}/ & wait_for_it $(hostname -f):8081 wait_for_it hadoop-datanode.demo:7048 -tail -f /var/log/ssm/* \ No newline at end of file +tail -f /var/log/ssm/* From 1fe21f6d95d026cba88ee3d12dc44dba239e5c01 Mon Sep 17 00:00:00 2001 From: Ivan Lapa Date: Wed, 6 May 2026 19:15:28 +0300 Subject: [PATCH 6/6] ADH-8212: Fix Hive exec commons-lang runtime conflict Add smart-hive-exec module that repackages Hive exec without embedded commons-lang and commons-lang3 classes. --- pom.xml | 1 + smart-hive-exec/pom.xml | 89 ++++++++++++++++++++++++++++++++++++++ smart-hive-support/pom.xml | 8 +++- 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 smart-hive-exec/pom.xml diff --git a/pom.xml b/pom.xml index 8058133481..ff19c7951e 100644 --- a/pom.xml +++ b/pom.xml @@ -67,6 +67,7 @@ smart-web-server smart-hadoop-support smart-tests + smart-hive-exec smart-hive-support diff --git a/smart-hive-exec/pom.xml b/smart-hive-exec/pom.xml new file mode 100644 index 0000000000..08f7f59fc1 --- /dev/null +++ b/smart-hive-exec/pom.xml @@ -0,0 +1,89 @@ + + + + 4.0.0 + + org.smartdata + smartdata-project + 2.3.0-SNAPSHOT + .. + + + smart-hive-exec + 2.3.0-SNAPSHOT + jar + + + + org.apache.hive + hive-exec + ${hive.version} + true + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + false + + + org.apache.hive:hive-exec + + + + + org.apache.hive:hive-exec + + org/apache/commons/lang/** + org/apache/commons/lang3/** + META-INF/maven/commons-lang/commons-lang/** + META-INF/maven/org.apache.commons/commons-lang3/** + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/smart-hive-support/pom.xml b/smart-hive-support/pom.xml index 331f905f69..cdc397f99a 100644 --- a/smart-hive-support/pom.xml +++ b/smart-hive-support/pom.xml @@ -49,10 +49,16 @@ lombok provided + + org.smartdata + smart-hive-exec + ${project.version} + org.apache.hive hive-exec ${hive.version} + provided log4j @@ -151,4 +157,4 @@ - \ No newline at end of file +