ADH-8212: Update dependencies: Add Hadoop Yarn dependencies and configure default activation for Hadoop 3.3 profile

pom.xml

@@ -81,9 +81,11 @@
     <scala.version>2.11.12</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <mockito.version>2.28.2</mockito.version>
-    <testcontainers.version>1.20.4</testcontainers.version>
+    <testcontainers.version>1.21.4</testcontainers.version>
+    <docker-java.version>3.7.1</docker-java.version>
     <bouncycastle.version>1.68</bouncycastle.version>
     <protobuf.version>3.7.1</protobuf.version>
+    <protoc.artifact>com.google.protobuf:protoc:${protobuf.version}</protoc.artifact>
     <dbunitVersion>2.5.3</dbunitVersion>
     <akka.version>2.5.32</akka.version>
     <akka.config.version>1.2.1</akka.config.version>
@@ -109,7 +111,7 @@
     <spring.boot.version>2.7.18</spring.boot.version>
     <spring.boot.security.kerberos.version>1.0.1.RELEASE</spring.boot.security.kerberos.version>
     <openapi.generator.version>6.6.0</openapi.generator.version>
-    <commons.lang3.version>3.12.0</commons.lang3.version>
+    <commons.lang3.version>3.18.0</commons.lang3.version>
     <mapstruct.version>1.5.5.Final</mapstruct.version>
     <lombok-mapstruct-binding.version>0.2.0</lombok-mapstruct-binding.version>
     <aspectj.version>1.9.22</aspectj.version>
@@ -604,6 +606,24 @@
         <version>${testcontainers.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>com.github.docker-java</groupId>
+        <artifactId>docker-java-api</artifactId>
+        <version>${docker-java.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.github.docker-java</groupId>
+        <artifactId>docker-java-transport</artifactId>
+        <version>${docker-java.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.github.docker-java</groupId>
+        <artifactId>docker-java-transport-zerodep</artifactId>
+        <version>${docker-java.version}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>

smart-common/pom.xml

@@ -148,7 +148,7 @@
                             <goal>run</goal>
                         </goals>
                         <configuration>
-                            <protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact>
+                            <protocArtifact>${protoc.artifact}</protocArtifact>
                             <protocVersion>${protobuf.version}</protocVersion>
                             <inputDirectories>
                                 <include>${basedir}/src/main/proto</include>
@@ -260,4 +260,19 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>apple-silicon-protoc</id>
+            <activation>
+                <os>
+                    <family>mac</family>
+                    <arch>aarch64</arch>
+                </os>
+            </activation>
+            <properties>
+                <protoc.artifact>com.google.protobuf:protoc:${protobuf.version}:exe:osx-x86_64</protoc.artifact>
+            </properties>
+        </profile>
+    </profiles>
 </project>

smart-server/src/main/java/org/smartdata/server/SmartServer.java

@@ -372,7 +372,7 @@ public void run() {
           Thread.sleep(1000);
         }
       }
-    } catch (Exception e) {
+    } catch (Throwable e) {
       LOG.error("Failed to create SmartServer", e);
       errorCode = 1;
     } finally {

supports/tools/docker/multihost/common.sh

@@ -30,6 +30,28 @@ function wait_for_it()
     echo "[$i/$max_try] $service:${port} is available."
 }
 
+function wait_for_file()
+{
+    local path=$1
+    local retry_seconds=5
+    local max_try=100
+    let i=1
+
+    until [ -f "$path" ]; do
+      echo "[$i/$max_try] check for ${path}..."
+      echo "[$i/$max_try] ${path} is not available yet"
+      if (( $i == $max_try )); then
+        echo "[$i/$max_try] ${path} is still not available; giving up after ${max_try} tries. :/"
+        exit 1
+      fi
+
+      echo "[$i/$max_try] try in ${retry_seconds}s once again ..."
+      let "i++"
+      sleep $retry_seconds
+    done
+    echo "[$i/$max_try] $path is available."
+}
+
 function addProperty() {
   local path=$1
   local name=$2
@@ -63,4 +85,4 @@ function configure() {
         echo " - Setting $name=$value"
         addProperty $path $name "$value"
     done
-}
+}

supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh

@@ -2,12 +2,23 @@
 
 . ./common.sh
 
+SSM_RUNTIME_CONF_DIR=/tmp/ssm-conf
+rm -rf "$SSM_RUNTIME_CONF_DIR"
+mkdir -p "$SSM_RUNTIME_CONF_DIR"
+cp -R /opt/ssm/conf/. "$SSM_RUNTIME_CONF_DIR"/
+cp /opt/ssm/smart-site.xml "$SSM_RUNTIME_CONF_DIR"/smart-site.xml
+
 cp /etc/ssm/shared/id_rsa /root/.ssh/id_rsa
 cp /etc/ssm/shared/id_rsa.pub /root/.ssh/id_rsa.pub
 cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
 service ssh start
 ssh-keyscan ssm-server.demo >> /root/.ssh/known_hosts
 echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc
+echo "export SMART_CONF_DIR=${SSM_RUNTIME_CONF_DIR}/" >> /root/.bashrc
+
+wait_for_file /etc/secrets/datanode.keytab
+wait_for_file /etc/secrets/agent.keytab
+wait_for_file /etc/secrets/http.keytab
 
 datadir=`echo $HDFS_CONF_dfs_datanode_data_dir | perl -pe 's#file://##'`
 if [ ! -d $datadir ]; then
@@ -18,7 +29,8 @@ fi
 chmod +r /etc/secrets/*.keytab
 
 moveHadoopConfFiles /etc/conf ${HADOOP_CONF_DIR}
+configure "$HADOOP_CONF_DIR"/hdfs-site.xml hdfs HDFS_CONF
 
 $HADOOP_HOME/bin/hdfs --debug --config $HADOOP_CONF_DIR datanode
 
-tail -f /dev/null
+tail -f /dev/null

supports/tools/docker/multihost/docker-compose.yaml

@@ -31,7 +31,8 @@ services:
     networks:
       - demo
     depends_on:
-      - hadoop-datanode
+      kdc-server:
+        condition: service_healthy
 
   hadoop-datanode:
     image: hub.adsw.io/ssm/ssm-hadoop-base:${HADOOP_VERSION:-3.4.3}
@@ -48,7 +49,7 @@ services:
       - ./kerberos/krb5.conf:/etc/krb5.conf
       - ./conf:/etc/conf
       - ./conf:/opt/ssm/conf
-      - ./ssm-conf/smart-site-agent.xml:/opt/ssm/conf/smart-site.xml
+      - ./ssm-conf/smart-site-agent.xml:/opt/ssm/smart-site.xml
       - ./datanode/hadoop-datanode-entrypoint.sh:/hadoop-datanode-entrypoint.sh
     entrypoint: ["/hadoop-datanode-entrypoint.sh"]
     ports:
@@ -70,7 +71,12 @@ services:
     networks:
       - demo
     depends_on:
-      - ssm-server
+      hadoop-namenode:
+        condition: service_healthy
+      kdc-server:
+        condition: service_healthy
+      ssm-server:
+        condition: service_started
 
   ssm-server:
     image: hub.adsw.io/ssm/ssm-server:${SSM_VERSION:-2.3.0-SNAPSHOT}
@@ -82,7 +88,7 @@ services:
       - secrets:/etc/secrets
       - ./kerberos/krb5.conf:/etc/krb5.conf
       - ./conf:/opt/ssm/conf
-      - ./ssm-conf/smart-site-master.xml:/opt/ssm/conf/smart-site.xml
+      - ./ssm-conf/smart-site-master.xml:/opt/ssm/smart-site.xml
       - ./common.sh:/common.sh
       - ./ssm/ssm-server-entrypoint.sh:/ssm-server-entrypoint.sh
     entrypoint: ["/ssm-server-entrypoint.sh"]
@@ -96,15 +102,23 @@ services:
     environment:
       SSM_DEBUG_OPT: ${SSM_DEBUG_OPT}
     healthcheck:
-      test: ["CMD-SHELL", "curl http://ssm-server.demo:8081 || exit 1"]
+      test: ["CMD-SHELL", "curl -fsS http://ssm-server.demo:8081/actuator/health || exit 1"]
       interval: 30s
       timeout: 15s
       retries: 5
     networks:
       - demo
     depends_on:
-      - ssm-metastore-db
-      - kdc-server
+      ssm-metastore-db:
+        condition: service_healthy
+      kdc-server:
+        condition: service_healthy
+      samba:
+        condition: service_healthy
+      hadoop-namenode:
+        condition: service_healthy
+      hive-metastore:
+        condition: service_healthy
 
   ssm-metastore-db:
     image: "docker.io/library/postgres:14.0"
@@ -173,6 +187,14 @@ services:
     image: apache/hive:4.0.1
     container_name: hive-metastore
     hostname: hive-metastore.demo
+    restart: unless-stopped
+    depends_on:
+      ssm-metastore-db:
+        condition: service_healthy
+      kdc-server:
+        condition: service_healthy
+      hadoop-namenode:
+        condition: service_healthy
     environment:
       SERVICE_NAME: metastore
       DB_DRIVER: postgres
@@ -190,6 +212,8 @@ services:
       - ./kerberos/krb5.conf:/etc/krb5.conf
       - ./hive/hive-site.xml:/opt/hive/conf/hive-site.xml
       - ./hive/core-site.xml:/opt/hive/conf/core-site.xml
+      - ./conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml
+      - ./conf/yarn-site.xml:/opt/hive/conf/yarn-site.xml
     ports:
       - "9083:9083"
     healthcheck:
@@ -198,31 +222,40 @@ services:
       timeout: 15s
       retries: 5
     networks:
-      - demo
+      demo:
+        aliases:
+          - hive-metastore.demo
 
   hive-server2:
     image: apache/hive:4.0.1
     container_name: hive-server2
     hostname: hive-server2.demo
+    restart: unless-stopped
     depends_on:
-      - hive-metastore
-      - hadoop-namenode
+      hive-metastore:
+        condition: service_healthy
+      hadoop-namenode:
+        condition: service_healthy
+      kdc-server:
+        condition: service_healthy
     environment:
       SERVICE_NAME: hiveserver2
       IS_RESUME: true
       SERVICE_OPTS: >-
         -Dhive.metastore.uris=thrift://hive-metastore.demo:9083
         -Dhive.metastore.warehouse.dir=hdfs://hadoop-namenode.demo:8020/user/hive/warehouse
     healthcheck:
-      test: [ "CMD", "sh", "-c", "beeline -u 'jdbc:hive2://localhost:10000/'" ]
+      test: [ "CMD", "sh", "-c", "/opt/hive/bin/beeline -u 'jdbc:hive2://localhost:10000/default' -e 'show databases;'" ]
       interval: 30s
       timeout: 10s
-      retries: 3
+      retries: 5
     volumes:
       - secrets:/etc/secrets
       - ./kerberos/krb5.conf:/etc/krb5.conf
       - ./hive/hive-site.xml:/opt/hive/conf/hive-site.xml
       - ./hive/core-site.xml:/opt/hive/conf/core-site.xml
+      - ./conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml
+      - ./conf/yarn-site.xml:/opt/hive/conf/yarn-site.xml
     ports:
       - "10000:10000"
       - "10002:10002"

supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh

@@ -6,6 +6,10 @@ service ssh start
 ssh-keyscan ssm-server.demo >> /root/.ssh/known_hosts
 echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc
 
+wait_for_file /etc/secrets/namenode.keytab
+wait_for_file /etc/secrets/yarn.keytab
+chmod +r /etc/secrets/*.keytab
+
 namedir=`echo $HDFS_CONF_dfs_namenode_name_dir | perl -pe 's#file://##'`
 if [ ! -d $namedir ]; then
   echo "Namenode name directory not found: $namedir"
@@ -18,6 +22,7 @@ if [ -z "$CLUSTER_NAME" ]; then
 fi
 
 moveHadoopConfFiles /etc/conf ${HADOOP_CONF_DIR}
+configure "$HADOOP_CONF_DIR"/hdfs-site.xml hdfs HDFS_CONF
 
 # HDFS
 addProperty "$HADOOP_CONF_DIR"/hdfs-site.xml dfs.namenode.rpc-bind-host 0.0.0.0
@@ -61,4 +66,4 @@ echo "----------------------"
 $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager &
 wait_for_it $(hostname -f):8088
 
-tail -f /dev/null
+tail -f /dev/null

supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh

@@ -2,13 +2,19 @@
 
 . ./common.sh
 
+SSM_RUNTIME_CONF_DIR=/tmp/ssm-conf
+rm -rf "$SSM_RUNTIME_CONF_DIR"
+mkdir -p "$SSM_RUNTIME_CONF_DIR"
+cp -R "$SSM_HOME"/conf/. "$SSM_RUNTIME_CONF_DIR"/
+cp "$SSM_HOME"/smart-site.xml "$SSM_RUNTIME_CONF_DIR"/smart-site.xml
+
 cp /root/.ssh/id_rsa /tmp/shared/id_rsa
 cp /root/.ssh/id_rsa.pub /tmp/shared/id_rsa.pub
 service ssh start
 ssh-keyscan "$HOSTNAME" >> /root/.ssh/known_hosts
 echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc
 echo "export SMART_HOME=${SSM_HOME}" >> /root/.bashrc
-echo "export SMART_CONF_DIR=${SSM_HOME}/conf/" >> /root/.bashrc
+echo "export SMART_CONF_DIR=${SSM_RUNTIME_CONF_DIR}/" >> /root/.bashrc
 
 # Starting Smart Storage Manager
 cd $SSM_HOME || exit
@@ -17,8 +23,15 @@ echo "---------------------------"
 echo "Starting SSM server and agents"
 echo "---------------------------"
 
-source bin/start-ssm.sh ${SSM_DEBUG_OPT} --config ${SSM_HOME}/conf/ &
+wait_for_file /etc/secrets/ssm.keytab
+wait_for_file /etc/secrets/http.keytab
+wait_for_it ssm-metastore-db.demo:5432
+wait_for_it samba:389
+wait_for_it hadoop-namenode.demo:8020
+wait_for_it hadoop-datanode.demo:22
+
+source bin/start-ssm.sh ${SSM_DEBUG_OPT} --config ${SSM_RUNTIME_CONF_DIR}/ &
 wait_for_it $(hostname -f):8081
 wait_for_it hadoop-datanode.demo:7048
 
-tail -f /var/log/ssm/*
+tail -f /var/log/ssm/*