diff --git a/pom.xml b/pom.xml index 805530a8a0..8058133481 100644 --- a/pom.xml +++ b/pom.xml @@ -81,9 +81,11 @@ 2.11.12 2.11 2.28.2 - 1.20.4 + 1.21.4 + 3.7.1 1.68 3.7.1 + com.google.protobuf:protoc:${protobuf.version} 2.5.3 2.5.32 1.2.1 @@ -109,7 +111,7 @@ 2.7.18 1.0.1.RELEASE 6.6.0 - 3.12.0 + 3.18.0 1.5.5.Final 0.2.0 1.9.22 @@ -604,6 +606,24 @@ ${testcontainers.version} test + + com.github.docker-java + docker-java-api + ${docker-java.version} + test + + + com.github.docker-java + docker-java-transport + ${docker-java.version} + test + + + com.github.docker-java + docker-java-transport-zerodep + ${docker-java.version} + test + junit junit diff --git a/smart-common/pom.xml b/smart-common/pom.xml index 1e136d2152..7f906daf8c 100644 --- a/smart-common/pom.xml +++ b/smart-common/pom.xml @@ -148,7 +148,7 @@ run - com.google.protobuf:protoc:${protobuf.version} + ${protoc.artifact} ${protobuf.version} ${basedir}/src/main/proto @@ -260,4 +260,19 @@ + + + + apple-silicon-protoc + + + mac + aarch64 + + + + com.google.protobuf:protoc:${protobuf.version}:exe:osx-x86_64 + + + diff --git a/smart-server/src/main/java/org/smartdata/server/SmartServer.java b/smart-server/src/main/java/org/smartdata/server/SmartServer.java index 44eddf1e44..a65b188e81 100644 --- a/smart-server/src/main/java/org/smartdata/server/SmartServer.java +++ b/smart-server/src/main/java/org/smartdata/server/SmartServer.java @@ -372,7 +372,7 @@ public void run() { Thread.sleep(1000); } } - } catch (Exception e) { + } catch (Throwable e) { LOG.error("Failed to create SmartServer", e); errorCode = 1; } finally { diff --git a/supports/tools/docker/multihost/common.sh b/supports/tools/docker/multihost/common.sh index c967002991..0f236ba2c1 100755 --- a/supports/tools/docker/multihost/common.sh +++ b/supports/tools/docker/multihost/common.sh @@ -30,6 +30,28 @@ function wait_for_it() echo "[$i/$max_try] $service:${port} is available." } +function wait_for_file() +{ + local path=$1 + local retry_seconds=5 + local max_try=100 + let i=1 + + until [ -f "$path" ]; do + echo "[$i/$max_try] check for ${path}..." + echo "[$i/$max_try] ${path} is not available yet" + if (( $i == $max_try )); then + echo "[$i/$max_try] ${path} is still not available; giving up after ${max_try} tries. :/" + exit 1 + fi + + echo "[$i/$max_try] try in ${retry_seconds}s once again ..." + let "i++" + sleep $retry_seconds + done + echo "[$i/$max_try] $path is available." +} + function addProperty() { local path=$1 local name=$2 @@ -63,4 +85,4 @@ function configure() { echo " - Setting $name=$value" addProperty $path $name "$value" done -} \ No newline at end of file +} diff --git a/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh b/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh index daa24fa3f7..e04b9dbc3f 100755 --- a/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh +++ b/supports/tools/docker/multihost/datanode/hadoop-datanode-entrypoint.sh @@ -2,12 +2,23 @@ . ./common.sh +SSM_RUNTIME_CONF_DIR=/tmp/ssm-conf +rm -rf "$SSM_RUNTIME_CONF_DIR" +mkdir -p "$SSM_RUNTIME_CONF_DIR" +cp -R /opt/ssm/conf/. "$SSM_RUNTIME_CONF_DIR"/ +cp /opt/ssm/smart-site.xml "$SSM_RUNTIME_CONF_DIR"/smart-site.xml + cp /etc/ssm/shared/id_rsa /root/.ssh/id_rsa cp /etc/ssm/shared/id_rsa.pub /root/.ssh/id_rsa.pub cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys service ssh start ssh-keyscan ssm-server.demo >> /root/.ssh/known_hosts echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc +echo "export SMART_CONF_DIR=${SSM_RUNTIME_CONF_DIR}/" >> /root/.bashrc + +wait_for_file /etc/secrets/datanode.keytab +wait_for_file /etc/secrets/agent.keytab +wait_for_file /etc/secrets/http.keytab datadir=`echo $HDFS_CONF_dfs_datanode_data_dir | perl -pe 's#file://##'` if [ ! -d $datadir ]; then @@ -18,7 +29,8 @@ fi chmod +r /etc/secrets/*.keytab moveHadoopConfFiles /etc/conf ${HADOOP_CONF_DIR} +configure "$HADOOP_CONF_DIR"/hdfs-site.xml hdfs HDFS_CONF $HADOOP_HOME/bin/hdfs --debug --config $HADOOP_CONF_DIR datanode -tail -f /dev/null \ No newline at end of file +tail -f /dev/null diff --git a/supports/tools/docker/multihost/docker-compose.yaml b/supports/tools/docker/multihost/docker-compose.yaml index 6da25745b2..5b3a6c50d3 100644 --- a/supports/tools/docker/multihost/docker-compose.yaml +++ b/supports/tools/docker/multihost/docker-compose.yaml @@ -31,7 +31,8 @@ services: networks: - demo depends_on: - - hadoop-datanode + kdc-server: + condition: service_healthy hadoop-datanode: image: hub.adsw.io/ssm/ssm-hadoop-base:${HADOOP_VERSION:-3.4.3} @@ -48,7 +49,7 @@ services: - ./kerberos/krb5.conf:/etc/krb5.conf - ./conf:/etc/conf - ./conf:/opt/ssm/conf - - ./ssm-conf/smart-site-agent.xml:/opt/ssm/conf/smart-site.xml + - ./ssm-conf/smart-site-agent.xml:/opt/ssm/smart-site.xml - ./datanode/hadoop-datanode-entrypoint.sh:/hadoop-datanode-entrypoint.sh entrypoint: ["/hadoop-datanode-entrypoint.sh"] ports: @@ -70,7 +71,12 @@ services: networks: - demo depends_on: - - ssm-server + hadoop-namenode: + condition: service_healthy + kdc-server: + condition: service_healthy + ssm-server: + condition: service_started ssm-server: image: hub.adsw.io/ssm/ssm-server:${SSM_VERSION:-2.3.0-SNAPSHOT} @@ -82,7 +88,7 @@ services: - secrets:/etc/secrets - ./kerberos/krb5.conf:/etc/krb5.conf - ./conf:/opt/ssm/conf - - ./ssm-conf/smart-site-master.xml:/opt/ssm/conf/smart-site.xml + - ./ssm-conf/smart-site-master.xml:/opt/ssm/smart-site.xml - ./common.sh:/common.sh - ./ssm/ssm-server-entrypoint.sh:/ssm-server-entrypoint.sh entrypoint: ["/ssm-server-entrypoint.sh"] @@ -96,15 +102,23 @@ services: environment: SSM_DEBUG_OPT: ${SSM_DEBUG_OPT} healthcheck: - test: ["CMD-SHELL", "curl http://ssm-server.demo:8081 || exit 1"] + test: ["CMD-SHELL", "curl -fsS http://ssm-server.demo:8081/actuator/health || exit 1"] interval: 30s timeout: 15s retries: 5 networks: - demo depends_on: - - ssm-metastore-db - - kdc-server + ssm-metastore-db: + condition: service_healthy + kdc-server: + condition: service_healthy + samba: + condition: service_healthy + hadoop-namenode: + condition: service_healthy + hive-metastore: + condition: service_healthy ssm-metastore-db: image: "docker.io/library/postgres:14.0" @@ -173,6 +187,14 @@ services: image: apache/hive:4.0.1 container_name: hive-metastore hostname: hive-metastore.demo + restart: unless-stopped + depends_on: + ssm-metastore-db: + condition: service_healthy + kdc-server: + condition: service_healthy + hadoop-namenode: + condition: service_healthy environment: SERVICE_NAME: metastore DB_DRIVER: postgres @@ -190,6 +212,8 @@ services: - ./kerberos/krb5.conf:/etc/krb5.conf - ./hive/hive-site.xml:/opt/hive/conf/hive-site.xml - ./hive/core-site.xml:/opt/hive/conf/core-site.xml + - ./conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml + - ./conf/yarn-site.xml:/opt/hive/conf/yarn-site.xml ports: - "9083:9083" healthcheck: @@ -198,15 +222,22 @@ services: timeout: 15s retries: 5 networks: - - demo + demo: + aliases: + - hive-metastore.demo hive-server2: image: apache/hive:4.0.1 container_name: hive-server2 hostname: hive-server2.demo + restart: unless-stopped depends_on: - - hive-metastore - - hadoop-namenode + hive-metastore: + condition: service_healthy + hadoop-namenode: + condition: service_healthy + kdc-server: + condition: service_healthy environment: SERVICE_NAME: hiveserver2 IS_RESUME: true @@ -214,15 +245,17 @@ services: -Dhive.metastore.uris=thrift://hive-metastore.demo:9083 -Dhive.metastore.warehouse.dir=hdfs://hadoop-namenode.demo:8020/user/hive/warehouse healthcheck: - test: [ "CMD", "sh", "-c", "beeline -u 'jdbc:hive2://localhost:10000/'" ] + test: [ "CMD", "sh", "-c", "/opt/hive/bin/beeline -u 'jdbc:hive2://localhost:10000/default' -e 'show databases;'" ] interval: 30s timeout: 10s - retries: 3 + retries: 5 volumes: - secrets:/etc/secrets - ./kerberos/krb5.conf:/etc/krb5.conf - ./hive/hive-site.xml:/opt/hive/conf/hive-site.xml - ./hive/core-site.xml:/opt/hive/conf/core-site.xml + - ./conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml + - ./conf/yarn-site.xml:/opt/hive/conf/yarn-site.xml ports: - "10000:10000" - "10002:10002" diff --git a/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh b/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh index 4b11fc6359..5acffdab35 100755 --- a/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh +++ b/supports/tools/docker/multihost/namenode/hadoop-namenode-entrypoint.sh @@ -6,6 +6,10 @@ service ssh start ssh-keyscan ssm-server.demo >> /root/.ssh/known_hosts echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc +wait_for_file /etc/secrets/namenode.keytab +wait_for_file /etc/secrets/yarn.keytab +chmod +r /etc/secrets/*.keytab + namedir=`echo $HDFS_CONF_dfs_namenode_name_dir | perl -pe 's#file://##'` if [ ! -d $namedir ]; then echo "Namenode name directory not found: $namedir" @@ -18,6 +22,7 @@ if [ -z "$CLUSTER_NAME" ]; then fi moveHadoopConfFiles /etc/conf ${HADOOP_CONF_DIR} +configure "$HADOOP_CONF_DIR"/hdfs-site.xml hdfs HDFS_CONF # HDFS addProperty "$HADOOP_CONF_DIR"/hdfs-site.xml dfs.namenode.rpc-bind-host 0.0.0.0 @@ -61,4 +66,4 @@ echo "----------------------" $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager & wait_for_it $(hostname -f):8088 -tail -f /dev/null \ No newline at end of file +tail -f /dev/null diff --git a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh index 61d9ab116d..5ba8d160db 100755 --- a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh +++ b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh @@ -2,13 +2,19 @@ . ./common.sh +SSM_RUNTIME_CONF_DIR=/tmp/ssm-conf +rm -rf "$SSM_RUNTIME_CONF_DIR" +mkdir -p "$SSM_RUNTIME_CONF_DIR" +cp -R "$SSM_HOME"/conf/. "$SSM_RUNTIME_CONF_DIR"/ +cp "$SSM_HOME"/smart-site.xml "$SSM_RUNTIME_CONF_DIR"/smart-site.xml + cp /root/.ssh/id_rsa /tmp/shared/id_rsa cp /root/.ssh/id_rsa.pub /tmp/shared/id_rsa.pub service ssh start ssh-keyscan "$HOSTNAME" >> /root/.ssh/known_hosts echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc echo "export SMART_HOME=${SSM_HOME}" >> /root/.bashrc -echo "export SMART_CONF_DIR=${SSM_HOME}/conf/" >> /root/.bashrc +echo "export SMART_CONF_DIR=${SSM_RUNTIME_CONF_DIR}/" >> /root/.bashrc # Starting Smart Storage Manager cd $SSM_HOME || exit @@ -17,8 +23,15 @@ echo "---------------------------" echo "Starting SSM server and agents" echo "---------------------------" -source bin/start-ssm.sh ${SSM_DEBUG_OPT} --config ${SSM_HOME}/conf/ & +wait_for_file /etc/secrets/ssm.keytab +wait_for_file /etc/secrets/http.keytab +wait_for_it ssm-metastore-db.demo:5432 +wait_for_it samba:389 +wait_for_it hadoop-namenode.demo:8020 +wait_for_it hadoop-datanode.demo:22 + +source bin/start-ssm.sh ${SSM_DEBUG_OPT} --config ${SSM_RUNTIME_CONF_DIR}/ & wait_for_it $(hostname -f):8081 wait_for_it hadoop-datanode.demo:7048 -tail -f /var/log/ssm/* \ No newline at end of file +tail -f /var/log/ssm/*