diff --git a/.github/workflows/docker-gpdb-build-conatiners.yml b/.github/workflows/docker-gpdb-build-conatiners.yml index 3b47f7a..da66c21 100644 --- a/.github/workflows/docker-gpdb-build-conatiners.yml +++ b/.github/workflows/docker-gpdb-build-conatiners.yml @@ -95,6 +95,8 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} org.opencontainers.image.created=${{ steps.version.outputs.BUILD_DATE }} org.opencontainers.image.version=${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} + org.opencontainers.image.description="Containerized environment for building and testing Greenplum Database. Includes all necessary dependencies, tools, and configurations to streamline the development and build process." + org.opencontainers.image.title="Greenplum Build Environment" # Generate a detailed build summary in GitHub Actions UI # This provides quick access to build information and image usage instructions diff --git a/build_automation/gpdb/scripts/build-gpdb.sh b/build_automation/gpdb/scripts/build-gpdb.sh old mode 100644 new mode 100755 index ac5e351..ccd92b8 --- a/build_automation/gpdb/scripts/build-gpdb.sh +++ b/build_automation/gpdb/scripts/build-gpdb.sh @@ -20,7 +20,7 @@ # -------------------------------------------------------------------- # # Script: build-gpdb.sh -# Description: Builds Apache Cloudberry from source code and installs +# Description: Builds Greenplum from source code and installs # it. # Performs the following steps: # 1. Builds main open-gpdb database components @@ -29,7 +29,7 @@ # Uses parallel compilation based on available CPU cores. # # Required Environment Variables: -# SRC_DIR - Root source directory containing Apache Cloudberry +# SRC_DIR - Root source directory containing Greenplum # source code # # Optional Environment Variables: @@ -45,7 +45,7 @@ # ./build-gpdb.sh # # Prerequisites: -# - configure-cloudberry.sh must be run first +# - configure-gpdb.sh must be run first # - Required build dependencies must be installed # - /usr/local/cloudberry-db/lib must exist and be writable # diff --git a/build_automation/gpdb/scripts/configure-gpdb.sh b/build_automation/gpdb/scripts/configure-gpdb.sh old mode 100644 new mode 100755 diff --git a/build_automation/gpdb/scripts/create-gpdb-demo-cluster.sh b/build_automation/gpdb/scripts/create-gpdb-demo-cluster.sh old mode 100644 new mode 100755 index df25cf4..102ccef --- a/build_automation/gpdb/scripts/create-gpdb-demo-cluster.sh +++ b/build_automation/gpdb/scripts/create-gpdb-demo-cluster.sh @@ -19,8 +19,8 @@ # # -------------------------------------------------------------------- # -# Script: create-cloudberry-demo-cluster.sh -# Description: Creates and configures a demo Apache Cloudbery cluster. +# Script: create-gpdb-demo-cluster.sh +# Description: Creates and configures a demo Greenplum cluster. # Performs the following steps: # 1. Sets up required environment variables # 2. Verifies SSH connectivity @@ -35,19 +35,19 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Prerequisites: -# - Apache Cloudberry must be installed (/usr/local/cloudberry-db) +# - Greenplum must be installed (/opt/greenplum-db-6) # - SSH must be configured for passwordless access to localhost # - User must have permissions to create cluster directories # - PostgreSQL client tools (psql) must be available # # Usage: # Export required variables: -# export SRC_DIR=/path/to/cloudberry/source +# export SRC_DIR=/path/to/gpdb/source # Then run: -# ./create-cloudberry-demo-cluster.sh +# ./create-gpdb-demo-cluster.sh # # Verification Checks: -# - Apache Cloudberry version +# - Greenplum version # - Segment configuration # - Available extensions # - Active sessions @@ -66,35 +66,38 @@ set -euo pipefail +export BUILD_DESTINATION="/opt/greenplum-db-6" + # Source common utilities SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "${SCRIPT_DIR}/cloudberry-utils.sh" +source "${SCRIPT_DIR}/gpdb-utils.sh" # Define log directory export LOG_DIR="${SRC_DIR}/build-logs" +mkdir -p "${LOG_DIR}" CLUSTER_LOG="${LOG_DIR}/cluster.log" # Initialize environment -init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +init_environment "Greenplum Demo Cluster Script" "${CLUSTER_LOG}" "/opt/greenplum-db-6" # Setup environment log_section "Environment Setup" -source /usr/local/cloudberry-db/greenplum_path.sh || exit 1 +source /opt/greenplum-db-6/greenplum_path.sh || exit 1 log_section_end "Environment Setup" -# Verify SSH access +# # Verify SSH access log_section "SSH Verification" execute_cmd ssh $(hostname) 'whoami; hostname' || exit 2 log_section_end "SSH Verification" # Create demo cluster log_section "Demo Cluster Creation" -execute_cmd make create-demo-cluster --directory ${SRC_DIR}/../cloudberry || exit 3 +execute_cmd make create-demo-cluster --directory ${SRC_DIR}/../gpdb || exit 3 log_section_end "Demo Cluster Creation" # Source demo environment log_section "Source Environment" -source ${SRC_DIR}/../cloudberry/gpAux/gpdemo/gpdemo-env.sh || exit 1 +source ${SRC_DIR}/../gpdb/gpAux/gpdemo/gpdemo-env.sh || exit 1 log_section_end "Source Environment" # Manage cluster state @@ -121,5 +124,5 @@ fi log_section_end "Installation Verification" # Log completion -log_completion "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +log_completion "Greenplum Demo Cluster Script" "${CLUSTER_LOG}" exit 0 \ No newline at end of file diff --git a/build_automation/gpdb/scripts/destroy-gpdb-demo-cluster.sh b/build_automation/gpdb/scripts/destroy-gpdb-demo-cluster.sh old mode 100644 new mode 100755 index a0e1e2f..64c3d18 --- a/build_automation/gpdb/scripts/destroy-gpdb-demo-cluster.sh +++ b/build_automation/gpdb/scripts/destroy-gpdb-demo-cluster.sh @@ -19,8 +19,8 @@ # # -------------------------------------------------------------------- # -# Script: destroy-cloudberry-demo-cluster.sh -# Description: Destroys and cleans up a demo Apache Cloudberry +# Script: destroy-gpdb-demo-cluster.sh +# Description: Destroys and cleans up a demo Greenplum # cluster. # Performs the following steps: # 1. Sources required environment variables @@ -35,15 +35,15 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Prerequisites: -# - Apache Cloudberry environment must be available +# - Greenplum environment must be available # - User must have permissions to remove cluster directories # - No active connections to the cluster # # Usage: # Export required variables: -# export SRC_DIR=/path/to/cloudberry/source +# export SRC_DIR=/path/to/gpdb/source # Then run: -# ./destroy-cloudberry-demo-cluster.sh +# ./destroy-gpdb-demo-cluster.sh # # Exit Codes: # 0 - Cluster destroyed successfully @@ -51,7 +51,7 @@ # 2 - Cluster destruction failed # # Related Scripts: -# - create-cloudberry-demo-cluster.sh: Creates a new demo cluster +# - create-gpdb-demo-cluster.sh: Creates a new demo cluster # # Notes: # - This script will forcefully terminate all cluster processes @@ -64,26 +64,26 @@ set -euo pipefail # Source common utilities SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "${SCRIPT_DIR}/cloudberry-utils.sh" +source "${SCRIPT_DIR}/gpdb-utils.sh" # Define log directory export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/destroy-cluster.log" # Initialize environment -init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +init_environment "Destroy Greenplum Demo Cluster Script" "${CLUSTER_LOG}" "/opt/greenplum-db-6" -# Source Cloudberry environment +# Source Greenplum environment log_section "Environment Setup" -source_cloudberry_env || { - echo "Failed to source Cloudberry environment" | tee -a "${CLUSTER_LOG}" +source_greenplum_env || { + echo "Failed to source Greenplum environment" | tee -a "${CLUSTER_LOG}" exit 1 } log_section_end "Environment Setup" # Destroy demo cluster log_section "Destroy Demo Cluster" -execute_cmd make destroy-demo-cluster --directory ${SRC_DIR}/../cloudberry || { +execute_cmd make destroy-demo-cluster --directory ${SRC_DIR}/../gpdb || { echo "Failed to destroy demo cluster" | tee -a "${CLUSTER_LOG}" exit 2 } @@ -91,11 +91,11 @@ log_section_end "Destroy Demo Cluster" # Verify cleanup log_section "Cleanup Verification" -if [ -d "${SRC_DIR}/../cloudberry/gpAux/gpdemo/data" ]; then +if [ -d "${SRC_DIR}/../gpdb/gpAux/gpdemo/data" ]; then echo "Warning: Data directory still exists after cleanup" | tee -a "${CLUSTER_LOG}" fi log_section_end "Cleanup Verification" # Log completion -log_completion "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +log_completion "Destroy Greenplum Demo Cluster Script" "${CLUSTER_LOG}" exit 0 \ No newline at end of file diff --git a/build_automation/gpdb/scripts/gpdb-utils.sh b/build_automation/gpdb/scripts/gpdb-utils.sh old mode 100644 new mode 100755 index 36c405c..6f7858f --- a/build_automation/gpdb/scripts/gpdb-utils.sh +++ b/build_automation/gpdb/scripts/gpdb-utils.sh @@ -56,11 +56,11 @@ run_psql_cmd() { return ${PIPESTATUS[0]} } -# Function to source Cloudberry environment -source_cloudberry_env() { - echo "=== Sourcing Cloudberry environment ===" | tee -a "${LOG_DIR}/environment.log" - source /usr/local/cloudberry-db/greenplum_path.sh - source ${SRC_DIR}/../cloudberry/gpAux/gpdemo/gpdemo-env.sh +# Function to source Greenplum environment +source_greenplum_env() { + echo "=== Sourcing Greenplum environment ===" | tee -a "${LOG_DIR}/environment.log" + source ${BUILD_DESTINATION}/greenplum_path.sh + source ${SRC_DIR}/../gpdb/gpAux/gpdemo/gpdemo-env.sh } # Function to log section start diff --git a/build_automation/gpdb/scripts/parse-results.pl b/build_automation/gpdb/scripts/parse-results.pl old mode 100644 new mode 100755 index 366f7e1..6963f79 --- a/build_automation/gpdb/scripts/parse-results.pl +++ b/build_automation/gpdb/scripts/parse-results.pl @@ -20,7 +20,7 @@ # -------------------------------------------------------------------- # # Script: parse_results.pl -# Description: Processes Cloudberry test output to extract statistics +# Description: Processes Greenplum test output to extract statistics # and results. # Analyzes test log files to determine: # 1. Overall test status (pass/fail) diff --git a/build_automation/gpdb/scripts/parse-test-results.sh b/build_automation/gpdb/scripts/parse-test-results.sh old mode 100644 new mode 100755 index ca6bb4f..c1f8e19 --- a/build_automation/gpdb/scripts/parse-test-results.sh +++ b/build_automation/gpdb/scripts/parse-test-results.sh @@ -20,7 +20,7 @@ # -------------------------------------------------------------------- # # Script: parse-test-results.sh -# Description: Parses Apache Cloudberry test results and processes the +# Description: Parses Greenplum test results and processes the # output. # Provides GitHub Actions integration and environment # variable export functionality. This script is a wrapper diff --git a/build_automation/gpdb/scripts/test-gpdb.sh b/build_automation/gpdb/scripts/test-gpdb.sh old mode 100644 new mode 100755 index 6604f27..7839301 --- a/build_automation/gpdb/scripts/test-gpdb.sh +++ b/build_automation/gpdb/scripts/test-gpdb.sh @@ -19,10 +19,10 @@ # # -------------------------------------------------------------------- # -# Script: test-cloudberry.sh -# Description: Executes Apache Cloudberry test suite using specified +# Script: test-gpdb.sh +# Description: Executes Greenplum test suite using specified # make target. Supports different test types through make -# target configuration. Sources Cloudberry environment +# target configuration. Sources Greenplum environment # before running tests. # # Required Environment Variables: @@ -34,13 +34,17 @@ # LOG_DIR - Directory for logs (defaults to build-logs) # PGOPTIONS - PostgreSQL server options # +# Prerequisites: +# - Greenplum must be installed (/opt/greenplum-db-6) +# - create-gpdb-demo-cluster.sh must be run first +# # Usage: # Export required variables: # export MAKE_TARGET=installcheck-world # export MAKE_DIRECTORY="/path/to/make/dir" # export MAKE_NAME="Install Check" # Then run: -# ./test-cloudberry.sh +# ./test-gpdb.sh # # Exit Codes: # 0 - All tests passed successfully @@ -51,20 +55,22 @@ set -euo pipefail +export BUILD_DESTINATION="/opt/greenplum-db-6" + # Source common utilities SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "${SCRIPT_DIR}/cloudberry-utils.sh" +source "${SCRIPT_DIR}/gpdb-utils.sh" # Define log directory and files export LOG_DIR="build-logs" TEST_LOG="${LOG_DIR}/test.log" # Initialize environment -init_environment "Cloudberry Test Script" "${TEST_LOG}" +init_environment "Greenplum Test Script" "${TEST_LOG}" "/opt/greenplum-db-6" -# Source Cloudberry environment +# Source Greenplum environment log_section "Environment Setup" -source_cloudberry_env || exit 1 +source_greenplum_env || exit 1 log_section_end "Environment Setup" echo "MAKE_TARGET: ${MAKE_TARGET}" @@ -77,5 +83,5 @@ execute_cmd make ${MAKE_TARGET} ${MAKE_DIRECTORY} || exit 2 log_section_end "Install Check" # Log completion -log_completion "Cloudberry Test Script" "${TEST_LOG}" +log_completion "Greenplum Test Script" "${TEST_LOG}" exit 0 \ No newline at end of file diff --git a/images/docker/gpdb/build/jammy/Dockerfile b/images/docker/gpdb/build/jammy/Dockerfile index 4d6d3a1..f61641b 100644 --- a/images/docker/gpdb/build/jammy/Dockerfile +++ b/images/docker/gpdb/build/jammy/Dockerfile @@ -48,6 +48,7 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install g++-11 \ g++-11-multilib \ git \ + iputils-ping \ krb5-multidev \ libapr1-dev \ libbz2-dev \ @@ -65,10 +66,12 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install libaprutil1-dev \ libpam0g-dev \ libpam0g \ + libpopt0 \ libcgroup1 \ libyaml-0-2 \ libldap-2.5-0 \ libssl3 \ + net-tools \ ninja-build \ python-setuptools \ quilt \ @@ -80,6 +83,9 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install libgpgme11 \ python2.7 \ python2.7-dev \ + python-pip \ + python-six \ + rsync \ sudo \ iproute2 \ less \ @@ -88,6 +94,9 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install openssh-server \ locales + +RUN pip2 install psutil + RUN apt-get install -y locales \ && locale-gen "en_US.UTF-8" \ && update-locale LC_ALL="en_US.UTF-8" @@ -123,3 +132,15 @@ USER gpadmin WORKDIR /home/gpadmin RUN sudo DEBIAN_FRONTEND=noninteractive apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev + +# -------------------------------------------------------------------- +# Set the Default User and Command +# -------------------------------------------------------------------- +# The default user is set to 'gpadmin', and the container starts by +# running the init_system.sh script. This container serves as a base +# environment, and the Greenplum Database DEB package can be installed +# for testing and functional verification. +# -------------------------------------------------------------------- +USER gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] \ No newline at end of file diff --git a/images/docker/gpdb/build/jammy/configs/init_system.sh b/images/docker/gpdb/build/jammy/configs/init_system.sh index 53e70bd..a50d8b1 100644 --- a/images/docker/gpdb/build/jammy/configs/init_system.sh +++ b/images/docker/gpdb/build/jammy/configs/init_system.sh @@ -2,6 +2,24 @@ # Container Initialization Script # -------------------------------------------------------------------- +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +ssh-keygen -f /home/gpadmin/.ssh/id_rsa -N '' +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +eval "$(ssh-agent -s)" +ssh-add ~/.ssh/id_rsa +sudo service ssh start +ssh -o StrictHostKeyChecking=no gpadmin@$(hostname) "echo 'Hello world'" # -------------------------------------------------------------------- # Display a Welcome Banner @@ -36,4 +54,78 @@ cat <<-'EOF' |_| |_|\___/|_| \___/|___/_| |_|_|\_\__,_|_____/|____/ ---------------------------------------------------------------------- -EOF \ No newline at end of file +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# - Greenplum version (if installed) +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +# Check if Greenplum is installed and display its version +if dpkg -l greenplum-db-6 > /dev/null 2>&1; then + GPDB_VERSION=$(/opt/greenplum-db-6/bin/postgres --gp-version) +else + GPDB_VERSION="Not installed" +fi + +cat <<-EOF +Welcome to the Greebplum Test Environment! + +Greenplum version ... : $GPDB_VERSION +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash \ No newline at end of file diff --git a/packaging/deb/gpdb_bionic/debian/rules b/packaging/deb/gpdb_bionic/debian/rules index 0858c61..2056836 100755 --- a/packaging/deb/gpdb_bionic/debian/rules +++ b/packaging/deb/gpdb_bionic/debian/rules @@ -3,7 +3,7 @@ DH_VERBOSE = 1 DPKG_EXPORT_BUILDFLAGS = 1 -GPDB_SOURCE_PEFIX := /opt/greenplum-db-6 +GPDB_SOURCE_PREFIX := /opt/greenplum-db-6 DEBIAN_DESTINATION := $(shell pwd)/debian/build # assumes that CWD is root of gpdb source @@ -27,7 +27,7 @@ gpinstall: override_dh_auto_install: gpinstall # the staging directory for creating a debian is NOT the right GPHOME. # change GPHOME to point to the post-install target install directory. - sed -i "s#GPHOME=.*#GPHOME=${GPDB_HOME_PREFIX}-${GPDB_MAJOR_VERSION}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh + sed -i "s#GPHOME=.*#GPHOME=${GPDB_SOURCE_PREFIX}-${GPDB_MAJOR_VERSION}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh override_dh_auto_build: echo "Skipping build" diff --git a/packaging/deb/jammy/debian/control b/packaging/deb/jammy/debian/control index 5215fb7..c87166d 100644 --- a/packaging/deb/jammy/debian/control +++ b/packaging/deb/jammy/debian/control @@ -69,18 +69,8 @@ Depends: curl, openssl, python2.7, python2.7-dev, - python-lockfile, - python-paramiko, - python-psutil, - python-yaml, python-pip, - python-behave, - python-epydoc, - python-ldap, - python-mock, python-six, - python-parse, - python-is-python2, rsync, wget, zlib1g, diff --git a/packaging/deb/jammy/debian/rules b/packaging/deb/jammy/debian/rules index ac54ca7..cc6679f 100755 --- a/packaging/deb/jammy/debian/rules +++ b/packaging/deb/jammy/debian/rules @@ -3,7 +3,7 @@ DH_VERBOSE = 1 DPKG_EXPORT_BUILDFLAGS = 1 -GPDB_SOURCE_PEFIX := /opt/greenplum-db-6 +GPDB_SOURCE_PREFIX := /opt/greenplum-db DEBIAN_DESTINATION := $(shell pwd)/debian/build # assumes that CWD is root of gpdb source @@ -27,7 +27,7 @@ gpinstall: override_dh_auto_install: gpinstall # the staging directory for creating a debian is NOT the right GPHOME. # change GPHOME to point to the post-install target install directory. - sed -i "s#GPHOME=.*#GPHOME=${GPDB_SOURCE_PEFIX}-${GPDB_MAJOR_VERSION}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh + sed -i "s#GPHOME=.*#GPHOME=${GPDB_SOURCE_PREFIX}-${GPDB_MAJOR_VERSION}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh override_dh_auto_build: echo "Skipping build" diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh old mode 100644 new mode 100755