From 98933986fdeb844a7ba2b9eb1461d86c9be0bea9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 3 Jun 2026 17:44:22 +0200 Subject: [PATCH] WIP --- .github/workflows/cpp.yml | 2 +- .github/workflows/cpp_extra.yml | 4 +- .hadolint.yaml | 1 + ci/docker/almalinux-10-verify-rc.dockerfile | 2 +- ci/docker/almalinux-8-verify-rc.dockerfile | 2 +- ci/docker/alpine-linux-3.22-cpp.dockerfile | 2 +- ci/docker/alpine-linux-3.22-r.dockerfile | 2 +- ci/docker/conda-cpp.dockerfile | 3 +- ci/docker/conda-integration.dockerfile | 6 +- ci/docker/conda-python-dask.dockerfile | 5 +- ci/docker/conda-python-emscripten.dockerfile | 3 +- ci/docker/conda-python-hdfs.dockerfile | 5 +- ci/docker/conda-python-jpype.dockerfile | 5 +- ci/docker/conda-python-pandas.dockerfile | 5 +- ci/docker/conda-python-spark.dockerfile | 5 +- ci/docker/conda-python.dockerfile | 3 +- ci/docker/conda.dockerfile | 2 +- ci/docker/debian-13-cpp.dockerfile | 3 +- ci/docker/debian-experimental-cpp.dockerfile | 3 +- ci/docker/fedora-42-cpp.dockerfile | 3 +- ci/docker/fedora-42-r-clang.dockerfile | 2 +- ci/docker/linux-apt-c-glib.dockerfile | 3 +- ci/docker/linux-apt-python-3.dockerfile | 3 +- ci/docker/linux-apt-r.dockerfile | 3 +- ci/docker/linux-apt-ruby.dockerfile | 3 +- ci/docker/linux-dnf-python-3.dockerfile | 3 +- .../python-wheel-manylinux-test.dockerfile | 2 +- ci/docker/ubuntu-22.04-cpp-minimal.dockerfile | 5 +- ci/docker/ubuntu-22.04-cpp.dockerfile | 5 +- ci/docker/ubuntu-22.04-verify-rc.dockerfile | 3 +- ci/docker/ubuntu-24.04-cpp-minimal.dockerfile | 5 +- ci/docker/ubuntu-24.04-cpp.dockerfile | 5 +- ci/docker/ubuntu-24.04-verify-rc.dockerfile | 3 +- compose.yaml | 321 ++++++++++-------- dev/archery/archery/docker/core.py | 4 +- dev/tasks/python-wheels/github.linux.yml | 2 +- 36 files changed, 238 insertions(+), 200 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 155133f6a92f..a6ac94b4c3f0 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -96,7 +96,7 @@ jobs: runs-on: ubuntu-latest title: AMD64 Ubuntu 24.04 C++ ASAN UBSAN ubuntu: 24.04 - - arch: arm64v8 + - arch: arm64/v8 clang-tools: 18 image: ubuntu-cpp llvm: 18 diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index e4e51608dbe3..e14d4c7660de 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -221,7 +221,7 @@ jobs: platform: - arch: "amd64" runs-on: ubuntu-latest - - arch: "arm64v8" + - arch: "arm64/v8" runs-on: ubuntu-24.04-arm env: ARCH: ${{ matrix.platform.arch }} @@ -584,7 +584,7 @@ jobs: ARROW_BUILD_STATIC: OFF ARROW_BUILD_TESTS: ON ARROW_BUILD_TYPE: release - # Turn Arrow CSV off to disable `find_package(Arrow)` check on MSVC CI. + # Turn Arrow CSV off to disable `find_package(Arrow)` check on MSVC CI. # GH-49050 TODO: enable `find_package(Arrow)` check on MSVC CI. ARROW_CSV: OFF ARROW_DEPENDENCY_SOURCE: VCPKG diff --git a/.hadolint.yaml b/.hadolint.yaml index 6d326d7317ee..0eebc405e192 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -22,4 +22,5 @@ ignored: - DL3018 - DL3028 # Ruby gem version pinning - DL3007 # r-sanitizer must use latest + - DL3029 # Do not use --platform flag with FROM - DL3041 # Specify version with `dnf install -y -`. diff --git a/ci/docker/almalinux-10-verify-rc.dockerfile b/ci/docker/almalinux-10-verify-rc.dockerfile index efd77a86d109..099674d31d2d 100644 --- a/ci/docker/almalinux-10-verify-rc.dockerfile +++ b/ci/docker/almalinux-10-verify-rc.dockerfile @@ -16,7 +16,7 @@ # under the License. ARG arch=amd64 -FROM ${arch}/almalinux:10 +FROM --platform=linux/${arch} almalinux:10 COPY dev/release/setup-rhel-rebuilds.sh / RUN /setup-rhel-rebuilds.sh && \ diff --git a/ci/docker/almalinux-8-verify-rc.dockerfile b/ci/docker/almalinux-8-verify-rc.dockerfile index e9544e6becc7..a07e0b27fbdc 100644 --- a/ci/docker/almalinux-8-verify-rc.dockerfile +++ b/ci/docker/almalinux-8-verify-rc.dockerfile @@ -16,7 +16,7 @@ # under the License. ARG arch=amd64 -FROM ${arch}/almalinux:8 +FROM --platform=linux/${arch} almalinux:8 COPY dev/release/setup-rhel-rebuilds.sh / RUN /setup-rhel-rebuilds.sh && \ diff --git a/ci/docker/alpine-linux-3.22-cpp.dockerfile b/ci/docker/alpine-linux-3.22-cpp.dockerfile index c3a2a58ef959..17073b3f2634 100644 --- a/ci/docker/alpine-linux-3.22-cpp.dockerfile +++ b/ci/docker/alpine-linux-3.22-cpp.dockerfile @@ -16,7 +16,7 @@ # under the License. ARG arch=amd64 -FROM ${arch}/alpine:3.22 +FROM --platform=linux/${arch} alpine:3.22 RUN apk add \ apache-orc-dev \ diff --git a/ci/docker/alpine-linux-3.22-r.dockerfile b/ci/docker/alpine-linux-3.22-r.dockerfile index 887cb6445e7d..31ffddd29d94 100644 --- a/ci/docker/alpine-linux-3.22-r.dockerfile +++ b/ci/docker/alpine-linux-3.22-r.dockerfile @@ -20,7 +20,7 @@ # Replicates CRAN's Alpine environment as closely as possible ARG arch=amd64 -FROM ${arch}/alpine:3.22 +FROM --platform=linux/${arch} alpine:3.22 # Install R and essential build tools # Note: bash is needed for Arrow CI scripts, even though CRAN's Alpine uses BusyBox diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index a387fb266990..474c18a9ade5 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -17,7 +17,8 @@ ARG repo ARG arch -FROM ${repo}:${arch}-conda +ARG arch_short +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda COPY ci/scripts/install_minio.sh /arrow/ci/scripts RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile index 20ccd8f89e34..272141b604e0 100644 --- a/ci/docker/conda-integration.dockerfile +++ b/ci/docker/conda-integration.dockerfile @@ -16,10 +16,10 @@ # under the License. ARG repo -ARG arch=amd64 -FROM ${repo}:${arch}-conda-cpp +ARG arch +ARG arch_short +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-cpp -ARG arch=amd64 # We need to synchronize the following values with the values in .env # and services.conda-integration in compose.yaml. ARG maven=3.9.9 diff --git a/ci/docker/conda-python-dask.dockerfile b/ci/docker/conda-python-dask.dockerfile index 5317a64e01c6..6991f1008602 100644 --- a/ci/docker/conda-python-dask.dockerfile +++ b/ci/docker/conda-python-dask.dockerfile @@ -16,9 +16,10 @@ # under the License. ARG repo -ARG arch=amd64 +ARG arch +ARG arch_short ARG python=3.10 -FROM ${repo}:${arch}-conda-python-${python} +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG dask=latest COPY ci/scripts/install_dask.sh /arrow/ci/scripts/ diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index c56bf4f0c593..5154d8c41918 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -17,8 +17,9 @@ ARG repo ARG arch +ARG arch_short ARG python="3.12" -FROM ${repo}:${arch}-conda-python-${python} +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG selenium_version="4.41.0" ARG pyodide_version="0.26.0" diff --git a/ci/docker/conda-python-hdfs.dockerfile b/ci/docker/conda-python-hdfs.dockerfile index 42db4d1ca4c9..263b38eaa587 100644 --- a/ci/docker/conda-python-hdfs.dockerfile +++ b/ci/docker/conda-python-hdfs.dockerfile @@ -16,9 +16,10 @@ # under the License. ARG repo -ARG arch=amd64 +ARG arch +ARG arch_short ARG python=3.10 -FROM ${repo}:${arch}-conda-python-${python} +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG jdk=11 ARG maven=3.9.9 diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile index 639796f1b3a7..d5aa747b861d 100644 --- a/ci/docker/conda-python-jpype.dockerfile +++ b/ci/docker/conda-python-jpype.dockerfile @@ -16,9 +16,10 @@ # under the License. ARG repo -ARG arch=amd64 +ARG arch +ARG arch_short ARG python=3.10 -FROM ${repo}:${arch}-conda-python-${python} +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG jdk=11 ARG maven=3.9.9 diff --git a/ci/docker/conda-python-pandas.dockerfile b/ci/docker/conda-python-pandas.dockerfile index 5a8ec7c6a8fe..e39650af8212 100644 --- a/ci/docker/conda-python-pandas.dockerfile +++ b/ci/docker/conda-python-pandas.dockerfile @@ -16,9 +16,10 @@ # under the License. ARG repo -ARG arch=amd64 +ARG arch +ARG arch_short ARG python=3.10 -FROM ${repo}:${arch}-conda-python-${python} +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG pandas=latest ARG numpy=latest diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile index 5765a59b43cd..89bc231aad34 100644 --- a/ci/docker/conda-python-spark.dockerfile +++ b/ci/docker/conda-python-spark.dockerfile @@ -16,9 +16,10 @@ # under the License. ARG repo -ARG arch=amd64 +ARG arch +ARG arch_short ARG python=3.10 -FROM ${repo}:${arch}-conda-python-${python} +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG jdk=11 ARG maven=3.9.9 diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index 3127ee6edd07..db40b1009f1f 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -17,7 +17,8 @@ ARG repo ARG arch -FROM ${repo}:${arch}-conda-cpp +ARG arch_short +FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-cpp # install python specific packages ARG python=3.10 diff --git a/ci/docker/conda.dockerfile b/ci/docker/conda.dockerfile index 6492e761fdda..fd34cd9801bd 100644 --- a/ci/docker/conda.dockerfile +++ b/ci/docker/conda.dockerfile @@ -16,7 +16,7 @@ # under the License. ARG arch=amd64 -FROM ${arch}/ubuntu:22.04 +FROM --platform=linux/${arch} ubuntu:24.04 # install build essentials RUN export DEBIAN_FRONTEND=noninteractive && \ diff --git a/ci/docker/debian-13-cpp.dockerfile b/ci/docker/debian-13-cpp.dockerfile index 951ee98e6a21..552800888e78 100644 --- a/ci/docker/debian-13-cpp.dockerfile +++ b/ci/docker/debian-13-cpp.dockerfile @@ -16,8 +16,7 @@ # under the License. ARG arch=amd64 -FROM ${arch}/debian:13 -ARG arch +FROM --platform=linux/${arch} debian:13 ENV DEBIAN_FRONTEND noninteractive diff --git a/ci/docker/debian-experimental-cpp.dockerfile b/ci/docker/debian-experimental-cpp.dockerfile index 7e00378ea6b6..d349fcb1b72a 100644 --- a/ci/docker/debian-experimental-cpp.dockerfile +++ b/ci/docker/debian-experimental-cpp.dockerfile @@ -16,8 +16,7 @@ # under the License. ARG arch=amd64 -FROM ${arch}/debian:experimental -ARG arch +FROM --platform=linux/${arch} debian:experimental ENV DEBIAN_FRONTEND noninteractive diff --git a/ci/docker/fedora-42-cpp.dockerfile b/ci/docker/fedora-42-cpp.dockerfile index 9a8533688fe0..60e7dab6780f 100644 --- a/ci/docker/fedora-42-cpp.dockerfile +++ b/ci/docker/fedora-42-cpp.dockerfile @@ -16,8 +16,7 @@ # under the License. ARG arch -FROM ${arch}/fedora:42 -ARG arch +FROM --platform=linux/${arch} fedora:42 # install dependencies RUN dnf update -y && \ diff --git a/ci/docker/fedora-42-r-clang.dockerfile b/ci/docker/fedora-42-r-clang.dockerfile index 9bc970e06097..57a0c6038d33 100644 --- a/ci/docker/fedora-42-r-clang.dockerfile +++ b/ci/docker/fedora-42-r-clang.dockerfile @@ -20,7 +20,7 @@ # See: https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang ARG arch=amd64 -FROM ${arch}/fedora:42 +FROM --platform=linux/${arch} fedora:42 # Install build dependencies RUN dnf update -y && \ diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile index 80742255b766..fbd7ee803190 100644 --- a/ci/docker/linux-apt-c-glib.dockerfile +++ b/ci/docker/linux-apt-c-glib.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. +ARG arch ARG base -FROM ${base} +FROM --platform=linux/${arch} ${base} RUN apt-get update -y -q && \ apt-get install -y -q \ diff --git a/ci/docker/linux-apt-python-3.dockerfile b/ci/docker/linux-apt-python-3.dockerfile index d5e7bf00886c..6a016423b7dd 100644 --- a/ci/docker/linux-apt-python-3.dockerfile +++ b/ci/docker/linux-apt-python-3.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. +ARG arch ARG base -FROM ${base} +FROM --platform=linux/${arch} ${base} COPY python/requirements-build.txt \ python/requirements-test.txt \ diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 83a7b8b9baad..7809cf554248 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. +ARG arch ARG base -FROM ${base} +FROM --platform=linux/${arch} ${base} ARG tz="UTC" ENV TZ=${tz} diff --git a/ci/docker/linux-apt-ruby.dockerfile b/ci/docker/linux-apt-ruby.dockerfile index 58fd65bd57a7..9652751d75c4 100644 --- a/ci/docker/linux-apt-ruby.dockerfile +++ b/ci/docker/linux-apt-ruby.dockerfile @@ -16,8 +16,9 @@ # under the License. # depends on a C GLib image +ARG arch ARG base -FROM ${base} +FROM --platform=linux/${arch} ${base} COPY ruby/ /arrow/ruby/ RUN bundle install --gemfile /arrow/ruby/Gemfile diff --git a/ci/docker/linux-dnf-python-3.dockerfile b/ci/docker/linux-dnf-python-3.dockerfile index 98f9f64885db..6c89ebf5eb53 100644 --- a/ci/docker/linux-dnf-python-3.dockerfile +++ b/ci/docker/linux-dnf-python-3.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. +ARG arch ARG base -FROM ${base} +FROM --platform=linux/${arch} ${base} RUN dnf install -y \ python3 \ diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile index 4b84117a1cbf..f79973d887f5 100644 --- a/ci/docker/python-wheel-manylinux-test.dockerfile +++ b/ci/docker/python-wheel-manylinux-test.dockerfile @@ -17,7 +17,7 @@ ARG arch ARG python_image_tag -FROM ${arch}/python:${python_image_tag} +FROM --platform=linux/${arch} python:${python_image_tag} # pandas doesn't provide wheel for aarch64 yet, so cache the compiled # test dependencies in a docker image diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile index 80e97c440a45..e77675b8a847 100644 --- a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. -ARG base=amd64/ubuntu:22.04 -FROM ${base} +ARG arch=amd64 +ARG base=ubuntu:22.04 +FROM --platform=linux/${arch} ${base} SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 3572f156ba8c..f9966b7a8b26 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. -ARG base=amd64/ubuntu:22.04 -FROM ${base} +ARG arch=amd64 +ARG base=ubuntu:22.04 +FROM --platform=linux/${arch} ${base} SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/ci/docker/ubuntu-22.04-verify-rc.dockerfile b/ci/docker/ubuntu-22.04-verify-rc.dockerfile index b9f130d24ea9..596410a5390b 100644 --- a/ci/docker/ubuntu-22.04-verify-rc.dockerfile +++ b/ci/docker/ubuntu-22.04-verify-rc.dockerfile @@ -16,7 +16,8 @@ # under the License. ARG arch=amd64 -FROM ${arch}/ubuntu:22.04 +ARG base=ubuntu:22.04 +FROM --platform=linux/${arch} ${base} ENV DEBIAN_FRONTEND=noninteractive COPY dev/release/setup-ubuntu.sh / diff --git a/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile index 904cfa99dfef..4ba362f82667 100644 --- a/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. -ARG base=amd64/ubuntu:24.04 -FROM ${base} +ARG arch=amd64 +ARG base=ubuntu:24.04 +FROM --platform=linux/${arch} ${base} SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 3798e0817326..e7a3d47ebd05 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. -ARG base=amd64/ubuntu:24.04 -FROM ${base} +ARG arch=amd64 +ARG base=ubuntu:24.04 +FROM --platform=linux/${arch} ${base} SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/ci/docker/ubuntu-24.04-verify-rc.dockerfile b/ci/docker/ubuntu-24.04-verify-rc.dockerfile index 42d71afcb099..07714016a047 100644 --- a/ci/docker/ubuntu-24.04-verify-rc.dockerfile +++ b/ci/docker/ubuntu-24.04-verify-rc.dockerfile @@ -16,7 +16,8 @@ # under the License. ARG arch=amd64 -FROM ${arch}/ubuntu:24.04 +ARG base=ubuntu:24.04 +FROM --platform=linux/${arch} ${base} ENV DEBIAN_FRONTEND=noninteractive COPY dev/release/setup-ubuntu.sh / diff --git a/compose.yaml b/compose.yaml index 4a4cf60cce13..c13bd831ecff 100644 --- a/compose.yaml +++ b/compose.yaml @@ -22,8 +22,8 @@ # defaults are set in .env file. # # Example: -# $ ARCH=arm64v8 docker compose build ubuntu-cpp -# $ ARCH=arm64v8 docker compose run ubuntu-cpp +# $ ARCH=arm64/v8 docker compose build ubuntu-cpp +# $ ARCH=arm64/v8 docker compose run ubuntu-cpp # # # Coredumps @@ -183,17 +183,17 @@ x-hierarchy: volumes: almalinux-ccache: - name: ${ARCH}-almalinux-ccache + name: ${ARCH_SHORT}-almalinux-ccache alpine-linux-ccache: - name: ${ARCH}-alpine-linux-ccache + name: ${ARCH_SHORT}-alpine-linux-ccache conda-ccache: - name: ${ARCH}-conda-ccache + name: ${ARCH_SHORT}-conda-ccache cpp-jni-ccache: - name: ${ARCH}-cpp-jni-ccache + name: ${ARCH_SHORT}-cpp-jni-ccache debian-ccache: - name: ${ARCH}-debian-${DEBIAN}-ccache + name: ${ARCH_SHORT}-debian-${DEBIAN}-ccache fedora-ccache: - name: ${ARCH}-fedora-${FEDORA}-ccache + name: ${ARCH_SHORT}-fedora-${FEDORA}-ccache maven-cache: name: maven-cache python-wheel-manylinux-2-28-ccache: @@ -201,7 +201,7 @@ volumes: python-wheel-musllinux-1-2-ccache: name: python-wheel-musllinux-1-2-ccache ubuntu-ccache: - name: ${ARCH}-ubuntu-${UBUNTU}-ccache + name: ${ARCH_SHORT}-ubuntu-${UBUNTU}-ccache services: @@ -222,13 +222,13 @@ services: # docker compose run --rm alpine-linux-cpp # Parameters: # ALPINE_LINUX: 3.22 - # ARCH: amd64, arm64v8, ... - image: ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp + # ARCH: amd64, arm64/v8, ... + image: ${REPO}:${ARCH_SHORT}-alpine-linux-${ALPINE_LINUX}-cpp build: context: . dockerfile: ci/docker/alpine-linux-${ALPINE_LINUX}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp + - ${REPO}:${ARCH_SHORT}-alpine-linux-${ALPINE_LINUX}-cpp args: arch: ${ARCH} shm_size: &shm-size 2G @@ -253,13 +253,13 @@ services: # docker compose build conda # docker compose run --rm conda # Parameters: - # ARCH: amd64, arm32v7 - image: ${REPO}:${ARCH}-conda + # ARCH: amd64, arm/v7 + image: ${REPO}:${ARCH_SHORT}-conda build: context: . dockerfile: ci/docker/conda.dockerfile cache_from: - - ${REPO}:${ARCH}-conda + - ${REPO}:${ARCH_SHORT}-conda args: arch: ${ARCH} volumes: @@ -273,16 +273,17 @@ services: # docker compose build conda-cpp # docker compose run --rm conda-cpp # Parameters: - # ARCH: amd64, arm32v7 - image: ${REPO}:${ARCH}-conda-cpp + # ARCH: amd64, arm/v7 + image: ${REPO}:${ARCH_SHORT}-conda-cpp build: context: . dockerfile: ci/docker/conda-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-cpp + - ${REPO}:${ARCH_SHORT}-conda-cpp args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} shm_size: *shm-size ulimits: *ulimits environment: @@ -308,16 +309,17 @@ services: # docker compose build conda-cpp # docker compose run --rm conda-cpp-valgrind # Parameters: - # ARCH: amd64, arm32v7 - image: ${REPO}:${ARCH}-conda-cpp + # ARCH: amd64, arm/v7 + image: ${REPO}:${ARCH_SHORT}-conda-cpp build: context: . dockerfile: ci/docker/conda-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-cpp + - ${REPO}:${ARCH_SHORT}-conda-cpp args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} shm_size: *shm-size environment: <<: [*common, *ccache, *sccache, *cpp] @@ -344,14 +346,14 @@ services: # docker compose build debian-cpp # docker compose run --rm debian-cpp # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # DEBIAN: 12, experimental - image: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp + image: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-cpp build: context: . dockerfile: ci/docker/debian-${DEBIAN}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-debian-${DEBIAN}-cpp + - ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-cpp args: arch: ${ARCH} gcc: ${GCC} @@ -375,17 +377,17 @@ services: # docker compose build ubuntu-cpp # docker compose run --rm ubuntu-cpp # Parameters: - # ARCH: amd64, arm64v8, s390x, ... + # ARCH: amd64, arm64/v8, s390x, ... # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp args: arch: ${ARCH} - base: "${ARCH}/ubuntu:${UBUNTU}" + base: "ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} cmake: ${CMAKE} gcc: ${GCC} @@ -412,17 +414,17 @@ services: # docker compose build ubuntu-cpp-static # docker compose run --rm ubuntu-cpp-static # Parameters: - # ARCH: amd64, arm64v8, s390x, ... + # ARCH: amd64, arm64/v8, s390x, ... # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-static build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-static args: arch: ${ARCH} - base: "${ARCH}/ubuntu:${UBUNTU}" + base: "ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} cmake: ${CMAKE} gcc: ${GCC} @@ -451,15 +453,15 @@ services: ubuntu-cpp-bundled: # Arrow build with BUNDLED dependencies - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-minimal build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-minimal args: arch: ${ARCH} - base: "${ARCH}/ubuntu:${UBUNTU}" + base: "ubuntu:${UBUNTU}" cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size @@ -473,15 +475,15 @@ services: ubuntu-cpp-bundled-offline: # Arrow build with BUNDLED dependencies with downloaded dependencies. - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-minimal build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-minimal args: arch: ${ARCH} - base: "${ARCH}/ubuntu:${UBUNTU}" + base: "ubuntu:${UBUNTU}" llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -524,15 +526,15 @@ services: ubuntu-cpp-minimal: # Arrow build with minimal components/dependencies - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-minimal build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp-minimal args: arch: ${ARCH} - base: "${ARCH}/ubuntu:${UBUNTU}" + base: "ubuntu:${UBUNTU}" llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -573,12 +575,12 @@ services: # ARCH: amd64 # UBUNTU: 22.04, 24.04 # CUDA: - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp args: arch: ${ARCH} base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} @@ -624,9 +626,9 @@ services: # docker compose build ubuntu-cpp-sanitizer # docker compose run --rm ubuntu-cpp-sanitizer # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # UBUNTU: 22.04, ... - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp cap_add: # For LeakSanitizer - SYS_PTRACE @@ -634,7 +636,7 @@ services: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} @@ -673,14 +675,14 @@ services: # docker compose build ubuntu-cpp-thread-sanitizer # docker compose run --rm ubuntu-cpp-thread-sanitizer # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} @@ -707,14 +709,14 @@ services: # docker compose build ubuntu-cpp-emscripten # docker compose run --rm ubuntu-cpp-emscripten # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # UBUNTU: 22.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} @@ -733,14 +735,14 @@ services: # docker compose build fedora-cpp # docker compose run --rm fedora-cpp # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # FEDORA: 42 - image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp + image: ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-cpp build: context: . dockerfile: ci/docker/fedora-${FEDORA}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-fedora-${FEDORA}-cpp + - ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-cpp args: arch: ${ARCH} llvm: ${LLVM} @@ -761,10 +763,10 @@ services: # Usage: # docker compose run --rm cpp-jni # Parameters: - # ARCH: amd64, arm64v8 + # ARCH: amd64, arm64/v8 # ARCH_ALIAS: x86_64, aarch64 # ARCH_SHORT: amd64, arm64 - image: ${REPO}:${ARCH}-cpp-jni-${VCPKG} + image: ${REPO}:${ARCH_SHORT}-cpp-jni-${VCPKG} build: args: arch: ${ARCH} @@ -777,7 +779,7 @@ services: context: . dockerfile: ci/docker/cpp-jni.dockerfile cache_from: - - ${REPO}:${ARCH}-cpp-jni-${VCPKG} + - ${REPO}:${ARCH_SHORT}-cpp-jni-${VCPKG} secrets: *vcpkg-build-secrets environment: <<: [*common, *ccache] @@ -794,16 +796,17 @@ services: # docker compose build debian-c-glib # docker compose run --rm debian-c-glib # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # DEBIAN: 12 - image: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib + image: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-c-glib build: context: . dockerfile: ci/docker/linux-apt-c-glib.dockerfile cache_from: - - ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib + - ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-c-glib args: - base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-cpp shm_size: *shm-size ulimits: *ulimits environment: @@ -822,16 +825,17 @@ services: # docker compose build ubuntu-c-glib # docker compose run --rm ubuntu-c-glib # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-c-glib build: context: . dockerfile: ci/docker/linux-apt-c-glib.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-c-glib args: - base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp shm_size: *shm-size ulimits: *ulimits environment: @@ -852,16 +856,17 @@ services: # docker compose build debian-ruby # docker compose run --rm debian-ruby # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # DEBIAN: 12 - image: ${REPO}:${ARCH}-debian-${DEBIAN}-ruby + image: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-ruby build: context: . dockerfile: ci/docker/linux-apt-ruby.dockerfile cache_from: - - ${REPO}:${ARCH}-debian-${DEBIAN}-ruby + - ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-ruby args: - base: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-c-glib shm_size: *shm-size ulimits: *ulimits environment: @@ -882,16 +887,17 @@ services: # docker compose build ubuntu-ruby # docker compose run --rm ubuntu-ruby # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-ruby build: context: . dockerfile: ci/docker/linux-apt-ruby.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-ruby args: - base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-c-glib shm_size: *shm-size ulimits: *ulimits environment: @@ -908,17 +914,18 @@ services: # docker compose build conda-python # docker compose run --rm conda-python # Parameters: - # ARCH: amd64, arm32v7 + # ARCH: amd64, arm/v7 # PYTHON: 3.10, 3.11, 3.12, 3.13 - image: ${REPO}:${ARCH}-conda-python-${PYTHON} + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON} build: context: . dockerfile: ci/docker/conda-python.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON} + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON} args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} python: ${PYTHON} shm_size: *shm-size environment: @@ -936,17 +943,18 @@ services: # docker compose build conda-python-emscripten # docker compose run --rm conda-python-emscripten # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # UBUNTU: 22.04 - image: ${REPO}:${ARCH}-conda-python-emscripten + image: ${REPO}:${ARCH_SHORT}-conda-python-emscripten build: context: . dockerfile: ci/docker/conda-python-emscripten.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON} + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON} args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} pyodide_version: "0.26.0" @@ -974,14 +982,15 @@ services: # UBUNTU: 22.04, 24.04 # NUMBA: master, latest, # NUMBA_CUDA: master, latest, - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 build: context: . dockerfile: ci/docker/linux-apt-python-3.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 args: - base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp numba: ${NUMBA} numba_cuda: ${NUMBA_CUDA} cuda: ${CUDA} @@ -1021,16 +1030,17 @@ services: # docker compose build debian-python # docker compose run --rm debian-python # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # DEBIAN: 12 - image: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 + image: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-python-3 build: context: . dockerfile: ci/docker/linux-apt-python-3.dockerfile cache_from: - - ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 + - ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-python-3 args: - base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-cpp shm_size: *shm-size environment: <<: [*common, *ccache] @@ -1043,16 +1053,17 @@ services: # docker compose build ubuntu-python # docker compose run --rm ubuntu-python # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-python-3 build: context: . dockerfile: ci/docker/linux-apt-python-3.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-python-3 args: - base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp shm_size: *shm-size environment: <<: [*common, *ccache] @@ -1065,16 +1076,17 @@ services: # docker compose build fedora-python # docker compose run --rm fedora-python # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # FEDORA: 42 - image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 + image: ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-python-3 build: context: . dockerfile: ci/docker/linux-dnf-python-3.dockerfile cache_from: - - ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 + - ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-python-3 args: - base: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-cpp shm_size: *shm-size environment: <<: [*common, *ccache] @@ -1107,17 +1119,18 @@ services: # docker compose build ubuntu-python-sdist-test # docker compose run --rm ubuntu-python-sdist-test # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64/v8, ... # PYARROW_VERSION: The test target pyarrow version such as "3.0.0" # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-python-3 build: context: . dockerfile: ci/docker/linux-apt-python-3.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-python-3 args: - base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + arch: ${ARCH} + base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp shm_size: *shm-size environment: <<: [*common, *ccache] @@ -1136,7 +1149,7 @@ services: # See available versions at: # https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags python-wheel-manylinux-2-28: - image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} build: args: arch: ${ARCH} @@ -1149,7 +1162,7 @@ services: context: . dockerfile: ci/docker/python-wheel-manylinux.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} + - ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} secrets: *vcpkg-build-secrets environment: <<: [*common, *ccache] @@ -1161,7 +1174,7 @@ services: # See available versions at: # https://quay.io/repository/pypa/musllinux_1_2_x86_64?tab=tags python-wheel-musllinux-1-2: - image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG} + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG} build: args: arch: ${ARCH} @@ -1174,7 +1187,7 @@ services: context: . dockerfile: ci/docker/python-wheel-musllinux.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG} + - ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG} secrets: *vcpkg-build-secrets environment: <<: [ *common, *ccache ] @@ -1202,17 +1215,17 @@ services: # TODO: Remove this when the official Docker Python image supports the free-threaded build. # See https://github.com/docker-library/python/issues/947 for more info. python-free-threaded-wheel-musllinux-test-imports: - image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-imports + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-imports build: args: - base: "${ARCH}/alpine:${ALPINE_LINUX}" + base: "alpine:${ALPINE_LINUX}" python_version: ${PYTHON} build_date: "20260510" # python-build-standalone release date arch: ${ARCH_ALIAS} context: . dockerfile: ci/docker/python-free-threaded-wheel-musllinux-test-imports.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-imports + - ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-imports shm_size: 2G volumes: - .:/arrow:delegated @@ -1224,7 +1237,7 @@ services: command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow python-wheel-musllinux-test-unittests: - image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-test + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-musllinux-test build: args: alpine_linux: ${ALPINE_LINUX} @@ -1232,7 +1245,7 @@ services: context: . dockerfile: ci/docker/python-wheel-musllinux-test.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-test + - ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-musllinux-test shm_size: 2G volumes: - .:/arrow:delegated @@ -1246,17 +1259,17 @@ services: # TODO: Remove this when the official Docker Python image supports the free-threaded build. # See https://github.com/docker-library/python/issues/947 for more info. python-free-threaded-wheel-musllinux-test-unittests: - image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-unittests + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-unittests build: args: - base: "${ARCH}/alpine:${ALPINE_LINUX}" + base: "alpine:${ALPINE_LINUX}" python_version: ${PYTHON} build_date: "20260510" # python-build-standalone release date arch: ${ARCH_ALIAS} context: . dockerfile: ci/docker/python-free-threaded-wheel-musllinux-test-unittests.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-unittests + - ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-unittests shm_size: 2G volumes: - .:/arrow:delegated @@ -1268,7 +1281,8 @@ services: command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow python-wheel-manylinux-test-imports: - image: ${ARCH}/python:${PYTHON_IMAGE_TAG} + image: python:${PYTHON_IMAGE_TAG} + platform: linux/${ARCH} shm_size: 2G volumes: - .:/arrow:delegated @@ -1282,15 +1296,15 @@ services: # TODO: Remove this when the official Docker Python image supports the free-threaded build. # See https://github.com/docker-library/python/issues/947 for more info. python-free-threaded-wheel-manylinux-test-imports: - image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-imports + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-imports build: args: - base: "${ARCH}/ubuntu:${UBUNTU}" + base: "ubuntu:${UBUNTU}" python_version: ${PYTHON} context: . dockerfile: ci/docker/python-free-threaded-wheel-manylinux-test-imports.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-imports + - ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-imports shm_size: 2G volumes: - .:/arrow:delegated @@ -1302,7 +1316,7 @@ services: command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow python-wheel-manylinux-test-unittests: - image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-manylinux-test build: args: arch: ${ARCH} @@ -1311,7 +1325,7 @@ services: context: . dockerfile: ci/docker/python-wheel-manylinux-test.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test + - ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-manylinux-test shm_size: 2G volumes: - .:/arrow:delegated @@ -1325,15 +1339,15 @@ services: # TODO: Remove this when the official Docker Python image supports the free-threaded build. # See https://github.com/docker-library/python/issues/947 for more info. python-free-threaded-wheel-manylinux-test-unittests: - image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-unittests + image: ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-unittests build: args: - base: "${ARCH}/ubuntu:${UBUNTU}" + base: "ubuntu:${UBUNTU}" python_version: ${PYTHON} context: . dockerfile: ci/docker/python-free-threaded-wheel-manylinux-test-unittests.dockerfile cache_from: - - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-unittests + - ${REPO}:${ARCH_SHORT}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-unittests shm_size: 2G volumes: - .:/arrow:delegated @@ -1436,15 +1450,16 @@ services: # docker compose build conda-python # docker compose build conda-python-pandas # docker compose run --rm conda-python-pandas - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-pandas-${PANDAS} build: context: . dockerfile: ci/docker/conda-python-pandas.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-pandas-${PANDAS} args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} python: ${PYTHON} numpy: ${NUMPY} pandas: ${PANDAS} @@ -1466,15 +1481,16 @@ services: # docker compose build conda-python # docker compose build conda-python-no-numpy # docker compose run --rm conda-python-no-numpy - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-no-numpy + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-no-numpy build: context: . dockerfile: ci/docker/conda-python.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON} + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON} args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} python: ${PYTHON} shm_size: *shm-size environment: @@ -1497,7 +1513,7 @@ services: # Only a single rule is enabled for now to check undocumented arguments. # We should extend the list of enabled rules after adding this build to # the CI pipeline. - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-pandas-${PANDAS} cap_add: - SYS_ADMIN environment: @@ -1529,15 +1545,16 @@ services: # docker compose build conda-python # docker compose build conda-python-dask # docker compose run --rm conda-python-dask - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK} + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-dask-${DASK} build: context: . dockerfile: ci/docker/conda-python-dask.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK} + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-dask-${DASK} args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} python: ${PYTHON} dask: ${DASK} shm_size: *shm-size @@ -1556,15 +1573,16 @@ services: # docker compose build conda-python # docker compose build conda-python-cpython-debug # docker compose run --rm conda-python-cpython-debug - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-cpython-debug build: context: . dockerfile: ci/docker/conda-python-cpython-debug.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-cpython-debug args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} python: ${PYTHON} shm_size: *shm-size environment: @@ -1580,15 +1598,15 @@ services: # docker compose build ubuntu-cpp # docker compose build ubuntu-r # docker compose run ubuntu-r - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R} + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-r-${R} build: context: . dockerfile: ci/docker/linux-apt-r.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R} + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-r-${R} args: arch: ${ARCH} - base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp gcc: ${GCC} r: ${R} r_duckdb_dev: ${R_DUCKDB_DEV:-} @@ -1747,13 +1765,13 @@ services: # Parameters: # ALPINE_LINUX: 3.22 # R: 4.5 (Alpine's R version) - # ARCH: amd64, arm64v8, ... - image: ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-r + # ARCH: amd64, arm64/v8, ... + image: ${REPO}:${ARCH_SHORT}-alpine-linux-${ALPINE_LINUX}-r build: context: . dockerfile: ci/docker/alpine-linux-${ALPINE_LINUX}-r.dockerfile cache_from: - - ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-r + - ${REPO}:${ARCH_SHORT}-alpine-linux-${ALPINE_LINUX}-r args: arch: ${ARCH} shm_size: *shm-size @@ -1780,12 +1798,12 @@ services: # Parameters: # FEDORA: 42 # ARCH: amd64 - image: ${REPO}:${ARCH}-fedora-${FEDORA}-r-clang + image: ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-r-clang build: context: . dockerfile: ci/docker/fedora-${FEDORA}-r-clang.dockerfile cache_from: - - ${REPO}:${ARCH}-fedora-${FEDORA}-r-clang + - ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-r-clang args: arch: ${ARCH} shm_size: *shm-size @@ -1807,15 +1825,16 @@ services: # docker compose build conda-cpp # docker compose build conda-integration # docker compose run conda-integration - image: ${REPO}:${ARCH}-conda-integration + image: ${REPO}:${ARCH_SHORT}-conda-integration build: context: . dockerfile: ci/docker/conda-integration.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-integration + - ${REPO}:${ARCH_SHORT}-conda-integration args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} # Use a newer JDK as it seems to improve stability jdk: 17 maven: ${MAVEN} @@ -1843,16 +1862,16 @@ services: # docker compose build debian-python # docker compose build debian-docs # docker compose run --rm debian-docs - image: ${REPO}:${ARCH}-debian-${DEBIAN}-docs + image: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-docs build: context: . dockerfile: ci/docker/linux-apt-docs.dockerfile cache_from: - - ${REPO}:${ARCH}-debian-${DEBIAN}-docs + - ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-docs args: r: ${R} node: ${NODE} - base: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 + base: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-python-3 # This is for Chromium used by Mermaid. Chromium uses namespace # isolation for security by default. cap_add: @@ -1919,15 +1938,16 @@ services: # docker compose build conda-python # docker compose build conda-python-hdfs # docker compose run conda-python-hdfs - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS} + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-hdfs-${HDFS} build: context: . dockerfile: ci/docker/conda-python-hdfs.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS} + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-hdfs-${HDFS} args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} python: ${PYTHON} jdk: ${JDK} maven: ${MAVEN} @@ -1960,15 +1980,16 @@ services: # docker compose build conda-python # docker compose build conda-python-spark # docker compose run conda-python-spark - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK} + image: ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-spark-${SPARK} build: context: . dockerfile: ci/docker/conda-python-spark.dockerfile cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK} + - ${REPO}:${ARCH_SHORT}-conda-python-${PYTHON}-spark-${SPARK} args: repo: ${REPO} arch: ${ARCH} + arch_short: ${ARCH_SHORT} python: ${PYTHON} jdk: ${JDK} maven: ${MAVEN} @@ -2011,12 +2032,12 @@ services: # docker compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 almalinux-verify-rc # Parameters: # ALMALINUX: 8 - image: ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc + image: ${REPO}:${ARCH_SHORT}-almalinux-${ALMALINUX}-verify-rc build: context: . dockerfile: ci/docker/almalinux-${ALMALINUX}-verify-rc.dockerfile cache_from: - - ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc + - ${REPO}:${ARCH_SHORT}-almalinux-${ALMALINUX}-verify-rc args: repo: ${REPO} arch: ${ARCH} @@ -2040,12 +2061,12 @@ services: # docker compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 ubuntu-verify-rc # Parameters: # UBUNTU: 22.04, 24.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc + image: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-verify-rc build: context: . dockerfile: ci/docker/ubuntu-${UBUNTU}-verify-rc.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc + - ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-verify-rc args: arch: ${ARCH} cmake: ${CMAKE} diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index bfb1e89f983e..80658629e6f2 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -45,11 +45,11 @@ def flatten(node, parents=None): _arch_short_mapping = { - 'arm64v8': 'arm64', + 'arm64/v8': 'arm64', } _arch_alias_mapping = { 'amd64': 'x86_64', - 'arm64v8': 'aarch64', + 'arm64/v8': 'aarch64', } diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index df5648f9e9ef..e211343953bf 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -38,7 +38,7 @@ jobs: {% if arch == "amd64" %} ARCH: amd64 {% else %} - ARCH: arm64v8 + ARCH: arm64/v8 {% endif %} PYTHON: "{{ python_version }}" PYTHON_ABI_TAG: "{{ python_abi_tag }}"