From 4959972ac4985f7636daf4672fdd98db43623edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 26 Jun 2026 09:31:32 +0200 Subject: [PATCH 01/15] feat(ci): add disconnected OCP smoke test for Helm and Operator Add end-to-end disconnected CI pipeline handlers that deploy RHDH in an isolated OCP cluster and run a Playwright smoke test. Helm path uses oc-mirror v2 (downloaded at runtime) for chart + image mirroring, following the documented air-gapped workflow: - GA (registry.redhat.io): chart from charts.openshift.io, oc-mirror discovers and mirrors default images automatically - CI/upstream: chart from oci://quay.io/rhdh/chart via helm.local, override images added as additionalImages - oc-mirror generates IDMS, patched with cross-registry entries for both quay.io and registry.redhat.io hub image sources - Chart installed from local tgz in oc-mirror workspace Operator path uses prepare-restricted-environment.sh from rhdh-operator for operator+operand mirroring and installation (documented approach). Both paths share: - Auth setup (REGISTRY_AUTH_FILE + XDG_RUNTIME_DIR/containers/auth.json) - Plugin mirroring via mirror-plugins.sh with registries.conf covering registry.access.redhat.com/rhdh, quay.io/rhdh, and ghcr.io/redhat-developer/rhdh-plugin-export-overlays (6 CI plugins) - Helm overrides for registries.conf volume mount (avoids array clobber) - CATALOG_INDEX_IMAGE override support for CI build verification Dispatcher routing in openshift-ci-tests.sh for *ocp*disconnected*helm*nightly* and *ocp*disconnected*operator*nightly* patterns, positioned before generic *ocp*helm*nightly* to prevent false matches. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 196 ++++++++++++ .../jobs/ocp-disconnected-operator.sh | 129 ++++++++ .ci/pipelines/lib/disconnected.sh | 287 ++++++++++++++++++ .ci/pipelines/openshift-ci-tests.sh | 14 + .../disconnected/helm-overrides.yaml | 52 ++++ .../disconnected/plugin-mirror-configmap.yaml | 17 ++ 6 files changed, 695 insertions(+) create mode 100644 .ci/pipelines/jobs/ocp-disconnected-helm.sh create mode 100644 .ci/pipelines/jobs/ocp-disconnected-operator.sh create mode 100644 .ci/pipelines/lib/disconnected.sh create mode 100644 .ci/pipelines/resources/disconnected/helm-overrides.yaml create mode 100644 .ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh new file mode 100644 index 0000000000..4925e410d1 --- /dev/null +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -0,0 +1,196 @@ +#!/bin/bash + +# shellcheck source=.ci/pipelines/lib/log.sh +source "$DIR"/lib/log.sh +# shellcheck source=.ci/pipelines/lib/common.sh +source "$DIR"/lib/common.sh +# shellcheck source=.ci/pipelines/utils.sh +source "$DIR"/utils.sh +# shellcheck source=.ci/pipelines/lib/testing.sh +source "$DIR"/lib/testing.sh +# shellcheck source=.ci/pipelines/playwright-projects.sh +source "$DIR"/playwright-projects.sh +# shellcheck source=.ci/pipelines/lib/disconnected.sh +source "$DIR"/lib/disconnected.sh + +export INSTALL_METHOD="helm" + +handle_ocp_disconnected_helm() { + export NAME_SPACE="${NAME_SPACE:-showcase-ci-disconnected}" + + disconnected::require_env + disconnected::setup_auth + + common::oc_login + + K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') + export K8S_CLUSTER_ROUTER_BASE + + # --- Section A: Install oc-mirror --- + log::section "oc-mirror Setup" + + disconnected::install_oc_mirror || { + log::error "Failed to install oc-mirror — aborting" + return 1 + } + + # --- Section B: Resolve chart source and pull locally --- + log::section "Chart Resolution" + + local is_ga="false" + if [[ "${IMAGE_REGISTRY}" == "registry.redhat.io" ]]; then + is_ga="true" + fi + + if [[ "${is_ga}" == "true" ]]; then + # GA: pull chart from charts.openshift.io + helm repo add openshift-helm-charts https://charts.openshift.io 2> /dev/null || true + helm repo update openshift-helm-charts + log::info "Pulling GA chart from charts.openshift.io (version: ${RELEASE_VERSION})" + helm pull openshift-helm-charts/redhat-developer-hub \ + --version "${RELEASE_VERSION}" \ + -d "${DISCONNECTED_TMPDIR}" || { + log::error "Failed to pull chart from charts.openshift.io" + return 1 + } + else + # CI/upstream: pull chart from OCI registry + log::info "Pulling CI chart from ${HELM_CHART_URL} (version: ${CHART_VERSION})" + helm pull "${HELM_CHART_URL}" --version "${CHART_VERSION}" \ + -d "${DISCONNECTED_TMPDIR}" || { + log::error "Failed to pull chart from ${HELM_CHART_URL}" + return 1 + } + fi + + CHART_LOCAL_TGZ=$(find "${DISCONNECTED_TMPDIR}" -maxdepth 1 -name '*.tgz' | head -1) + export CHART_LOCAL_TGZ + + if [[ -z "${CHART_LOCAL_TGZ}" ]]; then + log::error "No chart .tgz found in ${DISCONNECTED_TMPDIR}" + return 1 + fi + log::success "Chart pulled: ${CHART_LOCAL_TGZ}" + + # --- Section C: Resolve PostgreSQL image from chart --- + local helm_values + helm_values=$(helm show values "${CHART_LOCAL_TGZ}" 2> /dev/null || true) + + export PG_REGISTRY PG_REPO PG_TAG + PG_REGISTRY=$(echo "${helm_values}" | yq '.upstream.postgresql.image.registry' || true) + PG_REPO=$(echo "${helm_values}" | yq '.upstream.postgresql.image.repository' || true) + PG_TAG=$(echo "${helm_values}" | yq '.upstream.postgresql.image.tag' || true) + PG_REGISTRY="${PG_REGISTRY:-registry.redhat.io}" + PG_REPO="${PG_REPO:-rhel9/postgresql-15}" + PG_TAG="${PG_TAG:-latest}" + + log::info "PostgreSQL image from chart: ${PG_REGISTRY}/${PG_REPO}:${PG_TAG}" + + # --- Section D: Build ImageSetConfiguration --- + log::section "Image Mirroring" + + local imageset_config="${DISCONNECTED_TMPDIR}/imageset-config.yaml" + disconnected::build_imageset_config "${imageset_config}" || { + log::error "Failed to build ImageSetConfiguration" + return 1 + } + + # --- Section E: Run oc-mirror --- + local workspace="${DISCONNECTED_TMPDIR}/oc-mirror-workspace" + disconnected::run_oc_mirror "${imageset_config}" "${workspace}" || { + log::error "oc-mirror failed — aborting" + return 1 + } + + # --- Section F: Patch and apply IDMS --- + log::section "Cluster Resources" + + disconnected::patch_idms "${OC_MIRROR_IDMS_FILE}" + + oc apply -f "${OC_MIRROR_IDMS_FILE}" || { + log::error "Failed to apply IDMS — aborting" + return 1 + } + log::success "ImageDigestMirrorSet applied" + + if [[ -n "${OC_MIRROR_ITMS_FILE:-}" ]]; then + oc apply -f "${OC_MIRROR_ITMS_FILE}" || { + log::error "Failed to apply ITMS — aborting" + return 1 + } + log::success "ImageTagMirrorSet applied" + fi + + # --- Section G: Plugin mirroring --- + log::section "Plugin Mirroring" + + disconnected::fetch_script "mirror-plugins.sh" "${DISCONNECTED_TMPDIR}/mirror-plugins.sh" || { + log::error "Failed to fetch mirror-plugins.sh — aborting" + return 1 + } + + local plugin_index="oci://registry.access.redhat.com/rhdh/plugin-catalog-index:${RELEASE_VERSION}" + if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then + plugin_index="oci://${CATALOG_INDEX_IMAGE}" + fi + + bash "${DISCONNECTED_TMPDIR}/mirror-plugins.sh" \ + --plugin-index "${plugin_index}" \ + --to-registry "${MIRROR_REGISTRY_URL}" || { + log::error "mirror-plugins.sh failed — aborting" + return 1 + } + + # --- Section H: Namespace + registries.conf ConfigMap --- + namespace::configure "${NAME_SPACE}" + + envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ + | oc apply -n "${NAME_SPACE}" -f - || { + log::error "Failed to create registries.conf ConfigMap — aborting" + return 1 + } + log::success "ConfigMap rhdh-plugin-mirror-conf created in ${NAME_SPACE}" + + # --- Section I: Helm deployment from mirrored chart --- + log::section "Helm Deployment" + + # Prefer the chart from oc-mirror workspace, fall back to the pulled tgz + local chart_install_path + chart_install_path="${OC_MIRROR_CHART_PATH:-${CHART_LOCAL_TGZ}}" + log::info "Installing chart from: ${chart_install_path}" + + local helm_set_flags=( + --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" + --set upstream.backstage.image.registry="${MIRROR_REGISTRY_URL}" + --set upstream.backstage.image.repository="${IMAGE_REPO}" + --set upstream.backstage.image.tag="${TAG_NAME}" + --set upstream.postgresql.image.registry="${MIRROR_REGISTRY_URL}" + ) + + if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then + helm_set_flags+=( + --set global.catalogIndex.image.registry="${MIRROR_REGISTRY_URL}" + --set global.catalogIndex.image.repository="${CATALOG_INDEX_REPO}" + --set global.catalogIndex.image.tag="${CATALOG_INDEX_TAG}" + ) + fi + + helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \ + "${chart_install_path}" \ + -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" \ + -f "${DIR}/resources/disconnected/helm-overrides.yaml" \ + "${helm_set_flags[@]}" || { + log::error "Helm deployment failed" + return 1 + } + + log::success "RHDH deployed via Helm with mirrored images" + + # --- Section J: Smoke test --- + log::section "Smoke Test" + + local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SMOKE_TEST}" "${url}" + + log::success "Disconnected Helm smoke test completed" +} diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh new file mode 100644 index 0000000000..0c6093a63b --- /dev/null +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +# shellcheck source=.ci/pipelines/lib/log.sh +source "$DIR"/lib/log.sh +# shellcheck source=.ci/pipelines/lib/common.sh +source "$DIR"/lib/common.sh +# shellcheck source=.ci/pipelines/utils.sh +source "$DIR"/utils.sh +# shellcheck source=.ci/pipelines/install-methods/operator.sh +source "$DIR"/install-methods/operator.sh +# shellcheck source=.ci/pipelines/lib/testing.sh +source "$DIR"/lib/testing.sh +# shellcheck source=.ci/pipelines/playwright-projects.sh +source "$DIR"/playwright-projects.sh +# shellcheck source=.ci/pipelines/lib/disconnected.sh +source "$DIR"/lib/disconnected.sh + +export INSTALL_METHOD="operator" + +handle_ocp_disconnected_operator() { + export NAME_SPACE="${NAME_SPACE:-showcase-disconnected}" + + disconnected::require_env + disconnected::setup_auth + + common::oc_login + + K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') + export K8S_CLUSTER_ROUTER_BASE + + # --- Section A: Operator Mirroring + Installation --- + # Uses prepare-restricted-environment.sh from rhdh-operator, which handles + # mirroring operator/operand images and installing the operator CatalogSource. + log::section "Operator Mirroring and Installation" + + disconnected::fetch_script "prepare-restricted-environment.sh" "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" \ + || { + log::error "Failed to fetch prepare-restricted-environment.sh — aborting" + return 1 + } + + local prepare_args=( + --to-registry "${MIRROR_REGISTRY_URL}" + --filter-versions "${RELEASE_VERSION}" + ) + if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then + prepare_args=( + --to-registry "${MIRROR_REGISTRY_URL}" + --index-image "${CATALOG_INDEX_IMAGE}" + --ci-index true + --filter-versions "${RELEASE_VERSION}" + ) + fi + + bash "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" "${prepare_args[@]}" \ + || { + log::error "prepare-restricted-environment.sh failed — aborting" + return 1 + } + log::success "Operator installed via prepare-restricted-environment.sh" + + # --- Section B: Wait for Operator CRD --- + k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || { + log::error "Backstage CRD not available after operator installation" + return 1 + } + + # --- Section C: Plugin Mirroring --- + log::section "Plugin Mirroring" + + disconnected::fetch_script "mirror-plugins.sh" "${DISCONNECTED_TMPDIR}/mirror-plugins.sh" \ + || { + log::error "Failed to fetch mirror-plugins.sh — aborting" + return 1 + } + + local plugin_index="oci://registry.access.redhat.com/rhdh/plugin-catalog-index:${RELEASE_VERSION}" + if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then + plugin_index="oci://${CATALOG_INDEX_IMAGE}" + fi + + bash "${DISCONNECTED_TMPDIR}/mirror-plugins.sh" \ + --plugin-index "${plugin_index}" \ + --to-registry "${MIRROR_REGISTRY_URL}" || { + log::error "mirror-plugins.sh failed — aborting" + return 1 + } + + # --- Section D: Namespace + registries.conf ConfigMap --- + log::section "Cluster Resources" + + namespace::configure "${NAME_SPACE}" + + envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ + | oc apply -n "${NAME_SPACE}" -f - || { + log::error "Failed to create registries.conf ConfigMap — aborting" + return 1 + } + log::success "ConfigMap rhdh-plugin-mirror-conf created in ${NAME_SPACE}" + + # --- Section E: Backstage CR Deployment --- + log::section "Backstage CR Deployment" + + local rendered_cr + rendered_cr=$(envsubst < "${DIR}/resources/rhdh-operator/rhdh-start.yaml") + rendered_cr=$(echo "$rendered_cr" | yq eval \ + '.spec.application.extraFiles.configMaps = [ + { + "name": "rhdh-plugin-mirror-conf", + "key": "rhdh-registries.conf", + "mountPath": "/etc/containers/registries.conf.d", + "containers": ["install-dynamic-plugins"] + } + ]' -) + + local cr_temp="${DISCONNECTED_TMPDIR}/backstage-cr-disconnected.yaml" + echo "$rendered_cr" > "${cr_temp}" + + deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" + log::success "Backstage CR deployed in ${NAME_SPACE}" + + # --- Section F: Smoke Test --- + log::section "Smoke Test" + + local url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SMOKE_TEST}" "${url}" + + log::success "Disconnected Operator smoke test completed" +} diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh new file mode 100644 index 0000000000..619a54c85d --- /dev/null +++ b/.ci/pipelines/lib/disconnected.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash + +# Shared utility functions for disconnected CI pipeline handlers. +# Provides environment validation, oc-mirror-based image mirroring, +# auth setup, and external script fetching. +# +# Dependencies: lib/log.sh, lib/common.sh +# Consumers: jobs/ocp-disconnected-helm.sh, jobs/ocp-disconnected-operator.sh + +# Prevent re-sourcing +if [[ -n "${DISCONNECTED_LIB_SOURCED:-}" ]]; then + return 0 +fi +readonly DISCONNECTED_LIB_SOURCED=1 + +# shellcheck source=.ci/pipelines/lib/log.sh +source "${DIR}/lib/log.sh" +# shellcheck source=.ci/pipelines/lib/common.sh +source "${DIR}/lib/common.sh" + +# Create a dedicated temp directory for disconnected CI artifacts. +DISCONNECTED_TMPDIR=$(mktemp -d) +export DISCONNECTED_TMPDIR + +# oc-mirror binary path, set by disconnected::install_oc_mirror. +OC_MIRROR_BIN="" +export OC_MIRROR_BIN + +# Validate that all required disconnected environment variables are set. +# These are exported by the step-registry commands.sh before calling +# openshift-ci-tests.sh. +disconnected::require_env() { + if [[ "${DISCONNECTED:-}" != "true" ]]; then + log::error "DISCONNECTED is not set to 'true'. This handler requires a disconnected environment." + log::error "Ensure the step-registry commands.sh has run before this handler." + return 1 + fi + + common::require_vars \ + MIRROR_REGISTRY_URL \ + MIRROR_REGISTRY_PULL_SECRET \ + MIRROR_REGISTRY_CA +} + +# Configure container-tools authentication for skopeo, oc-mirror, and +# mirror-plugins.sh. Places the combined pull secret (which contains +# credentials for both source registries and the mirror registry) in +# the standard locations expected by these tools. +disconnected::setup_auth() { + export HOME="${HOME:-/tmp/home}" + export XDG_RUNTIME_DIR="${HOME}/run" + mkdir -p "${XDG_RUNTIME_DIR}/containers" + + # oc-mirror and skopeo read auth from ${XDG_RUNTIME_DIR}/containers/auth.json + cp "${MIRROR_REGISTRY_PULL_SECRET}" "${XDG_RUNTIME_DIR}/containers/auth.json" + + # REGISTRY_AUTH_FILE is respected by skopeo as an explicit override + export REGISTRY_AUTH_FILE="${MIRROR_REGISTRY_PULL_SECRET}" + + # oc-mirror requires this to be unset + unset REGISTRY_AUTH_PREFERENCE + + log::info "Container auth configured from ${MIRROR_REGISTRY_PULL_SECRET}" +} + +# Download the oc-mirror binary at runtime from mirror.openshift.com. +# Uses CONTAINER_PLATFORM_VERSION to select the matching version. +disconnected::install_oc_mirror() { + local arch + arch=$(uname -m) + case ${arch} in + x86_64) arch="amd64" ;; + aarch64) arch="arm64" ;; + esac + + local ocp_version="${CONTAINER_PLATFORM_VERSION:-4.21}" + local oc_mirror_version="" + + # Try stable channel for the OCP minor version, fall back to latest + local stable_url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/stable-${ocp_version}/" + if curl -sf --head --connect-timeout 10 "${stable_url}" > /dev/null 2>&1; then + oc_mirror_version="stable-${ocp_version}" + log::info "Using oc-mirror from stable-${ocp_version} channel" + else + oc_mirror_version="latest" + log::info "stable-${ocp_version} not available, using oc-mirror from latest channel" + fi + + local download_dir="${DISCONNECTED_TMPDIR}/oc-mirror-download" + mkdir -p "${download_dir}" + + local base_url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/${oc_mirror_version}" + + log::info "Downloading oc-mirror from ${base_url}/" + if ! curl -fL --retry 5 --connect-timeout 30 -o "${download_dir}/oc-mirror.tar.gz" "${base_url}/oc-mirror.tar.gz"; then + log::error "Failed to download oc-mirror" + return 1 + fi + + # Verify checksum + if curl -fL --retry 3 --connect-timeout 30 -o "${download_dir}/sha256sum.txt" "${base_url}/sha256sum.txt" 2> /dev/null; then + if grep "oc-mirror.tar.gz" "${download_dir}/sha256sum.txt" | (cd "${download_dir}" && sha256sum -c -); then + log::info "oc-mirror checksum verified" + else + log::warn "oc-mirror checksum verification failed — continuing anyway" + fi + fi + + tar -xzf "${download_dir}/oc-mirror.tar.gz" -C "${download_dir}" + chmod +x "${download_dir}/oc-mirror" + OC_MIRROR_BIN="${download_dir}/oc-mirror" + export OC_MIRROR_BIN + + log::success "oc-mirror installed: $(${OC_MIRROR_BIN} version --output=yaml 2>&1 | head -1)" +} + +# Build an ImageSetConfiguration for oc-mirror. +# The configuration is dynamically generated based on IMAGE_REGISTRY: +# - registry.redhat.io (GA): uses helm.local with chart pulled from charts.openshift.io +# - anything else (CI/upstream): uses helm.local with chart pulled from OCI +# Args: +# $1 - output_path: Path to write the ImageSetConfiguration YAML +disconnected::build_imageset_config() { + local output_path=$1 + + # Start with the base config + cat > "${output_path}" << EOF +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + helm: + local: + - name: redhat-developer-hub + path: ${CHART_LOCAL_TGZ} +EOF + + # Add additional images that need mirroring beyond what the chart references. + # The chart's default images are discovered automatically by oc-mirror. + local additional_images=() + + # When the hub image is overridden (different from chart defaults), add it + # so oc-mirror mirrors the actual image we'll deploy with. + if [[ "${IMAGE_REGISTRY}" != "registry.redhat.io" ]]; then + # CI/upstream: chart defaults to quay.io/rhdh/rhdh-hub-rhel9@sha256:..., + # but we may deploy with a different tag (e.g., rhdh-community/rhdh:next) + additional_images+=("${IMAGE_REGISTRY}/${IMAGE_REPO}:${TAG_NAME}") + fi + + # PG image: CI charts may use quay.io/fedora/postgresql-15 instead of + # registry.redhat.io/rhel9/postgresql-15. Add it if not from registry.redhat.io. + if [[ "${PG_REGISTRY:-registry.redhat.io}" != "registry.redhat.io" ]]; then + additional_images+=("${PG_REGISTRY}/${PG_REPO}:${PG_TAG}") + fi + + if [[ ${#additional_images[@]} -gt 0 ]]; then + { + echo " additionalImages:" + for img in "${additional_images[@]}"; do + echo " - name: ${img}" + done + } >> "${output_path}" + fi + + log::info "ImageSetConfiguration written to ${output_path}" + log::debug "$(cat "${output_path}")" +} + +# Run oc-mirror to mirror images to the disconnected mirror registry. +# Sets OC_MIRROR_IDMS_FILE, OC_MIRROR_ITMS_FILE, and OC_MIRROR_CHART_PATH. +# Args: +# $1 - imageset_config: Path to the ImageSetConfiguration YAML +# $2 - workspace_dir: Path to the oc-mirror workspace directory +disconnected::run_oc_mirror() { + local imageset_config=$1 + local workspace_dir=$2 + + mkdir -p "${workspace_dir}" + + log::info "Running oc-mirror --v2 → ${MIRROR_REGISTRY_URL}" + if ! "${OC_MIRROR_BIN}" \ + -c "${imageset_config}" \ + "docker://${MIRROR_REGISTRY_URL}" \ + --dest-tls-verify=false \ + --v2 \ + --workspace "file://${workspace_dir}"; then + log::error "oc-mirror failed" + return 1 + fi + + local result_dir="${workspace_dir}/working-dir" + + # IDMS (required) + OC_MIRROR_IDMS_FILE="${result_dir}/cluster-resources/idms-oc-mirror.yaml" + if [[ ! -s "${OC_MIRROR_IDMS_FILE}" ]]; then + log::error "oc-mirror did not generate IDMS at ${OC_MIRROR_IDMS_FILE}" + return 1 + fi + export OC_MIRROR_IDMS_FILE + + # ITMS (optional) + OC_MIRROR_ITMS_FILE="${result_dir}/cluster-resources/itms-oc-mirror.yaml" + if [[ ! -s "${OC_MIRROR_ITMS_FILE}" ]]; then + OC_MIRROR_ITMS_FILE="" + fi + export OC_MIRROR_ITMS_FILE + + # Chart path (in the workspace) + OC_MIRROR_CHART_PATH=$(find "${result_dir}/helm/charts" -name '*.tgz' 2> /dev/null | head -1) + export OC_MIRROR_CHART_PATH + + log::success "oc-mirror completed successfully" + log::info "IDMS: ${OC_MIRROR_IDMS_FILE}" + [[ -n "${OC_MIRROR_ITMS_FILE}" ]] && log::info "ITMS: ${OC_MIRROR_ITMS_FILE}" + [[ -n "${OC_MIRROR_CHART_PATH}" ]] && log::info "Chart: ${OC_MIRROR_CHART_PATH}" +} + +# Patch the oc-mirror-generated IDMS to ensure both quay.io and +# registry.redhat.io sources are covered, regardless of what oc-mirror +# discovered from the chart. This is needed because: +# - GA charts reference registry.redhat.io but CI verification may override to quay.io +# - CI charts reference quay.io but post-GA verification uses registry.redhat.io +# Args: +# $1 - idms_file: Path to the IDMS YAML to patch +disconnected::patch_idms() { + local idms_file=$1 + + log::info "Patching IDMS with cross-registry mirror entries" + + # Add mirror entries for the hub image from both registries + for source_registry in "quay.io" "registry.redhat.io"; do + local source="${source_registry}/${IMAGE_REPO}" + local mirror="${MIRROR_REGISTRY_URL}/${IMAGE_REPO}" + + # Skip if this source is already in the IDMS + if yq eval ".spec.imageDigestMirrors[].source" "${idms_file}" 2> /dev/null | grep -qF "${source}"; then + log::debug "IDMS already contains entry for ${source}" + continue + fi + + yq eval -i \ + ".spec.imageDigestMirrors += [{\"mirrors\": [\"${mirror}\"], \"source\": \"${source}\"}]" \ + "${idms_file}" + log::info "Added IDMS entry: ${source} → ${mirror}" + done + + # Add mirror entry for PG image if not already present + if [[ -n "${PG_REGISTRY:-}" && -n "${PG_REPO:-}" ]]; then + local pg_source="${PG_REGISTRY}/${PG_REPO}" + local pg_mirror="${MIRROR_REGISTRY_URL}/${PG_REPO}" + + if ! yq eval ".spec.imageDigestMirrors[].source" "${idms_file}" 2> /dev/null | grep -qF "${pg_source}"; then + yq eval -i \ + ".spec.imageDigestMirrors += [{\"mirrors\": [\"${pg_mirror}\"], \"source\": \"${pg_source}\"}]" \ + "${idms_file}" + log::info "Added IDMS entry: ${pg_source} → ${pg_mirror}" + fi + fi + + log::debug "Patched IDMS:" + log::debug "$(cat "${idms_file}")" +} + +# Fetch an external script from the rhdh-operator repository. +# Args: +# $1 - script_name: Name of the script (e.g., "mirror-plugins.sh") +# $2 - output_path: Local path to save the script +# $3 - branch: (optional) Branch name (defaults to $RELEASE_BRANCH_NAME) +disconnected::fetch_script() { + local script_name=$1 + local output_path=$2 + local branch="${3:-${RELEASE_BRANCH_NAME}}" + + local url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/refs/heads/${branch}/.rhdh/scripts/${script_name}" + + log::info "Fetching ${script_name} from rhdh-operator (branch: ${branch})..." + if ! curl -fL --max-time 30 -o "${output_path}" "${url}"; then + log::error "Failed to download ${script_name} from ${url}" + return 1 + fi + chmod +x "${output_path}" + log::success "Downloaded ${script_name} to ${output_path}" +} + +# Export functions for subshell usage (e.g., timeout bash -c "...") +export -f disconnected::require_env +export -f disconnected::setup_auth +export -f disconnected::fetch_script diff --git a/.ci/pipelines/openshift-ci-tests.sh b/.ci/pipelines/openshift-ci-tests.sh index 55b3679cbc..f6564e0b5c 100755 --- a/.ci/pipelines/openshift-ci-tests.sh +++ b/.ci/pipelines/openshift-ci-tests.sh @@ -128,6 +128,20 @@ main() { log::info "Calling handle_ocp_localization" handle_ocp_localization ;; + *ocp*disconnected*helm*nightly*) + log::info "Sourcing ocp-disconnected-helm.sh" + # shellcheck source=.ci/pipelines/jobs/ocp-disconnected-helm.sh + source "${DIR}/jobs/ocp-disconnected-helm.sh" + log::info "Calling handle_ocp_disconnected_helm" + handle_ocp_disconnected_helm + ;; + *ocp*disconnected*operator*nightly*) + log::info "Sourcing ocp-disconnected-operator.sh" + # shellcheck source=.ci/pipelines/jobs/ocp-disconnected-operator.sh + source "${DIR}/jobs/ocp-disconnected-operator.sh" + log::info "Calling handle_ocp_disconnected_operator" + handle_ocp_disconnected_operator + ;; *ocp*helm*nightly*) log::info "Sourcing ocp-nightly.sh" # shellcheck source=.ci/pipelines/jobs/ocp-nightly.sh diff --git a/.ci/pipelines/resources/disconnected/helm-overrides.yaml b/.ci/pipelines/resources/disconnected/helm-overrides.yaml new file mode 100644 index 0000000000..83b1fd197c --- /dev/null +++ b/.ci/pipelines/resources/disconnected/helm-overrides.yaml @@ -0,0 +1,52 @@ +# Helm values override for disconnected deployments. +# Adds the registries.conf ConfigMap volume + mount to the +# install-dynamic-plugins init container. +# +# Using a separate -f file avoids the Helm "array clobber" pitfall: +# --set on initContainers[]/extraVolumes[] replaces the entire array, +# losing the chart defaults. The -f merge keeps chart-default volumes +# intact and only adds the ConfigMap volume + mount. +upstream: + backstage: + extraVolumes: + - name: dynamic-plugins-root + emptyDir: {} + - name: dynamic-plugins + configMap: + defaultMode: 420 + name: dynamic-plugins + optional: true + - name: dynamic-plugins-npmrc + secret: + defaultMode: 420 + optional: true + secretName: dynamic-plugins-npmrc + - name: rhdh-plugin-mirror-conf + configMap: + name: rhdh-plugin-mirror-conf + initContainers: + - name: install-dynamic-plugins + command: + - ./install-dynamic-plugins.sh + - /dynamic-plugins-root + env: + - name: NPM_CONFIG_USERCONFIG + value: /opt/app-root/src/.npmrc.dynamic-plugins + image: null + imagePullPolicy: Always + volumeMounts: + - mountPath: /dynamic-plugins-root + name: dynamic-plugins-root + - mountPath: /opt/app-root/src/dynamic-plugins.yaml + name: dynamic-plugins + readOnly: true + subPath: dynamic-plugins.yaml + - mountPath: /opt/app-root/src/.npmrc.dynamic-plugins + name: dynamic-plugins-npmrc + readOnly: true + subPath: .npmrc + - mountPath: /etc/containers/registries.conf.d/rhdh-registries.conf + name: rhdh-plugin-mirror-conf + readOnly: true + subPath: rhdh-registries.conf + workingDir: /opt/app-root/src diff --git a/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml b/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml new file mode 100644 index 0000000000..2e14f555c9 --- /dev/null +++ b/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rhdh-plugin-mirror-conf +data: + rhdh-registries.conf: | + [[registry]] + prefix = "registry.access.redhat.com/rhdh" + location = "${MIRROR_REGISTRY_URL}/rhdh" + + [[registry]] + prefix = "quay.io/rhdh" + location = "${MIRROR_REGISTRY_URL}/rhdh" + + [[registry]] + prefix = "ghcr.io/redhat-developer/rhdh-plugin-export-overlays" + location = "${MIRROR_REGISTRY_URL}/rhdh-plugin-export-overlays" From eacf38712def9077c21b32bbfea74d26a6dcdeab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 29 Jun 2026 09:50:38 +0200 Subject: [PATCH 02/15] fix(ci): work around rhdh-operator yq install bug in disconnected operator handler Use the fixed prepare-restricted-environment.sh from redhat-developer/rhdh-operator#3109 until it merges. The upstream script has a bug where INSTALL_YQ=0 still triggers the yq install path because [[ 0 ]] is truthy in bash, causing a 'mv: cannot move yq_linux_amd64 to /tmp/.local/bin/yq_mf: No such file or directory' failure. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-operator.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 0c6093a63b..bad402e144 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -33,11 +33,16 @@ handle_ocp_disconnected_operator() { # mirroring operator/operand images and installing the operator CatalogSource. log::section "Operator Mirroring and Installation" - disconnected::fetch_script "prepare-restricted-environment.sh" "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" \ - || { - log::error "Failed to fetch prepare-restricted-environment.sh — aborting" - return 1 - } + # TODO: revert to disconnected::fetch_script once redhat-developer/rhdh-operator#3109 merges. + # The upstream script has a bug where INSTALL_YQ=0 triggers the yq install + # path because [[ 0 ]] is truthy in bash. Use the fixed version from the PR. + local _prepare_url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/refs/pull/3109/head/.rhdh/scripts/prepare-restricted-environment.sh" + log::info "Fetching prepare-restricted-environment.sh (fixed: rhdh-operator#3109)..." + if ! curl -fL --max-time 30 -o "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" "${_prepare_url}"; then + log::error "Failed to download prepare-restricted-environment.sh — aborting" + return 1 + fi + chmod +x "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" local prepare_args=( --to-registry "${MIRROR_REGISTRY_URL}" From fbad1822e35f9b179650c2f8baa83cb8f4104c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 29 Jun 2026 10:04:00 +0200 Subject: [PATCH 03/15] fix(ci): unset REGISTRY_AUTH_FILE for oc-mirror and save artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit oc-mirror panics when REGISTRY_AUTH_FILE is set because the distribution/distribution library interprets it as a storage driver config, not a container auth file path. Unset it before running oc-mirror and restore after — oc-mirror reads auth from ${XDG_RUNTIME_DIR}/containers/auth.json instead. Save key artifacts to ARTIFACT_DIR with disconnected- prefix for post-failure debugging: ImageSetConfiguration, generated IDMS/ITMS, patched IDMS, chart values, rendered ConfigMap, helm set flags, and Backstage CR. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 7 ++++++ .../jobs/ocp-disconnected-operator.sh | 5 +++++ .ci/pipelines/lib/disconnected.sh | 22 +++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 4925e410d1..4ff19d8d75 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -86,6 +86,8 @@ handle_ocp_disconnected_helm() { log::info "PostgreSQL image from chart: ${PG_REGISTRY}/${PG_REPO}:${PG_TAG}" + echo "${helm_values}" > "${ARTIFACT_DIR}/disconnected-helm-chart-values.yaml" 2> /dev/null || true + # --- Section D: Build ImageSetConfiguration --- log::section "Image Mirroring" @@ -151,6 +153,9 @@ handle_ocp_disconnected_helm() { } log::success "ConfigMap rhdh-plugin-mirror-conf created in ${NAME_SPACE}" + envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ + > "${ARTIFACT_DIR}/disconnected-plugin-mirror-configmap.yaml" 2> /dev/null || true + # --- Section I: Helm deployment from mirrored chart --- log::section "Helm Deployment" @@ -186,6 +191,8 @@ handle_ocp_disconnected_helm() { log::success "RHDH deployed via Helm with mirrored images" + printf '%s\n' "${helm_set_flags[@]}" > "${ARTIFACT_DIR}/disconnected-helm-set-flags.txt" 2> /dev/null || true + # --- Section J: Smoke test --- log::section "Smoke Test" diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index bad402e144..ecd88fea22 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -103,6 +103,9 @@ handle_ocp_disconnected_operator() { } log::success "ConfigMap rhdh-plugin-mirror-conf created in ${NAME_SPACE}" + envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ + > "${ARTIFACT_DIR}/disconnected-plugin-mirror-configmap.yaml" 2> /dev/null || true + # --- Section E: Backstage CR Deployment --- log::section "Backstage CR Deployment" @@ -121,6 +124,8 @@ handle_ocp_disconnected_operator() { local cr_temp="${DISCONNECTED_TMPDIR}/backstage-cr-disconnected.yaml" echo "$rendered_cr" > "${cr_temp}" + cp "${cr_temp}" "${ARTIFACT_DIR}/disconnected-backstage-cr.yaml" 2> /dev/null || true + deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" log::success "Backstage CR deployed in ${NAME_SPACE}" diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index 619a54c85d..e95a829b4c 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -163,6 +163,8 @@ EOF log::info "ImageSetConfiguration written to ${output_path}" log::debug "$(cat "${output_path}")" + + cp "${output_path}" "${ARTIFACT_DIR}/disconnected-imageset-config.yaml" 2> /dev/null || true } # Run oc-mirror to mirror images to the disconnected mirror registry. @@ -176,6 +178,13 @@ disconnected::run_oc_mirror() { mkdir -p "${workspace_dir}" + # oc-mirror panics when REGISTRY_AUTH_FILE is set because the + # distribution/distribution library interprets it as a storage driver + # config, not an auth file path. oc-mirror reads auth exclusively from + # ${XDG_RUNTIME_DIR}/containers/auth.json (set up by setup_auth). + local saved_registry_auth_file="${REGISTRY_AUTH_FILE:-}" + unset REGISTRY_AUTH_FILE + log::info "Running oc-mirror --v2 → ${MIRROR_REGISTRY_URL}" if ! "${OC_MIRROR_BIN}" \ -c "${imageset_config}" \ @@ -183,10 +192,17 @@ disconnected::run_oc_mirror() { --dest-tls-verify=false \ --v2 \ --workspace "file://${workspace_dir}"; then + # Restore before returning + [[ -n "${saved_registry_auth_file}" ]] && export REGISTRY_AUTH_FILE="${saved_registry_auth_file}" log::error "oc-mirror failed" return 1 fi + # Restore REGISTRY_AUTH_FILE for subsequent skopeo/mirror-plugins.sh calls + if [[ -n "${saved_registry_auth_file}" ]]; then + export REGISTRY_AUTH_FILE="${saved_registry_auth_file}" + fi + local result_dir="${workspace_dir}/working-dir" # IDMS (required) @@ -212,6 +228,10 @@ disconnected::run_oc_mirror() { log::info "IDMS: ${OC_MIRROR_IDMS_FILE}" [[ -n "${OC_MIRROR_ITMS_FILE}" ]] && log::info "ITMS: ${OC_MIRROR_ITMS_FILE}" [[ -n "${OC_MIRROR_CHART_PATH}" ]] && log::info "Chart: ${OC_MIRROR_CHART_PATH}" + + # Save artifacts for debugging + cp "${OC_MIRROR_IDMS_FILE}" "${ARTIFACT_DIR}/disconnected-idms-generated.yaml" 2> /dev/null || true + [[ -n "${OC_MIRROR_ITMS_FILE}" ]] && cp "${OC_MIRROR_ITMS_FILE}" "${ARTIFACT_DIR}/disconnected-itms-generated.yaml" 2> /dev/null || true } # Patch the oc-mirror-generated IDMS to ensure both quay.io and @@ -258,6 +278,8 @@ disconnected::patch_idms() { log::debug "Patched IDMS:" log::debug "$(cat "${idms_file}")" + + cp "${idms_file}" "${ARTIFACT_DIR}/disconnected-idms-patched.yaml" 2> /dev/null || true } # Fetch an external script from the rhdh-operator repository. From a23a11e11ddff1c46e8ed3848be635f51213633c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 29 Jun 2026 12:18:30 +0200 Subject: [PATCH 04/15] fix(ci): sanitize chart repo paths for IDMS and install podman for operator Strip @sha256 suffixes from chart-extracted repository paths before constructing IDMS entries. The RHDH Helm chart encodes digest references as repository: "repo@sha256" + tag: "", but IDMS source/mirror fields require clean registry/repo paths without @. Uses ${var%@*} which is a no-op on paths without @. Install podman at runtime for the operator disconnected handler. prepare-restricted-environment.sh requires podman for building custom operator index images. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 14 ++++++++++++-- .ci/pipelines/jobs/ocp-disconnected-operator.sh | 7 +++++++ .ci/pipelines/lib/disconnected.sh | 6 ++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 4ff19d8d75..0108cf7b8d 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -76,7 +76,7 @@ handle_ocp_disconnected_helm() { local helm_values helm_values=$(helm show values "${CHART_LOCAL_TGZ}" 2> /dev/null || true) - export PG_REGISTRY PG_REPO PG_TAG + export PG_REGISTRY PG_REPO PG_TAG PG_SEPARATOR PG_REGISTRY=$(echo "${helm_values}" | yq '.upstream.postgresql.image.registry' || true) PG_REPO=$(echo "${helm_values}" | yq '.upstream.postgresql.image.repository' || true) PG_TAG=$(echo "${helm_values}" | yq '.upstream.postgresql.image.tag' || true) @@ -84,7 +84,17 @@ handle_ocp_disconnected_helm() { PG_REPO="${PG_REPO:-rhel9/postgresql-15}" PG_TAG="${PG_TAG:-latest}" - log::info "PostgreSQL image from chart: ${PG_REGISTRY}/${PG_REPO}:${PG_TAG}" + # The chart encodes digest refs as repository: "repo@sha256" + tag: "". + # Normalize: extract the digest qualifier into PG_SEPARATOR so that: + # - PG_REPO is always a clean path (usable in IDMS source/mirror fields) + # - Full ref is ${PG_REGISTRY}/${PG_REPO}${PG_SEPARATOR}${PG_TAG} + PG_SEPARATOR=":" + if [[ "${PG_REPO}" == *"@"* ]]; then + PG_SEPARATOR="@${PG_REPO##*@}:" # e.g., "@sha256:" + PG_REPO="${PG_REPO%@*}" # e.g., "rhel9/postgresql-15" + fi + + log::info "PostgreSQL image from chart: ${PG_REGISTRY}/${PG_REPO}${PG_SEPARATOR}${PG_TAG}" echo "${helm_values}" > "${ARTIFACT_DIR}/disconnected-helm-chart-values.yaml" 2> /dev/null || true diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index ecd88fea22..a1e3b9051e 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -31,8 +31,15 @@ handle_ocp_disconnected_operator() { # --- Section A: Operator Mirroring + Installation --- # Uses prepare-restricted-environment.sh from rhdh-operator, which handles # mirroring operator/operand images and installing the operator CatalogSource. + # Requires podman for building custom operator index images. log::section "Operator Mirroring and Installation" + if ! command -v podman &> /dev/null; then + log::info "Installing podman (required by prepare-restricted-environment.sh)..." + apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq podman > /dev/null 2>&1 + log::success "podman installed: $(podman --version)" + fi + # TODO: revert to disconnected::fetch_script once redhat-developer/rhdh-operator#3109 merges. # The upstream script has a bug where INSTALL_YQ=0 triggers the yq install # path because [[ 0 ]] is truthy in bash. Use the fixed version from the PR. diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index e95a829b4c..ef9f09a3a7 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -148,8 +148,9 @@ EOF # PG image: CI charts may use quay.io/fedora/postgresql-15 instead of # registry.redhat.io/rhel9/postgresql-15. Add it if not from registry.redhat.io. + # PG_SEPARATOR accounts for digest (@sha256:) vs tag (:) encoding. if [[ "${PG_REGISTRY:-registry.redhat.io}" != "registry.redhat.io" ]]; then - additional_images+=("${PG_REGISTRY}/${PG_REPO}:${PG_TAG}") + additional_images+=("${PG_REGISTRY}/${PG_REPO}${PG_SEPARATOR}${PG_TAG}") fi if [[ ${#additional_images[@]} -gt 0 ]]; then @@ -263,7 +264,8 @@ disconnected::patch_idms() { log::info "Added IDMS entry: ${source} → ${mirror}" done - # Add mirror entry for PG image if not already present + # Add mirror entry for PG image if not already present. + # PG_REPO is already cleaned of @sha256 by the caller (via PG_SEPARATOR). if [[ -n "${PG_REGISTRY:-}" && -n "${PG_REPO:-}" ]]; then local pg_source="${PG_REGISTRY}/${PG_REPO}" local pg_mirror="${MIRROR_REGISTRY_URL}/${PG_REPO}" From a2feaaf1e555f9a5471c02ca9bb8aabdd65af67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 29 Jun 2026 15:16:47 +0200 Subject: [PATCH 05/15] fix(ci): fix helm init container image, podman install, and revert yq workaround MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Helm: replace image: null in helm-overrides.yaml with an envsubst placeholder (${MIRROR_REGISTRY_URL}/${IMAGE_REPO}:${TAG_NAME}). The chart validates initContainers[0].image as a string, rejecting null. The overrides file is now rendered through envsubst before passing to helm upgrade -i. Operator: make podman install visible (don't suppress output) and verify it succeeds before continuing. If apt-get fails (e.g. repos unreachable via proxy), abort with a clear message instead of silently proceeding. Revert the rhdh-operator#3109 PR ref workaround — the yq install bug has been fixed upstream. Use disconnected::fetch_script again. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 7 +++++- .../jobs/ocp-disconnected-operator.sh | 23 ++++++++++--------- .../disconnected/helm-overrides.yaml | 2 +- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 0108cf7b8d..7d79b7c394 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -190,10 +190,15 @@ handle_ocp_disconnected_helm() { ) fi + # Render the overrides with envsubst (resolves image refs for init container) + local rendered_overrides="${DISCONNECTED_TMPDIR}/helm-overrides-rendered.yaml" + envsubst < "${DIR}/resources/disconnected/helm-overrides.yaml" > "${rendered_overrides}" + cp "${rendered_overrides}" "${ARTIFACT_DIR}/disconnected-helm-overrides-rendered.yaml" 2> /dev/null || true + helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \ "${chart_install_path}" \ -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" \ - -f "${DIR}/resources/disconnected/helm-overrides.yaml" \ + -f "${rendered_overrides}" \ "${helm_set_flags[@]}" || { log::error "Helm deployment failed" return 1 diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index a1e3b9051e..aa51779977 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -36,20 +36,21 @@ handle_ocp_disconnected_operator() { if ! command -v podman &> /dev/null; then log::info "Installing podman (required by prepare-restricted-environment.sh)..." - apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq podman > /dev/null 2>&1 + apt-get update -qq 2>&1 + apt-get install -y -qq podman 2>&1 + hash -r + if ! command -v podman &> /dev/null; then + log::error "Failed to install podman. Add podman to the rhdh-e2e-runner Dockerfile." + return 1 + fi log::success "podman installed: $(podman --version)" fi - # TODO: revert to disconnected::fetch_script once redhat-developer/rhdh-operator#3109 merges. - # The upstream script has a bug where INSTALL_YQ=0 triggers the yq install - # path because [[ 0 ]] is truthy in bash. Use the fixed version from the PR. - local _prepare_url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/refs/pull/3109/head/.rhdh/scripts/prepare-restricted-environment.sh" - log::info "Fetching prepare-restricted-environment.sh (fixed: rhdh-operator#3109)..." - if ! curl -fL --max-time 30 -o "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" "${_prepare_url}"; then - log::error "Failed to download prepare-restricted-environment.sh — aborting" - return 1 - fi - chmod +x "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" + disconnected::fetch_script "prepare-restricted-environment.sh" "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" \ + || { + log::error "Failed to fetch prepare-restricted-environment.sh — aborting" + return 1 + } local prepare_args=( --to-registry "${MIRROR_REGISTRY_URL}" diff --git a/.ci/pipelines/resources/disconnected/helm-overrides.yaml b/.ci/pipelines/resources/disconnected/helm-overrides.yaml index 83b1fd197c..cff992745c 100644 --- a/.ci/pipelines/resources/disconnected/helm-overrides.yaml +++ b/.ci/pipelines/resources/disconnected/helm-overrides.yaml @@ -32,7 +32,7 @@ upstream: env: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins - image: null + image: "${MIRROR_REGISTRY_URL}/${IMAGE_REPO}:${TAG_NAME}" imagePullPolicy: Always volumeMounts: - mountPath: /dynamic-plugins-root From 233738183f08d7ba7d6c103877f7b40158f02774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 30 Jun 2026 10:26:55 +0200 Subject: [PATCH 06/15] fix(disconnected): replace static helm-overrides with post-renderer Replace the static helm-overrides.yaml values file with a Helm post-renderer script that appends the registries.conf volume and mount to the already-rendered Deployment manifests. This avoids the Helm 'array clobber' pitfall: a values file that defines extraVolumes[] or initContainers[] replaces the chart's entire default array, losing volumes added by newer chart versions. The post-renderer patches the final manifests so the chart's defaults are always preserved regardless of chart version changes. Also improves operator podman install error visibility by removing -qq flags and adding proper error checking for apt-get commands. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 11 ++-- .../jobs/ocp-disconnected-operator.sh | 10 ++-- .../disconnected/helm-overrides.yaml | 52 ------------------- .../disconnected/helm-post-renderer.sh | 22 ++++++++ 4 files changed, 35 insertions(+), 60 deletions(-) delete mode 100644 .ci/pipelines/resources/disconnected/helm-overrides.yaml create mode 100755 .ci/pipelines/resources/disconnected/helm-post-renderer.sh diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 7d79b7c394..1601a59fcf 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -190,15 +190,16 @@ handle_ocp_disconnected_helm() { ) fi - # Render the overrides with envsubst (resolves image refs for init container) - local rendered_overrides="${DISCONNECTED_TMPDIR}/helm-overrides-rendered.yaml" - envsubst < "${DIR}/resources/disconnected/helm-overrides.yaml" > "${rendered_overrides}" - cp "${rendered_overrides}" "${ARTIFACT_DIR}/disconnected-helm-overrides-rendered.yaml" 2> /dev/null || true + # Post-renderer appends registries.conf volume + mount to the rendered + # Deployment, avoiding the Helm "array clobber" pitfall where a values + # file that defines extraVolumes[] replaces the chart's entire default + # array (which grows across chart versions). + local post_renderer="${DIR}/resources/disconnected/helm-post-renderer.sh" helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \ "${chart_install_path}" \ -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" \ - -f "${rendered_overrides}" \ + --post-renderer "${post_renderer}" \ "${helm_set_flags[@]}" || { log::error "Helm deployment failed" return 1 diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index aa51779977..f41937ce7f 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -36,11 +36,15 @@ handle_ocp_disconnected_operator() { if ! command -v podman &> /dev/null; then log::info "Installing podman (required by prepare-restricted-environment.sh)..." - apt-get update -qq 2>&1 - apt-get install -y -qq podman 2>&1 + if ! apt-get update 2>&1; then + log::warn "apt-get update failed — continuing anyway" + fi + if ! apt-get install -y podman 2>&1; then + log::error "apt-get install podman failed" + fi hash -r if ! command -v podman &> /dev/null; then - log::error "Failed to install podman. Add podman to the rhdh-e2e-runner Dockerfile." + log::error "podman not available after install. Add podman to the rhdh-e2e-runner Dockerfile." return 1 fi log::success "podman installed: $(podman --version)" diff --git a/.ci/pipelines/resources/disconnected/helm-overrides.yaml b/.ci/pipelines/resources/disconnected/helm-overrides.yaml deleted file mode 100644 index cff992745c..0000000000 --- a/.ci/pipelines/resources/disconnected/helm-overrides.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Helm values override for disconnected deployments. -# Adds the registries.conf ConfigMap volume + mount to the -# install-dynamic-plugins init container. -# -# Using a separate -f file avoids the Helm "array clobber" pitfall: -# --set on initContainers[]/extraVolumes[] replaces the entire array, -# losing the chart defaults. The -f merge keeps chart-default volumes -# intact and only adds the ConfigMap volume + mount. -upstream: - backstage: - extraVolumes: - - name: dynamic-plugins-root - emptyDir: {} - - name: dynamic-plugins - configMap: - defaultMode: 420 - name: dynamic-plugins - optional: true - - name: dynamic-plugins-npmrc - secret: - defaultMode: 420 - optional: true - secretName: dynamic-plugins-npmrc - - name: rhdh-plugin-mirror-conf - configMap: - name: rhdh-plugin-mirror-conf - initContainers: - - name: install-dynamic-plugins - command: - - ./install-dynamic-plugins.sh - - /dynamic-plugins-root - env: - - name: NPM_CONFIG_USERCONFIG - value: /opt/app-root/src/.npmrc.dynamic-plugins - image: "${MIRROR_REGISTRY_URL}/${IMAGE_REPO}:${TAG_NAME}" - imagePullPolicy: Always - volumeMounts: - - mountPath: /dynamic-plugins-root - name: dynamic-plugins-root - - mountPath: /opt/app-root/src/dynamic-plugins.yaml - name: dynamic-plugins - readOnly: true - subPath: dynamic-plugins.yaml - - mountPath: /opt/app-root/src/.npmrc.dynamic-plugins - name: dynamic-plugins-npmrc - readOnly: true - subPath: .npmrc - - mountPath: /etc/containers/registries.conf.d/rhdh-registries.conf - name: rhdh-plugin-mirror-conf - readOnly: true - subPath: rhdh-registries.conf - workingDir: /opt/app-root/src diff --git a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh new file mode 100755 index 0000000000..79b02c8f83 --- /dev/null +++ b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Helm post-renderer for disconnected deployments. +# +# Appends the registries.conf ConfigMap volume and volumeMount to the +# RHDH Deployment's pod spec and install-dynamic-plugins init container. +# +# Using a post-renderer avoids the Helm "array clobber" pitfall: +# a values file that defines extraVolumes[] or initContainers[] replaces +# the chart's entire default array, losing any volumes added by newer +# chart versions. A post-renderer patches the already-rendered manifests +# so the chart's defaults are always preserved. +# +# Usage: helm upgrade -i ... --post-renderer ./helm-post-renderer.sh + +set -euo pipefail + +yq eval ' + (select(.kind == "Deployment" and .metadata.name == "*-developer-hub") | + .spec.template.spec.volumes += [{"name": "rhdh-plugin-mirror-conf", "configMap": {"name": "rhdh-plugin-mirror-conf"}}] | + .spec.template.spec.initContainers[0].volumeMounts += [{"mountPath": "/etc/containers/registries.conf.d/rhdh-registries.conf", "name": "rhdh-plugin-mirror-conf", "readOnly": true, "subPath": "rhdh-registries.conf"}] + ) // . +' From 43cf4f8ff30ef9a285bdeaeefb170c4a790df09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 30 Jun 2026 13:48:06 +0200 Subject: [PATCH 07/15] fix(disconnected): use minimal values for helm smoke test The full values_showcase.yaml references ConfigMaps (app-config-rhdh, dynamic-plugins-config) and secrets (rhdh-secrets) that are created by the normal apply_yaml_files() flow, which the disconnected handler skips. This caused pods to be stuck in Init:0/2 waiting for missing ConfigMaps. Add a minimal values file that provides only what the smoke test needs: guest auth with dangerouslyAllowOutsideDevelopment. The chart defaults handle everything else (internal PostgreSQL, backend secret, default dynamic plugins, route). Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 2 +- .../value_files/values_disconnected-smoke.yaml | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .ci/pipelines/value_files/values_disconnected-smoke.yaml diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 1601a59fcf..715642c0c9 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -198,7 +198,7 @@ handle_ocp_disconnected_helm() { helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \ "${chart_install_path}" \ - -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" \ + -f "${DIR}/value_files/values_disconnected-smoke.yaml" \ --post-renderer "${post_renderer}" \ "${helm_set_flags[@]}" || { log::error "Helm deployment failed" diff --git a/.ci/pipelines/value_files/values_disconnected-smoke.yaml b/.ci/pipelines/value_files/values_disconnected-smoke.yaml new file mode 100644 index 0000000000..1a3d5dac9a --- /dev/null +++ b/.ci/pipelines/value_files/values_disconnected-smoke.yaml @@ -0,0 +1,13 @@ +# Minimal Helm values for disconnected smoke test. +# The smoke test only verifies: login page renders → guest login → homepage. +# No GitHub integrations, external secrets, dynamic plugins, or OIDC needed. +upstream: + backstage: + image: + pullPolicy: Always + appConfig: + auth: + environment: development + providers: + guest: + dangerouslyAllowOutsideDevelopment: true From d4d526dcdf5ccd88ec2a06c6dd1cde21a9bfaf74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 30 Jun 2026 15:25:36 +0200 Subject: [PATCH 08/15] fix(disconnected): mount mirror registry CA in init container The install-dynamic-plugins init container runs skopeo to fetch the plugin catalog index. IDMS redirects quay.io pulls to the mirror registry, but skopeo inside the pod doesn't trust the mirror's self-signed CA certificate (x509: certificate signed by unknown authority). Create a ConfigMap containing the system CA bundle concatenated with the mirror registry CA, and mount it in the init container at /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem (Go's default system cert store path on RHEL/UBI). Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 14 ++++++++++++++ .../disconnected/helm-post-renderer.sh | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 715642c0c9..d6c79f6bbe 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -166,6 +166,20 @@ handle_ocp_disconnected_helm() { envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ > "${ARTIFACT_DIR}/disconnected-plugin-mirror-configmap.yaml" 2> /dev/null || true + # Combined CA bundle: system CAs + mirror registry CA. + # The init container's skopeo needs to trust the mirror registry when + # IDMS redirects quay.io/registry.redhat.io pulls to the mirror. + local combined_ca="${DISCONNECTED_TMPDIR}/combined-ca-bundle.crt" + cat /etc/pki/tls/certs/ca-bundle.crt "${MIRROR_REGISTRY_CA}" > "${combined_ca}" + oc create configmap mirror-registry-ca \ + --from-file="tls-ca-bundle.pem=${combined_ca}" \ + -n "${NAME_SPACE}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create mirror-registry-ca ConfigMap — aborting" + return 1 + } + log::success "ConfigMap mirror-registry-ca created in ${NAME_SPACE}" + # --- Section I: Helm deployment from mirrored chart --- log::section "Helm Deployment" diff --git a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh index 79b02c8f83..406c043ced 100755 --- a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh +++ b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh @@ -1,8 +1,11 @@ #!/bin/bash # Helm post-renderer for disconnected deployments. # -# Appends the registries.conf ConfigMap volume and volumeMount to the -# RHDH Deployment's pod spec and install-dynamic-plugins init container. +# Patches the RHDH Deployment's pod spec for disconnected environments: +# 1. Mounts registries.conf so the init container resolves plugin images +# from the mirror registry instead of the original registries. +# 2. Mounts a combined CA bundle (system CAs + mirror registry CA) so +# skopeo can verify the mirror registry's TLS certificate. # # Using a post-renderer avoids the Helm "array clobber" pitfall: # a values file that defines extraVolumes[] or initContainers[] replaces @@ -16,7 +19,13 @@ set -euo pipefail yq eval ' (select(.kind == "Deployment" and .metadata.name == "*-developer-hub") | - .spec.template.spec.volumes += [{"name": "rhdh-plugin-mirror-conf", "configMap": {"name": "rhdh-plugin-mirror-conf"}}] | - .spec.template.spec.initContainers[0].volumeMounts += [{"mountPath": "/etc/containers/registries.conf.d/rhdh-registries.conf", "name": "rhdh-plugin-mirror-conf", "readOnly": true, "subPath": "rhdh-registries.conf"}] + .spec.template.spec.volumes += [ + {"name": "rhdh-plugin-mirror-conf", "configMap": {"name": "rhdh-plugin-mirror-conf"}}, + {"name": "mirror-registry-ca", "configMap": {"name": "mirror-registry-ca"}} + ] | + .spec.template.spec.initContainers[0].volumeMounts += [ + {"mountPath": "/etc/containers/registries.conf.d/rhdh-registries.conf", "name": "rhdh-plugin-mirror-conf", "readOnly": true, "subPath": "rhdh-registries.conf"}, + {"mountPath": "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "name": "mirror-registry-ca", "readOnly": true, "subPath": "tls-ca-bundle.pem"} + ] ) // . ' From 1a2bd20fb1069ce03f8d5319a257bd4ecb7c88d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Wed, 1 Jul 2026 09:02:26 +0200 Subject: [PATCH 09/15] fix(disconnected): use per-registry CA path for mirror TLS trust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the standard container-tools per-registry CA mechanism instead of replacing the system trust store. Mount just the mirror registry CA at /etc/containers/certs.d//ca.crt — skopeo reads this path natively, no system CA bundle concatenation needed. The mirror registry URL is passed via --post-renderer-args so the mount path is constructed dynamically per CI run. Fixes: cat /etc/pki/tls/certs/ca-bundle.crt failing on the Ubuntu-based CI runner (Debian uses /etc/ssl/certs/). Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 12 ++++----- .../disconnected/helm-post-renderer.sh | 26 ++++++++++++------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index d6c79f6bbe..47955cd81c 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -166,13 +166,12 @@ handle_ocp_disconnected_helm() { envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ > "${ARTIFACT_DIR}/disconnected-plugin-mirror-configmap.yaml" 2> /dev/null || true - # Combined CA bundle: system CAs + mirror registry CA. - # The init container's skopeo needs to trust the mirror registry when - # IDMS redirects quay.io/registry.redhat.io pulls to the mirror. - local combined_ca="${DISCONNECTED_TMPDIR}/combined-ca-bundle.crt" - cat /etc/pki/tls/certs/ca-bundle.crt "${MIRROR_REGISTRY_CA}" > "${combined_ca}" + # Mirror registry CA — mounted at /etc/containers/certs.d//ca.crt + # inside the init container so skopeo trusts the mirror when IDMS redirects + # quay.io/registry.redhat.io pulls. Uses the standard container-tools + # per-registry CA mechanism; no system trust store replacement needed. oc create configmap mirror-registry-ca \ - --from-file="tls-ca-bundle.pem=${combined_ca}" \ + --from-file="ca.crt=${MIRROR_REGISTRY_CA}" \ -n "${NAME_SPACE}" \ --dry-run=client -o yaml | oc apply -f - || { log::error "Failed to create mirror-registry-ca ConfigMap — aborting" @@ -214,6 +213,7 @@ handle_ocp_disconnected_helm() { "${chart_install_path}" \ -f "${DIR}/value_files/values_disconnected-smoke.yaml" \ --post-renderer "${post_renderer}" \ + --post-renderer-args "${MIRROR_REGISTRY_URL}" \ "${helm_set_flags[@]}" || { log::error "Helm deployment failed" return 1 diff --git a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh index 406c043ced..68a6bd6de5 100755 --- a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh +++ b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh @@ -4,8 +4,9 @@ # Patches the RHDH Deployment's pod spec for disconnected environments: # 1. Mounts registries.conf so the init container resolves plugin images # from the mirror registry instead of the original registries. -# 2. Mounts a combined CA bundle (system CAs + mirror registry CA) so -# skopeo can verify the mirror registry's TLS certificate. +# 2. Mounts the mirror registry CA at the standard container-tools +# per-registry path (/etc/containers/certs.d//ca.crt) +# so skopeo trusts the mirror's TLS certificate. # # Using a post-renderer avoids the Helm "array clobber" pitfall: # a values file that defines extraVolumes[] or initContainers[] replaces @@ -13,19 +14,24 @@ # chart versions. A post-renderer patches the already-rendered manifests # so the chart's defaults are always preserved. # -# Usage: helm upgrade -i ... --post-renderer ./helm-post-renderer.sh +# Usage: +# helm upgrade -i ... \ +# --post-renderer ./helm-post-renderer.sh \ +# --post-renderer-args set -euo pipefail -yq eval ' - (select(.kind == "Deployment" and .metadata.name == "*-developer-hub") | +MIRROR_REGISTRY_URL="${1:?Usage: helm-post-renderer.sh }" + +yq eval " + (select(.kind == \"Deployment\" and .metadata.name == \"*-developer-hub\") | .spec.template.spec.volumes += [ - {"name": "rhdh-plugin-mirror-conf", "configMap": {"name": "rhdh-plugin-mirror-conf"}}, - {"name": "mirror-registry-ca", "configMap": {"name": "mirror-registry-ca"}} + {\"name\": \"rhdh-plugin-mirror-conf\", \"configMap\": {\"name\": \"rhdh-plugin-mirror-conf\"}}, + {\"name\": \"mirror-registry-ca\", \"configMap\": {\"name\": \"mirror-registry-ca\"}} ] | .spec.template.spec.initContainers[0].volumeMounts += [ - {"mountPath": "/etc/containers/registries.conf.d/rhdh-registries.conf", "name": "rhdh-plugin-mirror-conf", "readOnly": true, "subPath": "rhdh-registries.conf"}, - {"mountPath": "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "name": "mirror-registry-ca", "readOnly": true, "subPath": "tls-ca-bundle.pem"} + {\"mountPath\": \"/etc/containers/registries.conf.d/rhdh-registries.conf\", \"name\": \"rhdh-plugin-mirror-conf\", \"readOnly\": true, \"subPath\": \"rhdh-registries.conf\"}, + {\"mountPath\": \"/etc/containers/certs.d/${MIRROR_REGISTRY_URL}/ca.crt\", \"name\": \"mirror-registry-ca\", \"readOnly\": true, \"subPath\": \"ca.crt\"} ] ) // . -' +" From c80f94907fa02daf04f72bac25f170b628fa9cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Wed, 1 Jul 2026 22:12:46 +0200 Subject: [PATCH 10/15] fix(disconnected): provide registry auth for init container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install-dynamic-plugins init container uses skopeo to fetch the plugin catalog index. With IDMS redirecting quay.io to the mirror registry, skopeo gets 'authentication required' because the pod has no credentials for the mirror. Create the -dynamic-plugins-registry-auth secret from the combined pull secret. The chart already mounts this optional secret at /opt/app-root/src/.config/containers in the init container — no post-renderer changes needed. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 47955cd81c..8eae350afb 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -179,6 +179,18 @@ handle_ocp_disconnected_helm() { } log::success "ConfigMap mirror-registry-ca created in ${NAME_SPACE}" + # Registry auth for the init container — the chart mounts + # ${RELEASE_NAME}-dynamic-plugins-registry-auth at + # /opt/app-root/src/.config/containers (containers auth.json path). + oc create secret generic "${RELEASE_NAME}-dynamic-plugins-registry-auth" \ + --from-file="auth.json=${MIRROR_REGISTRY_PULL_SECRET}" \ + -n "${NAME_SPACE}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create registry auth secret — aborting" + return 1 + } + log::success "Secret ${RELEASE_NAME}-dynamic-plugins-registry-auth created in ${NAME_SPACE}" + # --- Section I: Helm deployment from mirrored chart --- log::section "Helm Deployment" From 02bc2f1921898a8b694194ece2d0eb9e6a5138e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Thu, 2 Jul 2026 11:25:54 +0200 Subject: [PATCH 11/15] refactor(disconnected): remove runtime tool installs (oc-mirror, podman) Both oc-mirror and podman are now pre-installed in the e2e-runner image (PR #5036). Remove: - disconnected::install_oc_mirror function and OC_MIRROR_BIN variable - Runtime apt-get install podman block in operator handler - Section lettering from both handlers Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 28 ++------- .../jobs/ocp-disconnected-operator.sh | 25 +------- .ci/pipelines/lib/disconnected.sh | 57 +------------------ 3 files changed, 7 insertions(+), 103 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 8eae350afb..ab69227b17 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -26,15 +26,6 @@ handle_ocp_disconnected_helm() { K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') export K8S_CLUSTER_ROUTER_BASE - # --- Section A: Install oc-mirror --- - log::section "oc-mirror Setup" - - disconnected::install_oc_mirror || { - log::error "Failed to install oc-mirror — aborting" - return 1 - } - - # --- Section B: Resolve chart source and pull locally --- log::section "Chart Resolution" local is_ga="false" @@ -43,7 +34,6 @@ handle_ocp_disconnected_helm() { fi if [[ "${is_ga}" == "true" ]]; then - # GA: pull chart from charts.openshift.io helm repo add openshift-helm-charts https://charts.openshift.io 2> /dev/null || true helm repo update openshift-helm-charts log::info "Pulling GA chart from charts.openshift.io (version: ${RELEASE_VERSION})" @@ -54,7 +44,6 @@ handle_ocp_disconnected_helm() { return 1 } else - # CI/upstream: pull chart from OCI registry log::info "Pulling CI chart from ${HELM_CHART_URL} (version: ${CHART_VERSION})" helm pull "${HELM_CHART_URL}" --version "${CHART_VERSION}" \ -d "${DISCONNECTED_TMPDIR}" || { @@ -72,7 +61,7 @@ handle_ocp_disconnected_helm() { fi log::success "Chart pulled: ${CHART_LOCAL_TGZ}" - # --- Section C: Resolve PostgreSQL image from chart --- + # Resolve PostgreSQL image from chart values local helm_values helm_values=$(helm show values "${CHART_LOCAL_TGZ}" 2> /dev/null || true) @@ -98,7 +87,6 @@ handle_ocp_disconnected_helm() { echo "${helm_values}" > "${ARTIFACT_DIR}/disconnected-helm-chart-values.yaml" 2> /dev/null || true - # --- Section D: Build ImageSetConfiguration --- log::section "Image Mirroring" local imageset_config="${DISCONNECTED_TMPDIR}/imageset-config.yaml" @@ -107,14 +95,12 @@ handle_ocp_disconnected_helm() { return 1 } - # --- Section E: Run oc-mirror --- local workspace="${DISCONNECTED_TMPDIR}/oc-mirror-workspace" disconnected::run_oc_mirror "${imageset_config}" "${workspace}" || { log::error "oc-mirror failed — aborting" return 1 } - # --- Section F: Patch and apply IDMS --- log::section "Cluster Resources" disconnected::patch_idms "${OC_MIRROR_IDMS_FILE}" @@ -133,7 +119,6 @@ handle_ocp_disconnected_helm() { log::success "ImageTagMirrorSet applied" fi - # --- Section G: Plugin mirroring --- log::section "Plugin Mirroring" disconnected::fetch_script "mirror-plugins.sh" "${DISCONNECTED_TMPDIR}/mirror-plugins.sh" || { @@ -153,7 +138,8 @@ handle_ocp_disconnected_helm() { return 1 } - # --- Section H: Namespace + registries.conf ConfigMap --- + log::section "Namespace and Secrets" + namespace::configure "${NAME_SPACE}" envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ @@ -191,7 +177,6 @@ handle_ocp_disconnected_helm() { } log::success "Secret ${RELEASE_NAME}-dynamic-plugins-registry-auth created in ${NAME_SPACE}" - # --- Section I: Helm deployment from mirrored chart --- log::section "Helm Deployment" # Prefer the chart from oc-mirror workspace, fall back to the pulled tgz @@ -215,10 +200,8 @@ handle_ocp_disconnected_helm() { ) fi - # Post-renderer appends registries.conf volume + mount to the rendered - # Deployment, avoiding the Helm "array clobber" pitfall where a values - # file that defines extraVolumes[] replaces the chart's entire default - # array (which grows across chart versions). + # Post-renderer appends disconnected volumes (registries.conf, mirror CA) + # to the rendered Deployment, avoiding the Helm "array clobber" pitfall. local post_renderer="${DIR}/resources/disconnected/helm-post-renderer.sh" helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \ @@ -235,7 +218,6 @@ handle_ocp_disconnected_helm() { printf '%s\n' "${helm_set_flags[@]}" > "${ARTIFACT_DIR}/disconnected-helm-set-flags.txt" 2> /dev/null || true - # --- Section J: Smoke test --- log::section "Smoke Test" local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index f41937ce7f..df8d7c1733 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -28,28 +28,10 @@ handle_ocp_disconnected_operator() { K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') export K8S_CLUSTER_ROUTER_BASE - # --- Section A: Operator Mirroring + Installation --- # Uses prepare-restricted-environment.sh from rhdh-operator, which handles # mirroring operator/operand images and installing the operator CatalogSource. - # Requires podman for building custom operator index images. log::section "Operator Mirroring and Installation" - if ! command -v podman &> /dev/null; then - log::info "Installing podman (required by prepare-restricted-environment.sh)..." - if ! apt-get update 2>&1; then - log::warn "apt-get update failed — continuing anyway" - fi - if ! apt-get install -y podman 2>&1; then - log::error "apt-get install podman failed" - fi - hash -r - if ! command -v podman &> /dev/null; then - log::error "podman not available after install. Add podman to the rhdh-e2e-runner Dockerfile." - return 1 - fi - log::success "podman installed: $(podman --version)" - fi - disconnected::fetch_script "prepare-restricted-environment.sh" "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" \ || { log::error "Failed to fetch prepare-restricted-environment.sh — aborting" @@ -76,13 +58,11 @@ handle_ocp_disconnected_operator() { } log::success "Operator installed via prepare-restricted-environment.sh" - # --- Section B: Wait for Operator CRD --- k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || { log::error "Backstage CRD not available after operator installation" return 1 } - # --- Section C: Plugin Mirroring --- log::section "Plugin Mirroring" disconnected::fetch_script "mirror-plugins.sh" "${DISCONNECTED_TMPDIR}/mirror-plugins.sh" \ @@ -103,8 +83,7 @@ handle_ocp_disconnected_operator() { return 1 } - # --- Section D: Namespace + registries.conf ConfigMap --- - log::section "Cluster Resources" + log::section "Namespace and Secrets" namespace::configure "${NAME_SPACE}" @@ -118,7 +97,6 @@ handle_ocp_disconnected_operator() { envsubst < "${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" \ > "${ARTIFACT_DIR}/disconnected-plugin-mirror-configmap.yaml" 2> /dev/null || true - # --- Section E: Backstage CR Deployment --- log::section "Backstage CR Deployment" local rendered_cr @@ -141,7 +119,6 @@ handle_ocp_disconnected_operator() { deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" log::success "Backstage CR deployed in ${NAME_SPACE}" - # --- Section F: Smoke Test --- log::section "Smoke Test" local url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index ef9f09a3a7..fa5171eee3 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -22,10 +22,6 @@ source "${DIR}/lib/common.sh" DISCONNECTED_TMPDIR=$(mktemp -d) export DISCONNECTED_TMPDIR -# oc-mirror binary path, set by disconnected::install_oc_mirror. -OC_MIRROR_BIN="" -export OC_MIRROR_BIN - # Validate that all required disconnected environment variables are set. # These are exported by the step-registry commands.sh before calling # openshift-ci-tests.sh. @@ -63,57 +59,6 @@ disconnected::setup_auth() { log::info "Container auth configured from ${MIRROR_REGISTRY_PULL_SECRET}" } -# Download the oc-mirror binary at runtime from mirror.openshift.com. -# Uses CONTAINER_PLATFORM_VERSION to select the matching version. -disconnected::install_oc_mirror() { - local arch - arch=$(uname -m) - case ${arch} in - x86_64) arch="amd64" ;; - aarch64) arch="arm64" ;; - esac - - local ocp_version="${CONTAINER_PLATFORM_VERSION:-4.21}" - local oc_mirror_version="" - - # Try stable channel for the OCP minor version, fall back to latest - local stable_url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/stable-${ocp_version}/" - if curl -sf --head --connect-timeout 10 "${stable_url}" > /dev/null 2>&1; then - oc_mirror_version="stable-${ocp_version}" - log::info "Using oc-mirror from stable-${ocp_version} channel" - else - oc_mirror_version="latest" - log::info "stable-${ocp_version} not available, using oc-mirror from latest channel" - fi - - local download_dir="${DISCONNECTED_TMPDIR}/oc-mirror-download" - mkdir -p "${download_dir}" - - local base_url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/${oc_mirror_version}" - - log::info "Downloading oc-mirror from ${base_url}/" - if ! curl -fL --retry 5 --connect-timeout 30 -o "${download_dir}/oc-mirror.tar.gz" "${base_url}/oc-mirror.tar.gz"; then - log::error "Failed to download oc-mirror" - return 1 - fi - - # Verify checksum - if curl -fL --retry 3 --connect-timeout 30 -o "${download_dir}/sha256sum.txt" "${base_url}/sha256sum.txt" 2> /dev/null; then - if grep "oc-mirror.tar.gz" "${download_dir}/sha256sum.txt" | (cd "${download_dir}" && sha256sum -c -); then - log::info "oc-mirror checksum verified" - else - log::warn "oc-mirror checksum verification failed — continuing anyway" - fi - fi - - tar -xzf "${download_dir}/oc-mirror.tar.gz" -C "${download_dir}" - chmod +x "${download_dir}/oc-mirror" - OC_MIRROR_BIN="${download_dir}/oc-mirror" - export OC_MIRROR_BIN - - log::success "oc-mirror installed: $(${OC_MIRROR_BIN} version --output=yaml 2>&1 | head -1)" -} - # Build an ImageSetConfiguration for oc-mirror. # The configuration is dynamically generated based on IMAGE_REGISTRY: # - registry.redhat.io (GA): uses helm.local with chart pulled from charts.openshift.io @@ -187,7 +132,7 @@ disconnected::run_oc_mirror() { unset REGISTRY_AUTH_FILE log::info "Running oc-mirror --v2 → ${MIRROR_REGISTRY_URL}" - if ! "${OC_MIRROR_BIN}" \ + if ! oc-mirror \ -c "${imageset_config}" \ "docker://${MIRROR_REGISTRY_URL}" \ --dest-tls-verify=false \ From 8cb4485af2d45c6f2b57a8c3213bdcf4fd3d9b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Thu, 2 Jul 2026 14:10:07 +0200 Subject: [PATCH 12/15] fix(disconnected): skip signature verification for mirrored plugins The install-dynamic-plugins init container fails to install Lightspeed plugins from the mirror registry: Source image rejected: A signature was required, but no signature exists The RHDH image ships a policy.json (RHDHBUGS-2799) that requires Red Hat GPG signatures for registry.access.redhat.com images. In a disconnected environment, IDMS redirects these pulls to the mirror registry, but the mirror doesn't host Red Hat's signature store. Add a permissive policy.json to the existing rhdh-plugin-mirror-conf ConfigMap and mount it at /etc/containers/policy.json in the init container. This is the standard approach for air-gapped environments where the signature server is unreachable. Assisted-by: OpenCode --- .ci/pipelines/resources/disconnected/helm-post-renderer.sh | 4 ++++ .../resources/disconnected/plugin-mirror-configmap.yaml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh index 68a6bd6de5..8bbda459ff 100755 --- a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh +++ b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh @@ -7,6 +7,9 @@ # 2. Mounts the mirror registry CA at the standard container-tools # per-registry path (/etc/containers/certs.d//ca.crt) # so skopeo trusts the mirror's TLS certificate. +# 3. Mounts a permissive policy.json so skopeo accepts unsigned images +# from the mirror (Red Hat signature server is unreachable in +# disconnected environments). # # Using a post-renderer avoids the Helm "array clobber" pitfall: # a values file that defines extraVolumes[] or initContainers[] replaces @@ -31,6 +34,7 @@ yq eval " ] | .spec.template.spec.initContainers[0].volumeMounts += [ {\"mountPath\": \"/etc/containers/registries.conf.d/rhdh-registries.conf\", \"name\": \"rhdh-plugin-mirror-conf\", \"readOnly\": true, \"subPath\": \"rhdh-registries.conf\"}, + {\"mountPath\": \"/etc/containers/policy.json\", \"name\": \"rhdh-plugin-mirror-conf\", \"readOnly\": true, \"subPath\": \"policy.json\"}, {\"mountPath\": \"/etc/containers/certs.d/${MIRROR_REGISTRY_URL}/ca.crt\", \"name\": \"mirror-registry-ca\", \"readOnly\": true, \"subPath\": \"ca.crt\"} ] ) // . diff --git a/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml b/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml index 2e14f555c9..7b5acdb5c5 100644 --- a/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml +++ b/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml @@ -15,3 +15,5 @@ data: [[registry]] prefix = "ghcr.io/redhat-developer/rhdh-plugin-export-overlays" location = "${MIRROR_REGISTRY_URL}/rhdh-plugin-export-overlays" + policy.json: | + {"default": [{"type": "insecureAcceptAnything"}]} From 8de2b86256cc10d706568272cf73cdd7a2a78b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 3 Jul 2026 09:36:01 +0200 Subject: [PATCH 13/15] fix(disconnected): configure proxy for Playwright and skip signature verification Two issues fixed: 1. Playwright page.goto timeout (50s) on disconnected clusters: The CI runner accesses the test cluster through a squid proxy (HTTPS_PROXY set by proxy-conf.sh). curl uses it automatically, but Playwright's Chromium does not pick up the env var without explicit configuration. Add proxy to Playwright's use config, reading from HTTPS_PROXY when set (no-op for connected envs). 2. Lightspeed plugin signature verification failure: 'Source image rejected: A signature was required, but no signature exists' The RHDH image's policy.json (RHDHBUGS-2799) requires Red Hat GPG signatures for registry.access.redhat.com images. In disconnected environments, IDMS redirects to the mirror which lacks signatures. Mount a permissive policy.json in the init container via the existing rhdh-plugin-mirror-conf ConfigMap. Assisted-by: OpenCode --- e2e-tests/playwright.config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 6c87fb3271..669982207e 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -58,6 +58,12 @@ export default defineConfig({ locale: process.env.LOCALE ?? "en", baseURL: process.env.BASE_URL, ignoreHTTPSErrors: true, + // Proxy for disconnected environments where the CI runner reaches the + // cluster through a squid proxy (HTTPS_PROXY set by proxy-conf.sh). + proxy: + process.env.HTTPS_PROXY !== undefined && process.env.HTTPS_PROXY !== "" + ? { server: process.env.HTTPS_PROXY } + : undefined, trace: "on", screenshot: "on", ...devices["Desktop Chrome"], From c63a3716c372698d89510bfc06aed83233334376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Sat, 4 Jul 2026 10:58:34 +0200 Subject: [PATCH 14/15] fix(disconnected): set BUILDAH_ISOLATION=chroot for operator handler The CI pod runs with nested_podman: true (hostUsers: false), placing it inside a Linux user namespace. When prepare-restricted-environment.sh calls podman build, podman tries to create another user namespace inside the existing one, which fails with: newuidmap: open of uid_map failed: Permission denied Error: cannot set up namespace using /usr/bin/newuidmap: exit status 1 Export BUILDAH_ISOLATION=chroot before invoking the script so all podman build / buildah calls use chroot isolation instead of nested user namespaces. The env var is respected by both podman and buildah without needing to modify the downstream rhdh-operator script. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-operator.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index df8d7c1733..fab3eb85b2 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -51,6 +51,12 @@ handle_ocp_disconnected_operator() { ) fi + # The CI pod runs with nested_podman: true (hostUsers: false), placing it + # inside a Linux user namespace. podman build tries to create another user + # namespace, which fails with "newuidmap: open of uid_map failed: Permission + # denied". BUILDAH_ISOLATION=chroot uses chroot instead of nested namespaces. + export BUILDAH_ISOLATION=chroot + bash "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" "${prepare_args[@]}" \ || { log::error "prepare-restricted-environment.sh failed — aborting" From 588bb84c751f9f7185d3b7842e9283605984f6ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Sat, 4 Jul 2026 13:31:26 +0200 Subject: [PATCH 15/15] fix(disconnected): parse proxy credentials and fix nested podman MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues fixed: 1. Playwright proxy credentials not parsed: HTTPS_PROXY is http://user:pass@host:3128 (credentials in URL). Playwright requires credentials as separate username/password fields, not embedded in the server URL. Add parseProxy() helper that extracts them via URL parsing. 2. Operator podman build fails with nested user namespaces: 'newuidmap: open of uid_map failed: Permission denied' BUILDAH_ISOLATION=chroot alone is insufficient — the error occurs during podman's rootless storage setup before build isolation applies. Set _CONTAINERS_USERNS_CONFIGURED=1 to skip newuidmap and force vfs storage driver to avoid fuse-overlayfs userns ops. 3. Added debug logging for proxy config and podman environment to aid future CI troubleshooting. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 4 ++++ .../jobs/ocp-disconnected-operator.sh | 17 +++++++++++--- e2e-tests/playwright.config.ts | 23 +++++++++++++++---- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index ab69227b17..781ccfd979 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -220,6 +220,10 @@ handle_ocp_disconnected_helm() { log::section "Smoke Test" + if [[ -n "${HTTPS_PROXY:-}" ]]; then + log::info "HTTPS_PROXY is set (Playwright will use it): ${HTTPS_PROXY%%@*}@***" + fi + local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SMOKE_TEST}" "${url}" diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index fab3eb85b2..4143630662 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -52,10 +52,21 @@ handle_ocp_disconnected_operator() { fi # The CI pod runs with nested_podman: true (hostUsers: false), placing it - # inside a Linux user namespace. podman build tries to create another user - # namespace, which fails with "newuidmap: open of uid_map failed: Permission - # denied". BUILDAH_ISOLATION=chroot uses chroot instead of nested namespaces. + # inside a Linux user namespace. podman's rootless setup calls newuidmap to + # create a nested user namespace, which fails with: + # newuidmap: open of uid_map failed: Permission denied + # Fix: + # _CONTAINERS_USERNS_CONFIGURED=1 — skip newuidmap (userns already set up) + # BUILDAH_ISOLATION=chroot — chroot instead of user namespace for builds + # storage driver=vfs — avoid fuse-overlayfs which may need userns ops + export _CONTAINERS_USERNS_CONFIGURED=1 export BUILDAH_ISOLATION=chroot + mkdir -p "${HOME}/.config/containers" + printf '[storage]\ndriver = "vfs"\n' > "${HOME}/.config/containers/storage.conf" + + log::info "Podman environment: uid=$(id -u), BUILDAH_ISOLATION=${BUILDAH_ISOLATION}" + log::info "Storage config: $(cat "${HOME}/.config/containers/storage.conf" | tr '\n' ' ')" + log::info "subuid: $(cat /etc/subuid 2> /dev/null || echo 'not found')" bash "${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" "${prepare_args[@]}" \ || { diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 669982207e..1bc187e1dc 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -6,6 +6,24 @@ import { PW_PROJECT } from "./playwright/projects"; process.env.JOB_NAME = process.env.JOB_NAME ?? ""; process.env.IS_OPENSHIFT = process.env.IS_OPENSHIFT ?? ""; +// Parse HTTPS_PROXY (http://user:pass@host:port) into Playwright's proxy +// config with separate username/password fields. No-op for connected envs. +function parseProxy( + proxyUrl: string | undefined, +): { server: string; username?: string; password?: string } | undefined { + if (proxyUrl === undefined || proxyUrl === "") return undefined; + try { + const u = new URL(proxyUrl); + return { + server: `${u.protocol}//${u.host}`, + ...(u.username !== "" && { username: decodeURIComponent(u.username) }), + ...(u.password !== "" && { password: decodeURIComponent(u.password) }), + }; + } catch { + return { server: proxyUrl }; + } +} + // Set LOCALE based on which project is being run const args = process.argv; @@ -60,10 +78,7 @@ export default defineConfig({ ignoreHTTPSErrors: true, // Proxy for disconnected environments where the CI runner reaches the // cluster through a squid proxy (HTTPS_PROXY set by proxy-conf.sh). - proxy: - process.env.HTTPS_PROXY !== undefined && process.env.HTTPS_PROXY !== "" - ? { server: process.env.HTTPS_PROXY } - : undefined, + proxy: parseProxy(process.env.HTTPS_PROXY), trace: "on", screenshot: "on", ...devices["Desktop Chrome"],