diff --git a/.github/workflows/00-daily-integration-26.04-lxd-container.yml b/.github/workflows/00-daily-integration-26.04-lxd-container.yml deleted file mode 100644 index 0c4193bd1b5..00000000000 --- a/.github/workflows/00-daily-integration-26.04-lxd-container.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Daily(26.04): lxd_container" - -on: - workflow_dispatch: - schedule: - - cron: '2 22 * * *' - -jobs: - resolute-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml - with: - release: resolute - platform: lxd_container diff --git a/.github/workflows/01-daily-integration-25.10-lxd-container.yml b/.github/workflows/01-daily-integration-25.10-lxd-container.yml deleted file mode 100644 index b2eb301cbee..00000000000 --- a/.github/workflows/01-daily-integration-25.10-lxd-container.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Daily(25.10): lxd_container" - -on: - workflow_dispatch: - schedule: - - cron: '2 22 * * *' - -jobs: - questing-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml - with: - release: questing - platform: lxd_container diff --git a/.github/workflows/02-daily-integration-24.04-lxd-container.yml b/.github/workflows/02-daily-integration-24.04-lxd-container.yml deleted file mode 100644 index acfd4c9e0f9..00000000000 --- a/.github/workflows/02-daily-integration-24.04-lxd-container.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Daily(24.04): lxd_container" - -on: - workflow_dispatch: - schedule: - - cron: '2 22 * * *' - -jobs: - noble-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml - with: - release: noble - platform: lxd_container diff --git a/.github/workflows/03-daily-integration-22.04-lxd-container.yml b/.github/workflows/03-daily-integration-22.04-lxd-container.yml deleted file mode 100644 index d17a36f39c5..00000000000 --- a/.github/workflows/03-daily-integration-22.04-lxd-container.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Daily(22.04): lxd_container" - -on: - workflow_dispatch: - schedule: - - cron: '2 22 * * *' - -jobs: - jammy-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml - with: - release: jammy - platform: lxd_container diff --git a/.github/workflows/10-daily-unit-lint.yml b/.github/workflows/10-daily-unit-lint.yml index 898c86ccc20..5e2452df679 100644 --- a/.github/workflows/10-daily-unit-lint.yml +++ b/.github/workflows/10-daily-unit-lint.yml @@ -28,6 +28,7 @@ jobs: run: tox -e hypothesis-slow devel_tests: name: "Tip: Python" + if: github.repository == 'canonical/cloud-init' runs-on: ubuntu-latest steps: - name: Checkout @@ -44,14 +45,15 @@ jobs: - name: Run unittest run: tox -e py3 -- --color=yes format_tip: + name: "Tip: Lint" + if: github.repository == 'canonical/cloud-init' + runs-on: ubuntu-latest env: FORCE_COLOR: 1 strategy: fail-fast: false matrix: env: [tip-ruff, tip-mypy, tip-pylint, tip-black, tip-isort, py3-fast] - name: "Tip: Lint" - runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -65,6 +67,7 @@ jobs: run: tox macos: name: Python ${{matrix.python-version}} ${{ matrix.slug }} + if: github.repository == 'canonical/cloud-init' runs-on: macos-latest strategy: matrix: diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml new file mode 100644 index 00000000000..92c628363b4 --- /dev/null +++ b/.github/workflows/100-dispatch-common.yml @@ -0,0 +1,165 @@ +# WARNING: This workflow handles repository secrets (PYCLOUDLIB_TOML_B64, SSH keys). +# It MUST NOT be triggered by PR-driven workflows to prevent secret exposure to +# untrusted code submitted via pull requests. +name: "Dispatch: Integration" + +on: + workflow_dispatch: + inputs: + release: + required: true + type: choice + options: + - resolute + - questing + - jammy + - noble + platform: + required: true + type: choice + options: + - lxd_container + - lxd_vm + - ec2 + image_type: + required: true + type: choice + options: + - generic + - minimal + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + filter_tests: + required: false + type: string + workflow_call: + inputs: + release: + required: true + type: string + platform: + required: true + type: string + image_type: + required: true + type: string + install_source: + required: true + type: string + filter_tests: + required: false + type: string + secrets: + PYCLOUDLIB_TOML_B64: + required: true + SSH_PRIVATE_KEY: + required: true + SSH_PUBLIC_KEY: + required: true + +jobs: + integration-test: + runs-on: ubuntu-latest + if: github.repository == 'canonical/cloud-init' + + env: + REQUIRED_SECRET: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + CLOUD_INIT_PLATFORM: ${{ inputs.platform }} + CLOUD_INIT_OS_IMAGE: ${{ inputs.release }} + CLOUD_INIT_OS_IMAGE_TYPE: ${{ inputs.image_type }} + CLOUD_INIT_CLOUD_INIT_SOURCE: ${{ inputs.install_source }} + + steps: + - name: Assert required repo secrets are set + run: | + if [ -z "$REQUIRED_SECRET" ]; then + echo "ERROR: Missing required repo secret. Please provide the necessary repo secret at ${{ github.repository }}/settings/secrets/actions." + exit 1 + fi + if [ -z "$SSH_PUBLIC_KEY" ]; then + echo "ERROR: Missing required repo secret. Please provide SSH_PUBLIC_KEY repo secret at ${{ github.repository }}/settings/secrets/actions." + exit 1 + fi + if [ -z "$SSH_PRIVATE_KEY" ]; then + echo "ERROR: Missing required repo secret. Please provide SSH_PRIVATE_KEY repo secret at ${{ github.repository }}/settings/secrets/actions." + exit 1 + fi + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup LXD + # This shared workflow supports manual dispatch with platform choices including + # lxd_vm and lxd_container. Only install the lxd snap for those platforms; + # cloud platforms such as EC2 do not need it. + if: ${{ contains(fromJSON('["lxd_vm", "lxd_container"]'), env.CLOUD_INIT_PLATFORM ) }} + uses: canonical/setup-lxd@8c6a87bfb56aa48f3fb9b830baa18562d8bfd4ee # v1 + with: + channel: 6/stable + - name: Clean workspace + run: | + rm -rf ${{ github.workspace }}/cloud_init_test_logs/ + - name: Setup SSH + run: | + mkdir -p ~/.ssh + sh -c 'printf "%s\n" "$SSH_PUBLIC_KEY" > ~/.ssh/cloudinit_id_rsa.pub' + # Dump secret using a subprocess to avoid accidental leaks when using set -x. + sh -c 'printf "%s\n" "$SSH_PRIVATE_KEY" | install -m 600 /dev/stdin ~/.ssh/cloudinit_id_rsa' + - name: Setup pycloudlib + env: + PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml + run: | + sh -c 'echo "$REQUIRED_SECRET" | base64 -d > "$PYCLOUDLIB_CONFIG"' + - name: Install Dependencies + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update + sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox distro-info-data devscripts + - name: Run integration Tests + env: + CLOUD_INIT_LOCAL_LOG_PATH: ${{ github.workspace }}/cloud_init_test_logs + PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml + run: | + tox -e integration-tests -- --junitxml="${{ github.workspace }}/junit-report.xml" --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} + - name: Publish Test Report with Insights + if: ${{ always() }} + uses: ctrf-io/github-test-reporter@024bc4b64d997ca9da86833c6b9548c55c620e40 # v1.0.26 + with: + report-path: '${{ github.workspace }}/junit-report.xml' + title: '${{ inputs.platform }}-${{ inputs.release }}-${{ inputs.image_type }}:${{ inputs.install_source }}' + integrations-config: | + { + "junit-to-ctrf": { + "enabled": true, + "action": "convert", + "options": { + "output": "./ctrf-report.json", + "toolname": "junit-to-ctrf", + "useSuiteName": false, + "env": { + "appName": "my-app" + } + } + } + } + summary-delta-report: true + insights-report: true + flaky-rate-report: true + slowest-report: true + upload-artifact: true + github-report: true + artifact-name: ctrf-report-${{ inputs.platform }}-${{ inputs.release }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload failure artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: failure-${{ github.job }} + path: ${{ github.workspace }}/cloud_init_test_logs/ + retention-days: 2 + - name: Clean pycloudlib + if: ${{ always() }} + run: | + rm -f ${{ runner.temp }}/pycloudlib.toml diff --git a/.github/workflows/11-dispatch-common.yml b/.github/workflows/11-dispatch-common.yml deleted file mode 100644 index 66643329e97..00000000000 --- a/.github/workflows/11-dispatch-common.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: "Dispatch: Integration" - -on: - workflow_dispatch: - inputs: - release: - required: true - type: choice - options: - - questing - - jammy - - noble - - resolute - platform: - required: true - type: choice - options: - - lxd_container - - lxd_vm - image_type: - type: choice - options: - - generic - - minimal - required: false - install_source: - required: false - type: string - filter_tests: - required: false - type: string - workflow_call: - inputs: - release: - required: true - type: string - platform: - required: true - type: string - image_type: - required: false - type: string - install_source: - required: false - type: string - filter_tests: - required: false - type: string - -jobs: - lxc: - runs-on: ubuntu-latest - if: github.repository == 'canonical/cloud-init' - - env: - CLOUD_INIT_PLATFORM: ${{ inputs.platform || 'lxd_container' }} - CLOUD_INIT_OS_IMAGE: ${{ inputs.release || 'questing' }} - CLOUD_INIT_OS_IMAGE_TYPE: ${{ inputs.image_type || 'generic' }} - CLOUD_INIT_CLOUD_INIT_SOURCE: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - CLOUD_INIT_LOCAL_LOG_PATH: ${{ github.workspace }}/cloud_init_test_logs - - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Setup LXD - uses: canonical/setup-lxd@8c6a87bfb56aa48f3fb9b830baa18562d8bfd4ee # v0.1.2 - with: - channel: 6/stable - - name: Clean workspace - run: | - rm -rf ${{ github.workspace }}/cloud_init_test_logs/ - - name: Setup pycloudlib - run: | - ssh-keygen -P "" -q -f ~/.ssh/id_rsa - echo "[lxd]" > /home/$USER/.config/pycloudlib.toml - - name: Install Dependencies - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update - sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox distro-info-data - - name: Run integration Tests - run: | - tox -e integration-tests -- --junitxml="${{ github.workspace }}/reports/junit-report-${{ inputs.platform }}-${{ inputs.release }}.xml" --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} - - name: Publish Test Report with Insights - if: always() - uses: ctrf-io/github-test-reporter@024bc4b64d997ca9da86833c6b9548c55c620e40 # v1.0.26 - with: - report-path: '${{ github.workspace }}/reports/junit-report-${{ inputs.platform}}-${{ inputs.release }}.xml' - integrations-config: | - { - "junit-to-ctrf": { - "enabled": true, - "action": "convert", - "options": { - "output": "./reports/ctrf-report-${{ inputs.platform }}-${{ inputs.release }}-${{ inputs.image_type }}.json", - "toolname": "junit-to-ctrf", - "useSuiteName": false, - "env": { - "appName": "my-app" - } - } - } - } - summary-delta-report: true - insights-report: true - flaky-rate-report: true - slowest-report: true - upload-artifact: true - github-report: true - artifact-name: ctrf-report-${{ inputs.platform }}-${{inputs.release}} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload failure artifacts - if: failure() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: failure-${{ github.job }} - path: ${{ github.workspace }}/cloud_init_test_logs/ - retention-days: 2 diff --git a/.github/workflows/110-daily-integration-22.04-lxd_container.yml b/.github/workflows/110-daily-integration-22.04-lxd_container.yml new file mode 100644 index 00000000000..bf719312c78 --- /dev/null +++ b/.github/workflows/110-daily-integration-22.04-lxd_container.yml @@ -0,0 +1,34 @@ +name: "Daily 22.04: lxd_container" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 22 * * *' + +jobs: + jammy-lxd_container: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: jammy + platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/111-daily-integration-24.04-lxd_container.yml b/.github/workflows/111-daily-integration-24.04-lxd_container.yml new file mode 100644 index 00000000000..73fcd9e302d --- /dev/null +++ b/.github/workflows/111-daily-integration-24.04-lxd_container.yml @@ -0,0 +1,34 @@ +name: "Daily 24.04: lxd_container" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 22 * * *' + +jobs: + noble-lxd_container: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: noble + platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/112-daily-integration-25.10-lxd_container.yml b/.github/workflows/112-daily-integration-25.10-lxd_container.yml new file mode 100644 index 00000000000..e59de2c3feb --- /dev/null +++ b/.github/workflows/112-daily-integration-25.10-lxd_container.yml @@ -0,0 +1,34 @@ +name: "Daily 25.10: lxd_container" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 22 * * *' + +jobs: + questing-lxd_container: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: questing + platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/113-daily-integration-26.04-lxd_container.yml b/.github/workflows/113-daily-integration-26.04-lxd_container.yml new file mode 100644 index 00000000000..2e12f25ce03 --- /dev/null +++ b/.github/workflows/113-daily-integration-26.04-lxd_container.yml @@ -0,0 +1,34 @@ +name: "Daily 26.04: lxd_container" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 22 * * *' + +jobs: + resolute-lxd_container: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: resolute + platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/120-daily-integration-22.04-lxd_vm.yml b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml new file mode 100644 index 00000000000..2b81dfca8e0 --- /dev/null +++ b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml @@ -0,0 +1,34 @@ +name: "Daily 22.04: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 23 * * *' + +jobs: + jammy-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: jammy + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/121-daily-integration-24.04-lxd_vm.yml b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml new file mode 100644 index 00000000000..fa1b671223a --- /dev/null +++ b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml @@ -0,0 +1,34 @@ +name: "Daily 24.04: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 23 * * *' + +jobs: + noble-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: noble + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/122-daily-integration-25.10-lxd_vm.yml b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml new file mode 100644 index 00000000000..c14c2abd9bb --- /dev/null +++ b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml @@ -0,0 +1,34 @@ +name: "Daily 25.10: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 23 * * *' + +jobs: + questing-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: questing + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/123-daily-integration-26.04-lxd_vm.yml b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml new file mode 100644 index 00000000000..54a5dc4413a --- /dev/null +++ b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml @@ -0,0 +1,34 @@ +name: "Daily 26.04: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + - cron: '2 23 * * *' + +jobs: + resolute-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: resolute + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/130-daily-integration-22.04-ec2.yml b/.github/workflows/130-daily-integration-22.04-ec2.yml new file mode 100644 index 00000000000..78d509cfea0 --- /dev/null +++ b/.github/workflows/130-daily-integration-22.04-ec2.yml @@ -0,0 +1,35 @@ +name: "Twice weekly 22.04: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + # Run Mon & Thurs to reduce additional cost on shared external tokens + - cron: '2 22 * * 1,4' + +jobs: + jammy-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: jammy + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/131-daily-integration-24.04-ec2.yml b/.github/workflows/131-daily-integration-24.04-ec2.yml new file mode 100644 index 00000000000..52d88e9ba8e --- /dev/null +++ b/.github/workflows/131-daily-integration-24.04-ec2.yml @@ -0,0 +1,35 @@ +name: "Twice weekly 24.04: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + # Run Mon & Thurs to reduce additional cost on shared external tokens + - cron: '2 22 * * 1,4' + +jobs: + noble-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: noble + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/132-daily-integration-25.10-ec2.yml b/.github/workflows/132-daily-integration-25.10-ec2.yml new file mode 100644 index 00000000000..aeb66287533 --- /dev/null +++ b/.github/workflows/132-daily-integration-25.10-ec2.yml @@ -0,0 +1,35 @@ +name: "Twice weekly 25.10: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + # Run Mon & Thurs to reduce additional cost on shared external tokens + - cron: '2 22 * * 1,4' + +jobs: + questing-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: questing + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/133-daily-integration-26.04-ec2.yml b/.github/workflows/133-daily-integration-26.04-ec2.yml new file mode 100644 index 00000000000..cabaf3e1dcb --- /dev/null +++ b/.github/workflows/133-daily-integration-26.04-ec2.yml @@ -0,0 +1,35 @@ +name: "Twice weekly 26.04: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string + schedule: + # Run Mon & Thurs to reduce additional cost on shared external tokens + - cron: '2 22 * * 1,4' + +jobs: + resolute-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: resolute + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} + secrets: + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/21-pr-check-format.yml b/.github/workflows/21-pr-check-format.yml index 9fb9063e2cb..afe4f25208b 100644 --- a/.github/workflows/21-pr-check-format.yml +++ b/.github/workflows/21-pr-check-format.yml @@ -13,11 +13,11 @@ defaults: shell: sh -ex {0} jobs: - check_format: + check_linters: strategy: fail-fast: false matrix: - env: [ruff, mypy, pylint, black, isort] + env: [check_format] name: Check ${{ matrix.env }} runs-on: ubuntu-latest env: @@ -30,6 +30,7 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox + sudo snap install shfmt - name: Print version run: python3 --version @@ -39,20 +40,12 @@ jobs: # matrix env: not to be confused w/environment variables or testenv TOXENV: ${{ matrix.env }} run: tox - schema-format: - strategy: - fail-fast: false - name: Check json format - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Test format + - name: Formatting check failed + if: failure() run: | - tools/check_json_format.sh cloudinit/config/schemas/schema-cloud-config-v1.json - tools/check_json_format.sh cloudinit/config/schemas/schema-network-config-v1.json - tools/check_json_format.sh cloudinit/config/schemas/versions.schema.cloud-config.json + echo "[31mResolve formatting errors with `tox -e do_format` (requires shfmt to format shell).\e[0m" + echo "[31mFor mypy and pylint failures see the warnings above.\e[0m" doc: strategy: diff --git a/.github/workflows/23-pr-unit-distro.yml b/.github/workflows/23-pr-unit-distro.yml index e2fe03ba414..cef37e14e25 100644 --- a/.github/workflows/23-pr-unit-distro.yml +++ b/.github/workflows/23-pr-unit-distro.yml @@ -42,7 +42,7 @@ jobs: lxc exec alpine -- ping -c 1 dl-cdn.alpinelinux.org || true - name: Install dependencies - run: lxc exec alpine -- apk add py3-tox git tzdata + run: lxc exec alpine -- apk add py3-tox py3-python-discovery git tzdata - name: Mount source into container directory run: lxc config device add alpine gitdir disk source=$(pwd) path=/root/cloud-init-ro diff --git a/.github/workflows/24-pr-integration.yml b/.github/workflows/24-pr-integration.yml index f25849eed3f..9c1c40e8cf0 100644 --- a/.github/workflows/24-pr-integration.yml +++ b/.github/workflows/24-pr-integration.yml @@ -67,4 +67,12 @@ jobs: echo "[lxd]" > /home/$USER/.config/pycloudlib.toml - name: Run integration Tests run: | - CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} tox -e integration-tests-ci -- --color=yes tests/integration_tests/ + CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} CLOUD_INIT_LOCAL_LOG_PATH=./cloudinit_logs tox -e integration-tests-ci -- --color=yes tests/integration_tests/ + - name: Upload cloudinit logs on failure + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: 'cloudinit-logs' + path: './cloudinit_logs' + retention-days: 3 + if-no-files-found: ignore diff --git a/.github/workflows/45-ci-label.yml b/.github/workflows/45-ci-label.yml index f76b8cf4acd..9372f1f3527 100644 --- a/.github/workflows/45-ci-label.yml +++ b/.github/workflows/45-ci-label.yml @@ -5,6 +5,9 @@ on: jobs: labeler: if: github.repository == 'canonical/cloud-init' + permissions: + contents: read + pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + - uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0 diff --git a/.pylintrc b/.pylintrc index 028e4ae73e7..2a3b18226b5 100644 --- a/.pylintrc +++ b/.pylintrc @@ -32,32 +32,3 @@ output-format=parseable # Just the errors please, no full report reports=no - - -[TYPECHECK] - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - http.client, - httplib, - pkg_resources, - # cloud_tests requirements. - boto3, - botocore, - paramiko, - pylxd, - simplestreams - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -# argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413 -ignored-classes=argparse.Namespace,optparse.Values,thread._local,ImageManager,ContainerManager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members=types,http.client,command_handlers,m_.*,enter_context diff --git a/Makefile b/Makefile index a425420c9d8..7e2ef07211e 100644 --- a/Makefile +++ b/Makefile @@ -9,40 +9,18 @@ distro ?= redhat GENERATOR_F=./systemd/cloud-init-generator DS_IDENTIFY=./tools/ds-identify -BENCHMARK=./tools/benchmark.sh all: check check: test -style-check: lint - -lint: - @$(CWD)/tools/run-lint - unittest: clean_pyc $(PYTHON) -m pytest -v tests/unittests cloudinit render-template: $(PYTHON) ./tools/render-template --variant=$(VARIANT) $(FILE) $(subst .tmpl,,$(FILE)) -# from systemd-generator(7) regarding generators: -# "We do recommend C code however, since generators are executed -# synchronously and hence delay the entire boot if they are slow." -# -# Our generator is a shell script. Make it easy to measure the -# generator. This should be monitored for performance regressions -benchmark-generator: FILE=$(GENERATOR_F).tmpl -benchmark-generator: VARIANT="benchmark" -benchmark-generator: export ITER=$(NUM_ITER) -benchmark-generator: render-template - $(BENCHMARK) $(GENERATOR_F) - -benchmark-ds-identify: export ITER=$(NUM_ITER) -benchmark-ds-identify: - $(BENCHMARK) $(DS_IDENTIFY) - ci-deps-ubuntu: @$(PYTHON) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro @@ -103,13 +81,6 @@ deb-src: doc: tox -e doc -fmt: - tox -e do_format && tox -e check_format - -fmt-tip: - tox -e do_format_tip && tox -e check_format_tip - - -.PHONY: all check test lint clean rpm srpm deb deb-src clean_pyc -.PHONY: unittest style-check render-template benchmark-generator +.PHONY: all check test clean rpm srpm deb deb-src clean_pyc +.PHONY: unittest render-template .PHONY: clean_pytest clean_packaging clean_release doc diff --git a/README.md b/README.md index cfbb470b6d0..039d1f15b26 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # cloud-init -![Unit Tests](https://github.com/canonical/cloud-init/actions/workflows/unit.yml/badge.svg?branch=main) -![Integration Tests](https://github.com/canonical/cloud-init/actions/workflows/integration.yml/badge.svg?branch=main) -![Documentation](https://github.com/canonical/cloud-init/actions/workflows/check_format.yml/badge.svg?branch=main) +![Unit Tests](https://github.com/canonical/cloud-init/actions/workflows/22-pr-unit-python.yml/badge.svg?branch=main) +![Integration Tests](https://github.com/canonical/cloud-init/actions/workflows/24-pr-integration.yml/badge.svg?branch=main) +![Documentation](https://github.com/canonical/cloud-init/actions/workflows/21-pr-check-format.yml/badge.svg?branch=main) Cloud-init is the *industry standard* multi-distribution method for cross-platform cloud instance initialization. It is supported across all diff --git a/cloudinit/analyze/__init__.py b/cloudinit/analyze/__init__.py index ea2144cb387..ae978ba7a01 100644 --- a/cloudinit/analyze/__init__.py +++ b/cloudinit/analyze/__init__.py @@ -115,7 +115,7 @@ def get_parser( return parser -def analyze_boot(name: str, args: argparse.Namespace) -> str: +def analyze_boot(name: str, args: argparse.Namespace) -> int: """Report a list of how long different boot operations took. For Example: @@ -199,7 +199,7 @@ def analyze_boot(name: str, args: argparse.Namespace) -> str: outfh.write(status_map[status_code].format(**kwargs)) clean_io(infh, outfh) - return status_code + return 1 if status_code == show.FAIL_CODE else 0 def analyze_blame(name, args: argparse.Namespace) -> None: diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 5411ad602d1..e233df04c65 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -32,11 +32,11 @@ def create_mime_message(files): ) content_type = sub_message.get_content_type().lower() if content_type not in get_content_types(): - msg = ("content type %r for attachment %s may be incorrect!") % ( - content_type, - i + 1, + err_msg = ( + f"content type {content_type!r} for attachment" + f" {i + 1} may be incorrect!" ) - errors.append(msg) + errors.append(err_msg) sub_messages.append(sub_message) combined_message = MIMEMultipart() for msg in sub_messages: diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index eafb11f16e9..d11ff64c024 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -18,6 +18,7 @@ network_manager, network_state, networkd, + renderer, sysconfig, ) @@ -156,15 +157,17 @@ def handle_args(name, args): pre_ns = azure.generate_network_config_from_instance_network_metadata( json.loads(net_data)["network"], apply_network_config_for_secondary_ips=True, + apply_network_config_set_name=True, ) elif args.kind == "vmware-imc": - config = guestcust_util.Config( + vmware_config = guestcust_util.Config( guestcust_util.ConfigFile(args.network_data.name) ) pre_ns = guestcust_util.get_network_data_from_vmware_cust_cfg( - config, False + vmware_config, False ) + r_cls: type[renderer.Renderer] distro_cls = distros.fetch(args.distro) distro = distro_cls(args.distro, {}, None) if args.output_kind == "eni": diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 042d89420e2..bc1fcdd87c4 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -19,7 +19,7 @@ import traceback import logging import yaml -from typing import Optional, Tuple, Callable, Union +from typing import Any, Optional, Tuple, Callable, Union from cloudinit import features, netinfo from cloudinit import signal_handler @@ -98,18 +98,20 @@ def error(self, message): if not self._raw_args: self._raw_args = sys.argv[1:] subcommand = None + subparsers_action = ( + self._subparsers._group_actions[0] if self._subparsers else None + ) + choices = getattr(subparsers_action, "choices", None) or {} if self._raw_args: for arg in self._raw_args: - if arg in self._subparsers._group_actions[0].choices: + if arg in choices: subcommand = arg break # Check if the subcommand exists and show its help if subcommand: - subparser = self._subparsers._group_actions[0].choices[subcommand] - subparser.print_help( - file=sys.stderr - ) # Print subcommand help to stderr + subparser = choices[subcommand] + subparser.print_help(file=sys.stderr) else: self.print_help(file=sys.stderr) sys.exit(2) @@ -159,15 +161,12 @@ def close_stdin(logger: Callable[[str], None] = LOG.debug): def extract_fns(args): - # Files are already opened so lets just pass that along - # since it would of broke if it couldn't have - # read that file already... fn_cfgs = [] if args.files: - for fh in args.files: + for filepath in args.files: # The realpath is more useful in logging # so lets resolve to that... - fn_cfgs.append(os.path.realpath(fh.name)) + fn_cfgs.append(os.path.realpath(filepath)) return fn_cfgs @@ -546,6 +545,13 @@ def main_init(name, args): bring_up_interfaces = _should_bring_up_interfaces(init, args) try: init.fetch(existing=existing) + if init.datasource is None: + LOG.debug( + "[%s] Exiting. datasource is None after fetch," + " cannot continue.", + mode, + ) + return (None, []) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: @@ -613,6 +619,13 @@ def main_init(name, args): ) util.write_file(init.paths.get_runpath(".skip-network"), "") + if init.datasource is None: + LOG.debug( + "[%s] Exiting. datasource is None in local mode," + " cannot check dsmode.", + mode, + ) + return (None, []) if init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s not in local mode.", @@ -912,13 +925,13 @@ def status_wrapper(name, args): "Invalid cloud init mode specified '{0}'".format(mode) ) - nullstatus = { + nullstatus: dict[str, Union[list[Any], dict[str, Any], float, None]] = { "errors": [], "recoverable_errors": {}, "start": None, "finished": None, } - status = { + status: dict[str, Any] = { "v1": { "datasource": None, "init": nullstatus.copy(), @@ -951,10 +964,15 @@ def status_wrapper(name, args): v1[mode]["start"] = float(util.uptime()) handler = next( - filter( - lambda h: isinstance(h, loggers.LogExporter), root_logger.handlers - ) + ( + h + for h in root_logger.handlers + if isinstance(h, loggers.LogExporter) + ), + None, ) + if not isinstance(handler, loggers.LogExporter): + raise RuntimeError("LogExporter handler not found in root logger") preexisting_recoverable_errors = handler.export_logs() # Write status.json prior to running init / module code @@ -1052,7 +1070,7 @@ def _maybe_set_hostname(init, stage, retry_stage): ) if hostname: # meta-data or user-data hostname content try: - cc_set_hostname.handle("set_hostname", init.cfg, cloud, None) + cc_set_hostname.handle("set_hostname", init.cfg, cloud, []) except cc_set_hostname.SetHostnameError as e: LOG.debug( "Failed setting hostname in %s stage. Will" @@ -1130,7 +1148,7 @@ def main(sysv_args=None): action="append", dest="files", help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), + type=str, ) # This is used so that we can know which action is selected + # the functor to use to run this subcommand @@ -1163,7 +1181,7 @@ def main(sysv_args=None): action="append", dest="files", help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), + type=str, ) parser_mod.set_defaults(action=("modules", main_modules)) @@ -1203,7 +1221,7 @@ def main(sysv_args=None): action="append", dest="files", help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), + type=str, ) parser_single.set_defaults(action=("single", main_single)) diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 3fce70f62bb..939b62d071e 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -30,13 +30,6 @@ "ca_cert_config": "/etc/ca-certificates/conf.d/cloud-init.conf", "ca_cert_update_cmd": ["update-ca-bundle"], }, - "fedora": { - "ca_cert_path": "/etc/pki/ca-trust/", - "ca_cert_local_path": "/usr/share/pki/ca-trust-source/", - "ca_cert_filename": "anchors/cloud-init-ca-cert-{cert_index}.crt", - "ca_cert_config": None, - "ca_cert_update_cmd": ["update-ca-trust"], - }, "rhel": { "ca_cert_path": "/etc/pki/ca-trust/", "ca_cert_local_path": "/usr/share/pki/ca-trust-source/", @@ -60,42 +53,28 @@ }, } -for distro in ( - "opensuse-microos", - "opensuse-tumbleweed", - "opensuse-leap", - "sle_hpc", - "sle-micro", - "sles", -): - DISTRO_OVERRIDES[distro] = DISTRO_OVERRIDES["opensuse"] - -for distro in ( - "almalinux", - "centos", - "cloudlinux", - "rocky", -): - DISTRO_OVERRIDES[distro] = DISTRO_OVERRIDES["rhel"] - -distros = [ - "almalinux", +DISTRO_FAMILY = { + "almalinux": "rhel", + "amazon": "rhel", + "centos": "rhel", + "cloudlinux": "rhel", + "fedora": "rhel", + "opensuse-microos": "opensuse", + "opensuse-tumbleweed": "opensuse", + "opensuse-leap": "opensuse", + "rocky": "rhel", + "sle_hpc": "opensuse", + "sle-micro": "opensuse", + "sles": "opensuse", +} + +distros = list(DISTRO_FAMILY.keys()) + [ "aosc", - "centos", - "cloudlinux", "alpine", "debian", - "fedora", "raspberry-pi-os", "rhel", - "rocky", "opensuse", - "opensuse-microos", - "opensuse-tumbleweed", - "opensuse-leap", - "sle_hpc", - "sle-micro", - "sles", "ubuntu", "photon", ] @@ -114,6 +93,8 @@ def _distro_ca_certs_configs(distro_name): @param distro_name: String providing the distro class name. @returns: Dict of distro configurations for ca_cert. """ + if distro_name in DISTRO_FAMILY: + distro_name = DISTRO_FAMILY[distro_name] cfg = DISTRO_OVERRIDES.get(distro_name, DEFAULT_CONFIG) cfg["ca_cert_full_path"] = os.path.join( cfg["ca_cert_local_path"], cfg["ca_cert_filename"] @@ -160,7 +141,7 @@ def disable_default_ca_certs(distro_name, distro_cfg): @param distro_name: String providing the distro class name. @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - if distro_name in ["rhel", "photon"]: + if distro_name in ["rhel", "photon", "amazon"]: remove_default_ca_certs(distro_cfg) elif distro_name in [ "alpine", diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e82ea36b4d3..26c22838365 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -805,7 +805,11 @@ def check_partition_gpt_layout_sfdisk(device, layout): # Use sfdisk's JSON output for reliability prt_cmd = ["sfdisk", "-l", "-J", device] try: - out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) + out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV, rcs=[0, 1]) + # Device has no partition table or other error, return empty list + if not out: + return [] + # Try to parse JSON output ptable = json.loads(out)["partitiontable"] if "partitions" in ptable: partitions = ptable["partitions"] diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index 56f3ebbcd98..42113ae8d4d 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -63,7 +63,7 @@ def configure_usb_gadget(enable: bool) -> bool: "-f", ], capture=False, - timeout=15, + timeout=30, ) return True diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index dc6adcf0482..2c55308cecf 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -9,6 +9,8 @@ import logging import pwd +import time +from contextlib import suppress from cloudinit import subp, util from cloudinit.cloud import Cloud @@ -148,13 +150,16 @@ def import_ssh_ids(ids, user): else: LOG.error("Neither sudo nor doas available! Unable to import SSH ids.") return - LOG.debug("Importing SSH ids for user %s.", user) + retry_ssh_import(cmd, 0.5) - try: + +def retry_ssh_import(cmd: list, delay: float) -> None: + """Retry ssh-import-id once if it exits in error.""" + with suppress(subp.ProcessExecutionError): subp.subp(cmd, capture=False) - except subp.ProcessExecutionError as exc: - util.logexc(LOG, "Failed to run command to import %s SSH ids", user) - raise exc + return + time.sleep(delay) + subp.subp(cmd, capture=False) def is_key_in_nested_dict(config: dict, search_key: str) -> bool: diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 9bfebbfd129..66e71801589 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -21,6 +21,7 @@ "id": "cc_yum_add_repo", "distros": [ "almalinux", + "amazon", "azurelinux", "centos", "cloudlinux", diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 9ed5540d7d7..2703ebd8357 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -1008,8 +1008,9 @@ def create_user(self, name, **kwargs): cloud_keys = kwargs.get("cloud_public_ssh_keys", []) if not cloud_keys: LOG.warning( - "Unable to disable SSH logins for %s given" - " ssh_redirect_user: %s. No cloud public-keys present.", + "Unable to disable SSH logins for %s." + " ssh_redirect_user was set to redirect logins to" + " %s, but no cloud public-keys are present.", name, kwargs["ssh_redirect_user"], ) diff --git a/cloudinit/distros/azurelinux.py b/cloudinit/distros/azurelinux.py index 591b870020e..e3880548de8 100644 --- a/cloudinit/distros/azurelinux.py +++ b/cloudinit/distros/azurelinux.py @@ -6,7 +6,6 @@ import logging -from cloudinit import subp, util from cloudinit.distros import rhel from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE @@ -22,8 +21,6 @@ class Distro(rhel.Distro): - usr_lib_exec = "/usr/lib" - def __init__(self, name, cfg, paths): super().__init__(name, cfg, paths) self.osfamily = "azurelinux" @@ -45,30 +42,3 @@ def __init__(self, name, cfg, paths): "postcmds": "True", }, } - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - if subp.which("dnf"): - LOG.debug("Using DNF for package management") - cmd = ["dnf"] - else: - LOG.debug("Using TDNF for package management") - cmd = ["tdnf"] - # Determines whether or not dnf/tdnf prompts for confirmation - # of critical actions. We don't want to prompt... - cmd.append("-y") - - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - cmd.append(command) - - pkglist = util.expand_package_list("%s-%s", pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - subp.subp(cmd, capture=False) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1936c34dccd..5422d4b1c33 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -1,14 +1,17 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2025 Raspberry Pi Ltd. # # Author: Scott Moser # Author: Juerg Haefliger # Author: Joshua Harlow +# Author: Paul Oberosler # # This file is part of cloud-init. See LICENSE file for license information. import logging import os +import re from typing import List from cloudinit import distros, subp, util @@ -28,6 +31,8 @@ """ LOCALE_CONF_FN = "/etc/default/locale" +LOCALE_GEN_FN = "/etc/locale.gen" +SUPPORTED_FN = "/usr/share/i18n/SUPPORTED" class Distro(distros.Distro): @@ -111,7 +116,7 @@ def apply_locale(self, locale, out_fn=None, keyname="LANG"): if need_regen: regenerate_locale( locale, - out_fn, + self.default_locale, keyname=keyname, install_function=self.install_packages, ) @@ -127,6 +132,7 @@ def apply_locale(self, locale, out_fn=None, keyname="LANG"): update_locale_conf( locale, out_fn, + default_locale=self.default_locale, keyname=keyname, install_function=self.install_packages, ) @@ -278,13 +284,13 @@ def read_system_locale(sys_path=LOCALE_CONF_FN, keyname="LANG"): def update_locale_conf( - locale, sys_path, keyname="LANG", install_function=None + locale, sys_path, default_locale, keyname="LANG", install_function=None ): """Update system locale config""" LOG.debug( "Updating %s with locale setting %s=%s", sys_path, keyname, locale ) - if not subp.which("update-locale"): + if not subp.which("update-locale") and install_function: install_function(["locales"]) subp.subp( [ @@ -292,27 +298,144 @@ def update_locale_conf( "--locale-file=" + sys_path, "%s=%s" % (keyname, locale), ], + update_env={ + "LANGUAGE": default_locale, + "LANG": default_locale, + "LC_ALL": default_locale, + }, capture=False, ) -def regenerate_locale(locale, sys_path, keyname="LANG", install_function=None): +def _lookup_supported_i18n_value(requested: str) -> str: """ - Run locale-gen for the provided locale and set the default - system variable `keyname` appropriately in the provided `sys_path`. + Return the canonical line from /usr/share/i18n/SUPPORTED for `requested`. + + Accepts: + - bare language_region: "fi_FI" + - with charset: "fi_FI.ISO-8859-1" or "fi_FI.UTF-8" + - with modifier: "fi_FI@euro" (works with/without charset) + Prefers UTF-8 only when the request didn’t specify a charset and multiple + candidates exist; otherwise returns the first match. + """ + try: + supported_lines = util.load_text_file(SUPPORTED_FN).splitlines() + except OSError: + supported_lines = [] + + # Parse requested into locale[.charset][@mod] + m = re.match( + r"^([A-Za-z_]+)(?:\.([A-Za-z0-9\-]+))?(?:@([A-Za-z0-9_\-]+))?$", + requested, + ) + if not m: + # fallback: treat whole string as a prefix + prefix = requested + wanted_charset = None + wanted_mod = None + else: + base, wanted_charset, wanted_mod = m.group(1), m.group(2), m.group(3) + prefix = base + (f"@{wanted_mod}" if wanted_mod else "") + + # Collect candidates that start with requested locale (+modifier), + # each SUPPORTED line is "locale[.charset][@mod] CHARMAP" + candidates = [] + rx = re.compile( + rf"^\s*{re.escape(prefix)}(?:\.[^\s@]+)?(?:@[^\s]+)?\s+[^\s]+$" + ) + for line in supported_lines: + if rx.match(line): + candidates.append(line.strip()) + + if not candidates: + # As a last resort, construct a reasonable default (don’t force UTF-8) + # If user gave a charset, use it; else use UTF-8. + if not wanted_charset: + wanted_charset = "UTF-8" + return f"{prefix}.{wanted_charset} {wanted_charset}" + + if wanted_charset: + # Find exact charset match on first field (before space) + rx_exact = re.compile( + rf"^\s*{re.escape(prefix)}\.{re.escape(wanted_charset)}(?:@[^\s]+)?\s+" + ) + for line in candidates: + if rx_exact.match(line): + return line + + # No explicit charset requested: prefer UTF-8 if + # present, else first candidate + for line in candidates: + if re.search(r"\sUTF-8$", line, re.IGNORECASE): + return line + return candidates[0] + + +def regenerate_locale( + locale, default_locale, keyname="LANG", install_function=None +): """ + Ensure `locale` is enabled in /etc/locale.gen, then run locale-gen. + Debian's locale-gen reads /etc/locale.gen and ignores positional args. + """ + # special case for locales which do not require regen # % locale -a # C # C.UTF-8 # POSIX - if locale.lower() in ["c", "c.utf-8", "posix"]: + if locale.lower() in ["c", "c.utf-8", "c.utf8", "posix"]: LOG.debug("%s=%s does not require rengeneration", keyname, locale) return - # finally, trigger regeneration - if not subp.which("locale-gen"): + if not subp.which("locale-gen") and install_function: install_function(["locales"]) + + # compute canonical line and NEW_LANG (first field) + line = _lookup_supported_i18n_value(locale) + + # ensure /etc/locale.gen contains the + # line (uncomment if present; append if absent) + existing = "" + try: + existing = util.load_text_file(LOCALE_GEN_FN) + except OSError: + existing = "" + + out_lines = [] + found_enabled = False + target_re = re.compile(rf"^#?\s*{re.escape(line)}\s*$") + + for raw in existing.splitlines(): + s = raw.strip() + if not s: + out_lines.append(raw) + continue + + if target_re.match(s.lstrip("# ")): + # enable target locale + out_lines.append(line) + found_enabled = True + else: + # keep all other locales as-is (don't disable them) + out_lines.append(raw) + + if not found_enabled: + out_lines.append(line) + + util.ensure_dir(os.path.dirname(LOCALE_GEN_FN)) + util.write_file(LOCALE_GEN_FN, "\n".join(out_lines).rstrip() + "\n") + + # finally, generate locales listed in /etc/locale.gen LOG.debug("Generating locales for %s", locale) - subp.subp(["locale-gen", locale], capture=False) + # Using --keep-existing to avoid removing already generated locales + subp.subp( + ["locale-gen", "--keep-existing"], + capture=False, + update_env={ + "LANGUAGE": default_locale, + "LANG": default_locale, + "LC_ALL": default_locale, + }, + ) diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py index 14cf3be2b8e..cce1073c408 100644 --- a/cloudinit/distros/openbsd.py +++ b/cloudinit/distros/openbsd.py @@ -13,6 +13,7 @@ class Distro(cloudinit.distros.netbsd.NetBSD): hostname_conf_fn = "/etc/myname" init_cmd = ["rcctl"] + usr_lib_exec = "/usr/local/libexec" # For OpenBSD (from https://man.openbsd.org/passwd.5) a password field # of "" indicates no password, and password field values of either diff --git a/cloudinit/distros/parsers/hosts.py b/cloudinit/distros/parsers/hosts.py index 8d2f73ac91f..2528264d1e8 100644 --- a/cloudinit/distros/parsers/hosts.py +++ b/cloudinit/distros/parsers/hosts.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from io import StringIO +from typing import Any, List, Tuple from cloudinit.distros.parsers import chop_comment @@ -13,69 +14,71 @@ # or https://linux.die.net/man/5/hosts # or https://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook/configtuning-configfiles.html # noqa class HostsConf: - def __init__(self, text): + def __init__(self, text: str) -> None: self._text = text - self._contents = None + self._contents: List[Tuple[str, List[Any]]] = [] - def parse(self): - if self._contents is None: + def parse(self) -> None: + if not self._contents: self._contents = self._parse(self._text) - def get_entry(self, ip): + def get_entry(self, ip: str) -> List[List[str]]: self.parse() - options = [] + options: List[List[str]] = [] for line_type, components in self._contents: if line_type == "option": - (pieces, _tail) = components + pieces, _tail = components if len(pieces) and pieces[0] == ip: options.append(pieces[1:]) return options - def del_entries(self, ip): + def del_entries(self, ip: str) -> None: self.parse() - n_entries = [] + n_entries: List[Tuple[str, List[Any]]] = [] for line_type, components in self._contents: if line_type != "option": n_entries.append((line_type, components)) continue else: - (pieces, _tail) = components + pieces, _tail = components if len(pieces) and pieces[0] == ip: pass elif len(pieces): n_entries.append((line_type, list(components))) self._contents = n_entries - def add_entry(self, ip, canonical_hostname, *aliases): + def add_entry( + self, ip: str, canonical_hostname: str, *aliases: str + ) -> None: self.parse() self._contents.append( - ("option", ([ip, canonical_hostname] + list(aliases), "")) + ("option", [[ip, canonical_hostname] + list(aliases), ""]) ) - def _parse(self, contents): - entries = [] + def _parse(self, contents: str) -> List[Tuple[str, List[Any]]]: + entries: List[Tuple[str, List[Any]]] = [] for line in contents.splitlines(): if not len(line.strip()): entries.append(("blank", [line])) continue - (head, tail) = chop_comment(line.strip(), "#") + head, tail = chop_comment(line.strip(), "#") if not len(head): entries.append(("all_comment", [line])) continue entries.append(("option", [head.split(None), tail])) return entries - def __str__(self): + def __str__(self) -> str: self.parse() contents = StringIO() for line_type, components in self._contents: if line_type == "blank": - contents.write("%s\n" % (components[0])) + contents.write("%s\n" % components[0]) elif line_type == "all_comment": - contents.write("%s\n" % (components[0])) + contents.write("%s\n" % components[0]) elif line_type == "option": - (pieces, tail) = components - pieces = [str(p) for p in pieces] - pieces = "\t".join(pieces) - contents.write("%s%s\n" % (pieces, tail)) + raw_pieces, tail = components + str_pieces = [str(p) for p in raw_pieces] + joined = "\t".join(str_pieces) + contents.write(f"{joined}{tail}\n") return contents.getvalue() diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py index 2d0a887e7c4..cd50d3b45dd 100644 --- a/cloudinit/distros/ug_util.py +++ b/cloudinit/distros/ug_util.py @@ -134,13 +134,14 @@ def _normalize_users(u_cfg, def_user_cfg=None): # Now merge the extracted groups with the default config provided users_groups = util.uniq_merge_sorted(parsed_groups, def_groups) parsed_config["groups"] = ",".join(users_groups) - # The real config for the default user is the combination of the - # default user config provided by the distro, the default user - # config provided by the above merging for the user 'default' and - # then the parsed config from the user's 'real name' which does not - # have to be 'default' (but could be) + # The real config for the default user is the combination of: + # - the parsed config from the user's 'real name' which does + # not have to be 'default' (but could be) + # - then the default user config provided by the above merging + # for the user 'default' + # - then the default user config provided by the distro users[def_user] = util.mergemanydict( - [def_user_cfg, def_config, parsed_config] + [parsed_config, def_config, def_user_cfg] ) # Ensure that only the default user that we found (if any) is actually diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index 388588d8029..b4400304786 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -5,7 +5,9 @@ import os import re from errno import EACCES -from typing import Optional, Type +from typing import Optional + +from jinja2 import exceptions, lexer from cloudinit import handlers from cloudinit.atomic_helper import b64d, json_dumps @@ -19,15 +21,6 @@ ) from cloudinit.util import load_json, load_text_file -JUndefinedError: Type[Exception] -try: - from jinja2.exceptions import UndefinedError as JUndefinedError - from jinja2.lexer import operator_re -except ImportError: - # No jinja2 dependency - JUndefinedError = Exception - operator_re = re.compile(r"[-.]") - LOG = logging.getLogger(__name__) @@ -147,7 +140,7 @@ def render_jinja_payload(payload, payload_fn, instance_data, debug=False): ) try: rendered_payload = render_string(payload, instance_jinja_vars) - except (TypeError, JUndefinedError) as e: + except (TypeError, exceptions.UndefinedError) as e: LOG.warning("Ignoring jinja template for %s: %s", payload_fn, str(e)) return None warnings = [ @@ -180,7 +173,7 @@ def get_jinja_variable_alias(orig_name: str) -> Optional[str]: :return: A string with any jinja operators replaced if needed. Otherwise, none if no alias required. """ - alias_name = re.sub(operator_re, "_", orig_name) + alias_name = re.sub(lexer.operator_re, "_", orig_name) if alias_name != orig_name: return alias_name return None diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index c2c1d5af45b..784f7baad09 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -63,9 +63,14 @@ def __init__(self, _files=None, _mac_addrs=None, _cmdline=None): self._mac_addrs[k] = mac_addr def is_applicable(self) -> bool: - """ - Return whether this system has klibc initramfs network config or not + """Return whether this system has klibc initramfs network config.""" + + if is_applicable := self._is_applicable(): + LOG.debug("Using initramfs network config from klibc") + return is_applicable + def _is_applicable(self) -> bool: + """ Will return True if: (a) klibc files exist in /run, AND (b) either: diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 6620b15da26..95a94fcfa5e 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -29,6 +29,7 @@ DHCLIENT_FALLBACK_LEASE_DIR = "/var/lib/dhclient" # Match something.lease or something.leases DHCLIENT_FALLBACK_LEASE_REGEX = r".+\.leases?$" +NMCLI = "nmcli" UDHCPC_SCRIPT = """#!/bin/sh log() { echo "udhcpc[$PPID]" "$interface: $2" @@ -147,6 +148,80 @@ def networkd_get_option_from_leases(keyname, leases_d=None): return None +def run_nmcli(nmcli_opts: List[str]) -> str: + nmcli_path = subp.which(NMCLI) + if not nmcli_path: + raise NoDHCPLeaseMissingDhclientError() + + command = [nmcli_path] + nmcli_opts + try: + out, _ = subp.subp( + command, + ) + except subp.ProcessExecutionError as error: + LOG.debug( + "nmcli command exited with code: %s stderr: %r stdout: %r", + error.exit_code, + error.stderr, + error.stdout, + ) + raise NoDHCPLeaseError from error + return out + + +def network_manager_load_leases(device: str) -> Dict[str, str]: + """Return a dictionary of lease options obtained from NM cli""" + + opts = ["--fields", "DHCP4", "device", "show", device] + nmcli_out = run_nmcli(opts) + + content = [] + for line in nmcli_out.splitlines(): + line = line.partition(":")[2].strip() + content.append(line) + + return dict(configobj.ConfigObj(content, list_values=False)) + + +def find_correct_device_nmcli() -> Optional[str]: + """Return the lease value for + the first connected device as returned by 'nmcli'""" + + device_list_opts = ["--terse", "device"] + nmcli_out = run_nmcli(device_list_opts) + + for line in nmcli_out.splitlines(): + if line == "": + continue + try: + dev_name, _, state, _ = line.split(":", 3) + except ValueError: + LOG.warning( + "Unexpected nmcli format: expected 4 colon-delimited" + " values, found %s", + line, + ) + continue + + # skip devices that are not connected + if state != "connected": + continue + # skip loopback + if dev_name == "lo": + continue + return dev_name + return None + + +def network_manager_get_option_from_leases(keyname: str) -> Optional[str]: + leases = None + dev = find_correct_device_nmcli() + if dev: + leases = network_manager_load_leases(dev) + + return leases.get(keyname) if leases else None + + class DhcpClient(abc.ABC): client_name = "" timeout = 10 diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 89292597145..fb8bc1dfb69 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -7,7 +7,7 @@ import os import re from contextlib import suppress -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Mapping, Optional from cloudinit import performance, subp, util from cloudinit.net import ( @@ -453,7 +453,7 @@ def has_same_ip_version(addr_or_net: str, is_ipv6: bool) -> bool: class Renderer(renderer.Renderer): """Renders network information in a /etc/network/interfaces format.""" - def __init__(self, config: Optional[dict] = None): + def __init__(self, config: Optional[Mapping[str, Any]] = None): if not config: config = {} self.eni_path = config.get("eni_path", "etc/network/interfaces") diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 261993fe1ce..87b4ab1fbe3 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -5,7 +5,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 -import functools import logging import os import os.path @@ -50,31 +49,6 @@ ) from cloudinit.url_helper import UrlError -try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - import crypt # pylint: disable=W4901 - - blowfish_hash: Any = functools.partial( - crypt.crypt, salt=f"$6${util.rand_str(strlen=16)}" - ) -except (ImportError, AttributeError): - try: - import passlib.hash - - blowfish_hash = passlib.hash.sha512_crypt.hash - except ImportError: - - def blowfish_hash(_): - """Raise when called so that importing this module doesn't throw - ImportError when ds_detect() returns false. In this case, crypt - and passlib are not needed. - """ - raise ImportError( - "crypt and passlib not found, missing dependency" - ) - - LOG = logging.getLogger(__name__) DS_NAME = "Azure" @@ -166,6 +140,35 @@ def find_dev_from_busdev(camcontrol_out: str, busdev: str) -> Optional[str]: return None +def hash_password(password: str) -> str: + """Hash a password using SHA-512 crypt. + + Try to use crypt, falling back to passlib. + + If neither are available, raise ReportableErrorImportError. + + :param password: plaintext password to hash. + :return: The hashed password string. + :raises ReportableErrorImportError: If crypt and passlib are unavailable. + """ + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + import crypt # pylint: disable=W4901 + + salt = crypt.mksalt(crypt.METHOD_SHA512) + return crypt.crypt(password, salt) + except (ImportError, AttributeError): + pass + + try: + import passlib.hash + + return passlib.hash.sha512_crypt.hash(password) + except ImportError as error: + raise errors.ReportableErrorImportError(error=error) from error + + def normalize_mac_address(mac: str) -> str: """Normalize mac address with colons and lower-case.""" if len(mac) == 12: @@ -292,6 +295,9 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: "disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH}, "apply_network_config": True, # Use IMDS published network configuration "apply_network_config_for_secondary_ips": True, # Configure secondary ips + "experimental_fail_on_missing_customdata": False, + "apply_network_config_set_name": True, # Use set-name for NICs + "experimental_skip_ready_report": False, # Skip final ready report } BUILTIN_CLOUD_EPHEMERAL_DISK_CONFIG = { @@ -358,6 +364,13 @@ def _unpickle(self, ci_pkl_version: int) -> None: self._system_uuid = None self._vm_id = None self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT + for key in ( + "apply_network_config_for_secondary_ips", + "experimental_fail_on_missing_customdata", + "apply_network_config_set_name", + "experimental_skip_ready_report", + ): + self.ds_cfg.setdefault(key, BUILTIN_DS_CONFIG[key]) def __str__(self): root = sources.DataSource.__str__(self) @@ -658,7 +671,6 @@ def crawl_metadata(self): # it determines the value of ret. More specifically, the first one in # the candidate list determines the path to take in order to get the # metadata we need. - ovf_source = None md = {"local-hostname": ""} cfg = {"system_info": {"default_user": {"name": ""}}} userdata_raw = "" @@ -680,9 +692,9 @@ def crawl_metadata(self): else: md, userdata_raw, cfg, files = load_azure_ds_dir(src) - ovf_source = src + self.seed = src report_diagnostic_event( - "Found provisioning metadata in %s" % ovf_source, + "Found provisioning metadata in %s" % self.seed, logger_func=LOG.debug, ) break @@ -710,7 +722,7 @@ def crawl_metadata(self): # not have UDF support. In either case, require IMDS metadata. # If we require IMDS metadata, try harder to obtain networking, waiting # for at least 20 minutes. Otherwise only wait 5 minutes. - requires_imds_metadata = bool(self._iso_dev) or ovf_source is None + requires_imds_metadata = bool(self._iso_dev) or self.seed is None timeout_minutes = 20 if requires_imds_metadata else 5 try: self._setup_ephemeral_networking(timeout_minutes=timeout_minutes) @@ -726,14 +738,18 @@ def crawl_metadata(self): imds_md = self.get_metadata_from_imds(report_failure=True) - if not imds_md and ovf_source is None: + if not imds_md and self.seed is None: msg = "No OVF or IMDS available" report_diagnostic_event(msg) raise sources.InvalidMetaDataException(msg) + self.seed = self.seed or "IMDS" + # Refresh PPS type using metadata. pps_type = self._determine_pps_type(cfg, imds_md) if pps_type != PPSType.NONE: + self.seed = "IMDS" + if util.is_FreeBSD(): msg = "Free BSD is not supported for PPS VMs" report_diagnostic_event(msg, logger_func=LOG.error) @@ -773,7 +789,6 @@ def crawl_metadata(self): # Report errors if IMDS network configuration is missing data. self.validate_imds_network_metadata(imds_md=imds_md) - self.seed = ovf_source or "IMDS" crawled_data.update( { "cfg": cfg, @@ -812,9 +827,31 @@ def crawl_metadata(self): logger_func=LOG.debug, ) - # only use userdata from imds if OVF did not provide custom data - # userdata provided by IMDS is always base64 encoded + # Only use userdata from IMDS if OVF did not provide custom data. + # Userdata provided by IMDS is always base64 encoded. if not userdata_raw: + # First, check to see if the OVF was supposed to provide custom + # data. If it was supposed to and did not, we report failure. + has_custom_data = _hascustomdata_from_imds(imds_md) + if has_custom_data: + if self.ds_cfg.get("experimental_fail_on_missing_customdata"): + self._report_failure( + errors.ReportableErrorMissingCustomData( + pps_type=pps_type.value, + provisioning_media=self.seed, + ) + ) + else: + report_diagnostic_event( + "Did not find custom data in %s, IMDS returned" + " extended.compute.hasCustomData=%r" + % ( + self.seed, + has_custom_data, + ), + logger_func=LOG.error, + ) + imds_userdata = _userdata_from_imds(imds_md) if imds_userdata: LOG.debug("Retrieved userdata from IMDS") @@ -827,7 +864,7 @@ def crawl_metadata(self): "Bad userdata in IMDS", logger_func=LOG.warning ) - if ovf_source == ddir: + if self.seed == ddir: report_diagnostic_event( "using files cached in %s" % ddir, logger_func=LOG.debug ) @@ -838,21 +875,28 @@ def crawl_metadata(self): crawled_data["metadata"]["instance-id"] = self._iid() if self._negotiated is False and self._is_ephemeral_networking_up(): - # Report ready and fetch public-keys from Wireserver, if required. - pubkey_info = self._determine_wireserver_pubkey_info( - cfg=cfg, imds_md=imds_md - ) - try: - ssh_keys = self._report_ready(pubkey_info=pubkey_info) - except Exception: - # Failed to report ready, but continue with best effort. - pass + if self.ds_cfg.get("experimental_skip_ready_report", False): + LOG.debug( + "Skipping final health report as " + "experimental_skip_ready_report is enabled." + ) else: - LOG.debug("negotiating returned %s", ssh_keys) - if ssh_keys: - crawled_data["metadata"]["public-keys"] = ssh_keys + # Report ready and fetch public-keys from Wireserver, + # if required. + pubkey_info = self._determine_wireserver_pubkey_info( + cfg=cfg, imds_md=imds_md + ) + try: + ssh_keys = self._report_ready(pubkey_info=pubkey_info) + except Exception: + # Failed to report ready, but continue with best effort. + pass + else: + LOG.debug("negotiating returned %s", ssh_keys) + if ssh_keys: + crawled_data["metadata"]["public-keys"] = ssh_keys - self._cleanup_markers() + self._cleanup_markers() return crawled_data @@ -1605,9 +1649,12 @@ def _generate_network_config(self): try: return generate_network_config_from_instance_network_metadata( self._metadata_imds["network"], - apply_network_config_for_secondary_ips=self.ds_cfg.get( + apply_network_config_for_secondary_ips=self.ds_cfg[ "apply_network_config_for_secondary_ips" - ), + ], + apply_network_config_set_name=self.ds_cfg[ + "apply_network_config_set_name" + ], ) except Exception as e: LOG.error( @@ -1713,6 +1760,13 @@ def _userdata_from_imds(imds_data): return None +def _hascustomdata_from_imds(imds_data: Dict) -> Optional[bool]: + try: + return imds_data["extended"]["compute"]["hasCustomData"] + except KeyError: + return None + + def _hostname_from_imds(imds_data): try: return imds_data["compute"]["osProfile"]["computerName"] @@ -1984,7 +2038,7 @@ def read_azure_ovf(contents): if ovf_env.password: defuser["lock_passwd"] = False if DEF_PASSWD_REDACTION != ovf_env.password: - defuser["hashed_passwd"] = encrypt_pass(ovf_env.password) + defuser["hashed_passwd"] = hash_password(ovf_env.password) if defuser: cfg["system_info"] = {"default_user": defuser} @@ -2009,10 +2063,6 @@ def read_azure_ovf(contents): return (md, ud, cfg) -def encrypt_pass(password): - return blowfish_hash(password) - - def find_primary_nic(): candidate_nics = net.find_candidate_nics() if candidate_nics: @@ -2088,6 +2138,7 @@ def generate_network_config_from_instance_network_metadata( network_metadata: dict, *, apply_network_config_for_secondary_ips: bool, + apply_network_config_set_name: bool, ) -> dict: """Convert imds network metadata dictionary to network v2 configuration. @@ -2101,7 +2152,11 @@ def generate_network_config_from_instance_network_metadata( # First IPv4 and/or IPv6 address will be obtained via DHCP. # Any additional IPs of each type will be set as static # addresses. - nicname = "eth{idx}".format(idx=idx) + mac = normalize_mac_address(intf["macAddress"]) + if apply_network_config_set_name: + nicname = "eth{idx}".format(idx=idx) + else: + nicname = "enx{mac}".format(mac=mac.replace(":", "")) dhcp_override = {"route-metric": (idx + 1) * 100} # DNS resolution through secondary NICs is not supported, disable it. if idx > 0: @@ -2145,10 +2200,9 @@ def generate_network_config_from_instance_network_metadata( "{ip}/{prefix}".format(ip=privateIp, prefix=netPrefix) ) if dev_config and has_ip_address: - mac = normalize_mac_address(intf["macAddress"]) - dev_config.update( - {"match": {"macaddress": mac.lower()}, "set-name": nicname} - ) + dev_config["match"] = {"macaddress": mac.lower()} + if apply_network_config_set_name: + dev_config["set-name"] = nicname driver = determine_device_driver_for_mac(mac) if driver: dev_config["match"]["driver"] = driver diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 265727cd60c..f5fa4c8e443 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -90,6 +90,7 @@ def _get_domainname(self): """Try obtaining a "domain-name" DHCP lease parameter: - From systemd-networkd lease (case-insensitive) - From ISC dhclient + - From network manager dhcp client - From dhcpcd (ephemeral) - Return empty string if not found (non-fatal) """ @@ -113,7 +114,19 @@ def _get_domainname(self): LOG.debug( "Could not obtain FQDN from ISC dhclient leases. Falling back to " - "%s", + "Network Manager leases" + ) + with suppress( + dhcp.NoDHCPLeaseMissingDhclientError, dhcp.NoDHCPLeaseError + ): + domain_name = dhcp.network_manager_get_option_from_leases( + "domain_name" + ) + if domain_name: + return domain_name.strip() + + LOG.debug( + "Could not obtain FQDN from NM leases. Falling back to %s", self.distro.dhcp_client.client_name, ) try: @@ -338,6 +351,15 @@ def get_vr_address(distro): LOG.debug("Found SERVER_ADDRESS '%s' via dhclient", latest_address) return latest_address + # try network manager DHCP lease information + with suppress(dhcp.NoDHCPLeaseMissingDhclientError, dhcp.NoDHCPLeaseError): + latest_address = dhcp.network_manager_get_option_from_leases( + "dhcp_server_identifier" + ) + if latest_address: + LOG.debug("Found SERVER_ADDRESS '%s' via nmcli", latest_address) + return latest_address + with suppress(FileNotFoundError): latest_lease = distro.dhcp_client.get_newest_lease( distro.fallback_interface diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 5ad3b6bc6c1..16b5f82c07c 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -20,6 +20,7 @@ import re import shlex import textwrap +from typing import Any, Dict, List, Optional, overload from cloudinit import atomic_helper, net, sources, subp, util @@ -56,8 +57,8 @@ def __str__(self): def _get_data(self): defaults = {"instance-id": DEFAULT_IID} - results = None - seed = None + results: Optional[Dict[str, Any]] = None + seed: Optional[str] = None # decide parseuser for context.sh shell reader parseuser = DEFAULT_PARSEUSER @@ -94,7 +95,7 @@ def _get_data(self): LOG.debug("found datasource in %s", cdev) break - if not seed: + if not seed or results is None: return False # merge fetched metadata with datasource defaults @@ -115,8 +116,10 @@ def _get_data(self): self.userdata_raw = results.get("userdata") return True - def _get_subplatform(self): + def _get_subplatform(self) -> str: """Return the subplatform metadata source details.""" + if self.seed is None: + return sources.METADATA_UNKNOWN if self.seed_dir in self.seed: subplatform_type = "seed-dir" else: @@ -148,50 +151,68 @@ class BrokenContextDiskDir(Exception): class OpenNebulaNetwork: - def __init__(self, context, distro, system_nics_by_mac=None): + def __init__( + self, + context: Dict[str, str], + distro: Any, + system_nics_by_mac: Optional[Dict[str, str]] = None, + ) -> None: self.context = context if system_nics_by_mac is None: system_nics_by_mac = get_physical_nics_by_mac(distro) - self.ifaces = collections.OrderedDict( - [ - k - for k in sorted( - system_nics_by_mac.items(), - key=lambda k: net.natural_sort_key(k[1]), - ) - ] + self.ifaces: collections.OrderedDict[str, str] = ( + collections.OrderedDict( + [ + k + for k in sorted( + system_nics_by_mac.items(), + key=lambda k: net.natural_sort_key(k[1]), + ) + ] + ) ) # OpenNebula 4.14+ provide macaddr for ETHX in variable ETH_MAC. # context_devname provides {mac.lower():ETHX, mac2.lower():ETHX} - self.context_devname = {} + self.context_devname: Dict[str, str] = {} for k, v in context.items(): m = re.match(r"^(.+)_MAC$", k) if m: self.context_devname[v.lower()] = m.group(1) - def mac2ip(self, mac): + def mac2ip(self, mac: str) -> str: return ".".join([str(int(c, 16)) for c in mac.split(":")[2:]]) - def get_nameservers(self, dev): - nameservers = {} - dns = self.get_field(dev, "dns", "").split() - dns.extend(self.context.get("DNS", "").split()) + def get_nameservers(self, dev: str) -> Dict[str, List[str]]: + nameservers: Dict[str, List[str]] = {} + dns: List[str] = [] + for server in ( + self.get_field(dev, "dns", "").split() + + self.context.get("DNS", "").split() + ): + if server not in dns: + dns.append(server) if dns: nameservers["addresses"] = dns - search_domain = self.get_field(dev, "search_domain", "").split() + search_domain: List[str] = [] + for domain in ( + self.get_field(dev, "search_domain", "").split() + + self.context.get("SEARCH_DOMAIN", "").split() + ): + if domain not in search_domain: + search_domain.append(domain) if search_domain: nameservers["search"] = search_domain return nameservers - def get_mtu(self, dev): + def get_mtu(self, dev: str) -> Optional[str]: return self.get_field(dev, "mtu") - def get_ip(self, dev, mac): + def get_ip(self, dev: str, mac: str) -> str: return self.get_field(dev, "ip", self.mac2ip(mac)) - def get_ip6(self, dev): - addresses6 = [] + def get_ip6(self, dev: str) -> List[str]: + addresses6: List[str] = [] ip6 = self.get_field(dev, "ip6") if ip6: addresses6.append(ip6) @@ -200,24 +221,59 @@ def get_ip6(self, dev): addresses6.append(ip6_ula) return addresses6 - def get_ip6_prefix(self, dev): + def get_ip6_prefix(self, dev: str) -> str: return self.get_field(dev, "ip6_prefix_length", "64") - def get_gateway(self, dev): + def get_gateway(self, dev: str) -> Optional[str]: return self.get_field(dev, "gateway") - def get_gateway6(self, dev): + def get_gateway6(self, dev: str) -> Optional[str]: # OpenNebula 6.1.80 introduced new context parameter ETHx_IP6_GATEWAY # to replace old ETHx_GATEWAY6. Old ETHx_GATEWAY6 will be removed in # OpenNebula 6.4.0 (https://github.com/OpenNebula/one/issues/5536). - return self.get_field( - dev, "ip6_gateway", self.get_field(dev, "gateway6") - ) + ip6_gateway = self.get_field(dev, "ip6_gateway") + if ip6_gateway is not None: + return ip6_gateway + return self.get_field(dev, "gateway6") - def get_mask(self, dev): + def get_mask(self, dev: str) -> str: return self.get_field(dev, "mask", "255.255.255.0") - def get_field(self, dev, name, default=None): + def get_routes(self, dev: str) -> List[Dict[str, str]]: + """Parse ETHx_ROUTES into a list of Netplan route dicts. + + Expected format: "NETWORK via GATEWAY[, NETWORK via GATEWAY, ...]" + e.g. "10.0.0.0/8 via 192.168.1.1, 192.168.100.0/24 via 10.0.0.1" + Returns an empty list when the variable is absent or empty. + """ + routes: List[Dict[str, str]] = [] + for entry in self.get_field(dev, "routes", "").split(","): + entry = entry.strip() + if not entry: + continue + m = re.match( + r"\s*(?P\S+)\s+via\s+(?P\S+)\s*$", + entry, + ) + if m: + routes.append({"to": m["route_to"], "via": m["route_via"]}) + else: + LOG.warning( + "Unparseable ETHx_ROUTES entry for %s: %r", dev, entry + ) + return routes + + @overload + def get_field(self, dev: str, name: str) -> Optional[str]: ... + @overload + def get_field( + self, dev: str, name: str, default: None + ) -> Optional[str]: ... + @overload + def get_field(self, dev: str, name: str, default: str) -> str: ... + def get_field( + self, dev: str, name: str, default: Optional[str] = None + ) -> Optional[str]: """return the field name in context for device dev. context stores _ (example: eth0_DOMAIN). @@ -233,12 +289,10 @@ def get_field(self, dev, name, default=None): # allow empty string to return the default. return default if val in (None, "") else val - def gen_conf(self): - netconf = {} - netconf["version"] = 2 - netconf["ethernets"] = {} + def gen_conf(self) -> Dict[str, Any]: + netconf: Dict[str, Any] = {"version": 2, "ethernets": {}} - ethernets = {} + ethernets: Dict[str, Dict[str, Any]] = {} for mac, dev in self.ifaces.items(): mac = mac.lower() @@ -246,7 +300,7 @@ def gen_conf(self): # dev stores the current system name. c_dev = self.context_devname.get(mac, dev) - devconf = {} + devconf: Dict[str, Any] = {} # Set MAC address devconf["match"] = {"macaddress": mac} @@ -283,7 +337,12 @@ def gen_conf(self): # Set MTU size mtu = self.get_mtu(c_dev) if mtu: - devconf["mtu"] = mtu + devconf["mtu"] = int(mtu) + + # Set static routes + extra_routes: List[Dict[str, str]] = self.get_routes(c_dev) + if extra_routes: + devconf["routes"] = extra_routes ethernets[dev] = devconf @@ -323,7 +382,9 @@ def varprinter(): ) -def parse_shell_config(content, asuser=None): +def parse_shell_config( + content: str, asuser: Optional[str] = None +) -> Dict[str, str]: """run content and return environment variables which changed WARNING: the special variable _start_ is used to delimit content @@ -394,13 +455,19 @@ def parse_shell_config(content, asuser=None): return ret -def read_context_disk_dir(source_dir, distro, asuser=None): - """ - read_context_disk_dir(source_dir): - read source_dir and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a NonContextDiskDir +def read_context_disk_dir( + source_dir: str, distro: Any, asuser: Optional[str] = None +) -> Dict[str, Any]: + """Read ``source_dir`` and return a dictionary containing context data. + + The returned dictionary always includes ``"metadata"`` and + ``"userdata"`` keys, and may also include ``"network-interfaces"`` + when network configuration can be generated from the context. + + If ``source_dir`` is not a valid context directory, raise + ``NonContextDiskDir``. """ - found = {} + found: Dict[str, str] = {} for af in CONTEXT_DISK_FILES: fn = os.path.join(source_dir, af) if os.path.isfile(fn): @@ -409,8 +476,8 @@ def read_context_disk_dir(source_dir, distro, asuser=None): if not found: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) - context = {} - results = {"userdata": None, "metadata": {}} + context: Dict[str, str] = {} + results: Dict[str, Any] = {"userdata": None, "metadata": {}} if "context.sh" in found: if asuser is not None: @@ -450,7 +517,7 @@ def read_context_disk_dir(source_dir, distro, asuser=None): ssh_key_var = "SSH_PUBLIC_KEY" if ssh_key_var: - lines = context.get(ssh_key_var).splitlines() + lines = context[ssh_key_var].splitlines() results["metadata"]["public-keys"] = [ line for line in lines if len(line) and not line.startswith("#") ] @@ -490,7 +557,7 @@ def read_context_disk_dir(source_dir, distro, asuser=None): return results -def get_physical_nics_by_mac(distro): +def get_physical_nics_by_mac(distro: Any) -> Dict[str, str]: devs = net.get_interfaces_by_mac() return dict( [(m, n) for m, n in devs.items() if distro.networking.is_physical(n)] diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py index e74e1cbad15..4a846a897b8 100644 --- a/cloudinit/sources/DataSourceOracle.py +++ b/cloudinit/sources/DataSourceOracle.py @@ -63,8 +63,8 @@ class KlibcOracleNetworkConfigSource(cmdline.KlibcNetworkConfigSource): `/run/initramfs/open-iscsi.interface` does not exist. """ - def is_applicable(self) -> bool: - """Override is_applicable""" + def _is_applicable(self) -> bool: + """Override _is_applicable""" return bool(self._files) diff --git a/cloudinit/sources/DataSourceWSL.py b/cloudinit/sources/DataSourceWSL.py index 222ec2a0f63..607f8686368 100644 --- a/cloudinit/sources/DataSourceWSL.py +++ b/cloudinit/sources/DataSourceWSL.py @@ -90,6 +90,7 @@ def find_home() -> PurePath: raises: IOError when no mountpoint with cmd.exe is found ProcessExecutionError when either cmd.exe is unable to retrieve the user's home directory + UnicodeDecodeError when cmd.exe /U outputs invalid UTF16LE """ cmd = cmd_executable() @@ -97,8 +98,13 @@ def find_home() -> PurePath: # But we know that `/init` is the interpreter, so we can run it directly. # See /proc/sys/fs/binfmt_misc/WSLInterop[-late] # inside any WSL instance for more details. - home, _ = subp.subp(["/init", cmd.as_posix(), "/C", "echo %USERPROFILE%"]) - home = home.rstrip() + # Invoking with "/U" makes it output UTF-16LE, which is more predictable + # than ANSI Code Pages for anything above the ASCII range. + home, _ = subp.subp( + ["/init", cmd.as_posix(), "/U", "/C", "echo.%USERPROFILE%"], + decode=False, + ) + home = home.decode("utf-16-le").rstrip() if not home: raise subp.ProcessExecutionError( "No output from cmd.exe to show the user profile dir." @@ -443,7 +449,7 @@ def _get_data(self) -> bool: try: user_home = find_home() - except IOError as e: + except (IOError, ValueError) as e: LOG.debug("Unable to detect WSL datasource: %s", e) return False diff --git a/cloudinit/sources/azure/errors.py b/cloudinit/sources/azure/errors.py index ae676a08d29..a752e37468a 100644 --- a/cloudinit/sources/azure/errors.py +++ b/cloudinit/sources/azure/errors.py @@ -161,6 +161,19 @@ def __init__(self, *, key: str, value: Any) -> None: self.supporting_data["type"] = type(value).__name__ +class ReportableErrorMissingCustomData(ReportableError): + def __init__( + self, + *, + pps_type: str, + provisioning_media: str, + ) -> None: + super().__init__("failure to read customData while hasCustomData=true") + + self.supporting_data["pps_type"] = pps_type + self.supporting_data["provisioning_media"] = provisioning_media + + class ReportableErrorImdsMetadataParsingException(ReportableError): def __init__(self, *, exception: ValueError) -> None: super().__init__("error parsing IMDS metadata") @@ -168,6 +181,13 @@ def __init__(self, *, exception: ValueError) -> None: self.supporting_data["exception"] = repr(exception) +class ReportableErrorImportError(ReportableError): + def __init__(self, *, error: ImportError) -> None: + super().__init__(f"error importing {error.name} library") + + self.supporting_data["error"] = repr(error) + + class ReportableErrorOsDiskPpsFailure(ReportableError): def __init__(self) -> None: super().__init__("error waiting for host shutdown") diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5abb19ab95a..69283dd83a0 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -558,10 +558,8 @@ def is_new_instance(self): or previous != self.ds.get_instance_id() ) - def fetch(self, existing="check"): - """optionally load datasource from cache, otherwise discover - datasource - """ + def fetch(self, existing="check") -> sources.DataSource: + """Load datasource from cache, otherwise discover datasource""" return self._get_data_source(existing=existing) def instancify(self): diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py index faa4aaa287a..f6b52e35d98 100644 --- a/cloudinit/temp_utils.py +++ b/cloudinit/temp_utils.py @@ -20,7 +20,10 @@ def get_tmp_ancestor(odir=None, needs_exe: bool = False): if needs_exe: return _EXE_ROOT_TMPDIR if os.getuid() == 0: - return _ROOT_TMPDIR + if util.is_BSD(): + return "/var/" + _ROOT_TMPDIR + else: + return _ROOT_TMPDIR return os.environ.get("TMPDIR", "/tmp") diff --git a/cloudinit/templater.py b/cloudinit/templater.py index b33f0c95a2e..d9853932bc3 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -16,29 +16,14 @@ import logging import re import sys -from typing import Any -from jinja2 import TemplateSyntaxError +from jinja2 import DebugUndefined, Template, TemplateSyntaxError from cloudinit import performance from cloudinit import type_utils as tu from cloudinit import util from cloudinit.atomic_helper import write_file -# After bionic EOL, mypy==1.0.0 will be able to type-analyse dynamic -# base types, substitute this by: -# JUndefined: typing.Type -JUndefined: Any -try: - from jinja2 import DebugUndefined as _DebugUndefined - from jinja2 import Template as JTemplate - - JINJA_AVAILABLE = True - JUndefined = _DebugUndefined -except (ImportError, AttributeError): - JINJA_AVAILABLE = False - JUndefined = object - LOG = logging.getLogger(__name__) MISSING_JINJA_PREFIX = "CI_MISSING_JINJA_VAR/" @@ -86,7 +71,7 @@ def format_error_message( # Mypy, and the PEP 484 ecosystem in general, does not support creating # classes with dynamic base types: https://stackoverflow.com/a/59636248 -class UndefinedJinjaVariable(JUndefined): +class UndefinedJinjaVariable(DebugUndefined): """Class used to represent any undefined jinja template variable.""" def __str__(self): @@ -150,7 +135,7 @@ def jinja_render(content, params): try: with performance.Timed("Rendering jinja2 template"): return ( - JTemplate( + Template( content, undefined=UndefinedJinjaVariable, trim_blocks=True, @@ -181,13 +166,7 @@ def jinja_render(content, params): "Unknown template rendering type '%s' requested" % template_type ) - if template_type == "jinja" and not JINJA_AVAILABLE: - LOG.warning( - "Jinja not available as the selected renderer for" - " desired template, reverting to the basic renderer." - ) - return ("basic", basic_render, rest) - elif template_type == "jinja" and JINJA_AVAILABLE: + elif template_type == "jinja": return ("jinja", jinja_render, rest) # Only thing left over is the basic renderer (it is always available). return ("basic", basic_render, rest) diff --git a/cloudinit/util.py b/cloudinit/util.py index cac5926f10f..2cdd73655ef 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1298,8 +1298,12 @@ def is_resolvable(url) -> bool: # Early return for IP addresses - no DNS resolution needed with suppress(ValueError): - if net.is_ip_address(parsed_url.netloc.strip("[]")): + if net.is_ip_address(name): return True + try: + hostname_result = socket.getaddrinfo(name, None) + except (socket.gaierror, socket.error): + return False if _DNS_REDIRECT_IP is None: badips = set() @@ -1324,13 +1328,11 @@ def is_resolvable(url) -> bool: if badresults: LOG.debug("detected dns redirection: %s", badresults) - try: - result = socket.getaddrinfo(name, None) - # check first result's sockaddr field - addr = result[0][4][0] - return addr not in _DNS_REDIRECT_IP - except (socket.gaierror, socket.error): + # check first result's sockaddr field + addr = hostname_result[0][4][0] + if addr in _DNS_REDIRECT_IP: return False + return True def get_hostname(): diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 558eab2433b..3926ea8b4d5 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -13,7 +13,6 @@ "raspberry-pi-os": "Raspberry Pi OS", "ubuntu": "Ubuntu", "unknown": "Ubuntu"}) %} {% set groups = ({"alpine": "adm, wheel", "aosc": "wheel", "arch": "wheel, users", - "azurelinux": "wheel", "debian": "adm, audio, cdrom, dialout, dip, floppy, netdev, plugdev, sudo, video", "gentoo": "users, wheel", "mariner": "wheel", "photon": "wheel", @@ -25,7 +24,7 @@ {% set shells = ({"alpine": "/bin/ash", "dragonfly": "/bin/sh", "freebsd": "/bin/tcsh", "netbsd": "/bin/sh", "openbsd": "/bin/ksh"}) %} -{% set usernames = ({"amazon": "ec2-user", "centos": "cloud-user", +{% set usernames = ({"amazon": "ec2-user", "azurelinux": "azureuser", "centos": "cloud-user", "openmandriva": "omv", "raspberry-pi-os": "pi", "rhel": "cloud-user", "unknown": "ubuntu"}) %} @@ -61,7 +60,7 @@ disable_root: false disable_root: true {% endif %} -{%- if variant in ["alpine", "amazon", "fedora", "OpenCloudOS", "openeuler", +{%- if variant in ["alpine", "amazon", "azurelinux", "fedora", "OpenCloudOS", "openeuler", "openmandriva", "photon", "TencentOS"] or is_rhel %} {% if is_rhel %} @@ -97,13 +96,6 @@ apt: # timeout: 5 # (defaults to 50 seconds) # max_wait: 10 # (defaults to 120 seconds) -{% if variant == "amazon" %} -# Amazon Linux relies on ec2-net-utils for network configuration -network: - config: disabled - -{% endif -%} - {% if is_rhel %} # Default redhat settings: ssh_deletekeys: true @@ -168,12 +160,12 @@ cloud_config_modules: {% if variant == "ubuntu" %} - ubuntu_pro {% endif %} -{% elif variant in ["azurelinux", "fedora", "mariner", "openeuler", +{% elif variant in ["amazon", "azurelinux", "fedora", "mariner", "openeuler", "openmandriva", "photon"] or is_rhel %} {% if is_rhel %} - rh_subscription {% endif %} -{% if variant not in ["azurelinux", "mariner", "photon"] %} +{% if variant not in ["amazon", "azurelinux", "mariner", "photon"] %} - spacewalk {% endif %} - yum_add_repo @@ -185,9 +177,7 @@ cloud_config_modules: {% if variant == "raspberry-pi-os" %} - raspberry_pi {% endif %} -{% if variant not in ["azurelinux"] %} - disable_ec2_metadata -{% endif %} - runcmd {% if variant in ["debian", "ubuntu", "unknown"] %} - byobu @@ -205,17 +195,13 @@ cloud_final_modules: - ubuntu_drivers {% endif %} - write_files_deferred -{% if variant not in ["azurelinux"] %} - puppet - chef -{% endif %} - ansible -{% if variant not in ["azurelinux"] %} - mcollective - salt_minion {% if variant not in ["alpine"] %} - reset_rmc -{% endif %} {% endif %} - scripts_vendor - scripts_per_once @@ -294,6 +280,9 @@ system_info: {% if variant == "alpine" %} network: renderers: ['eni'] +{% elif variant == "azurelinux" %} + network: + renderers: ['netplan', 'networkd', 'network-manager'] {% elif variant == "debian" %} network: renderers: ['netplan', 'eni', 'networkd'] @@ -307,7 +296,7 @@ system_info: {% elif variant in ["freebsd", "netbsd", "openbsd"] %} network: renderers: ['{{ variant }}'] -{% elif variant in ["azurelinux", "mariner", "photon"] %} +{% elif variant in ["mariner", "photon"] %} network: renderers: ['networkd'] {% elif variant == "openmandriva" %} @@ -359,7 +348,7 @@ system_info: security: https://deb.debian.org/debian-security {% elif variant in ["ubuntu", "unknown"] %} package_mirrors: - - arches: [i386, amd64] + - arches: [arm64, i386, amd64] failsafe: primary: http://archive.ubuntu.com/ubuntu security: http://security.ubuntu.com/ubuntu @@ -369,7 +358,7 @@ system_info: - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ security: [] - - arches: [arm64, armel, armhf] + - arches: [armel, armhf] failsafe: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports diff --git a/debian/changelog b/debian/changelog index 473d76acf5d..2ff9d571231 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ -cloud-init (26.2~1g275e0e03-0ubuntu1) UNRELEASED; urgency=medium +cloud-init (26.2~2gfa1bca7e-0ubuntu1) stonking; urgency=medium + * drop the following cherry-picks now included: + + pick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com * d/control: add python3-pyfakefs for testing + * Upstream snapshot based on upstream/main at fa1bca7e. + - Bugs fixed in this snapshot: (LP: #2147101) - -- Chad Smith Tue, 14 Apr 2026 09:55:08 -0600 + -- Chad Smith Thu, 02 Jul 2026 12:52:15 -0600 cloud-init (26.1-0ubuntu2) resolute; urgency=medium diff --git a/debian/patches/cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com b/debian/patches/cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com deleted file mode 100644 index cc441ab9bcf..00000000000 --- a/debian/patches/cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com +++ /dev/null @@ -1,32 +0,0 @@ -From d4566b1aa35951e6c32da330e627c023785026ea Mon Sep 17 00:00:00 2001 -From: Dave Jones -Date: Fri, 3 Apr 2026 02:31:16 +0100 -Subject: [PATCH] fix(ubuntu): Configure arm64 to use archive.ubuntu.com - (#6826) - -Fixes GH-6825 -LP: #2147101 ---- - config/cloud.cfg.tmpl | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - ---- a/config/cloud.cfg.tmpl -+++ b/config/cloud.cfg.tmpl -@@ -359,7 +359,7 @@ system_info: - security: https://deb.debian.org/debian-security - {% elif variant in ["ubuntu", "unknown"] %} - package_mirrors: -- - arches: [i386, amd64] -+ - arches: [arm64, i386, amd64] - failsafe: - primary: http://archive.ubuntu.com/ubuntu - security: http://security.ubuntu.com/ubuntu -@@ -369,7 +369,7 @@ system_info: - - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ - security: [] -- - arches: [arm64, armel, armhf] -+ - arches: [armel, armhf] - failsafe: - primary: http://ports.ubuntu.com/ubuntu-ports - security: http://ports.ubuntu.com/ubuntu-ports diff --git a/debian/patches/series b/debian/patches/series deleted file mode 100644 index db6ddb3ffad..00000000000 --- a/debian/patches/series +++ /dev/null @@ -1 +0,0 @@ -cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index cba70b593a7..87e5299a6bc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -360,9 +360,9 @@ timezone: US/Eastern # - name: user1 # password: password1 # type: text -# - user2 +# - name: user2 # type: RANDOM -# - user3 +# - name: user3 # password: $5$eriogqzq$Dg7PxHsKGzziuEGkZgkLvacjuEFeljJ.rLf.hZqKQLA # type: hash # expire: True diff --git a/doc/module-docs/cc_users_groups/data.yaml b/doc/module-docs/cc_users_groups/data.yaml index 03bb32594d7..a0185bb36af 100644 --- a/doc/module-docs/cc_users_groups/data.yaml +++ b/doc/module-docs/cc_users_groups/data.yaml @@ -23,16 +23,21 @@ cc_users_groups: ``default_user key``. Each ``users`` dictionary item must contain either a ``name`` or - ``snapuser`` key, otherwise it will be ignored. Omission of ``default`` as - the first item in the ``users`` list skips creation the default user. If - no ``users`` key is provided, the default behavior is to create the - default user via this config: + ``snapuser`` key, otherwise it will be ignored. If no ``users`` key is + provided, the default behavior is to create the default user via this + config: .. code-block:: yaml users: - default + .. note:: + If you provide a ``users`` list and still want cloud-init to create the + default distribution user, keep ``default`` as the first item in the + list. Omitting ``default`` skips creation of the default user and can + prevent cloud-provided SSH keys from being installed for that user. + .. note:: Specifying a hash of a user's password with ``passwd`` is a security risk if the cloud-config can be intercepted. SSH authentication is @@ -77,8 +82,6 @@ cc_users_groups: file: cc_users_groups/example3.yaml - comment: > Example 4: Skip creation of the ``default`` user and only create - ``newsuper``. Password-based login is rejected, but the GitHub user - ``TheRealFalcon`` and the Launchpad user ``falcojr`` can SSH as ``newsuper``. ``doas``/``opendoas`` is configured to permit this user to run commands as other users (without being prompted for a password) except not as root. @@ -97,8 +100,9 @@ cc_users_groups: - comment: > Example 7: Override any ``default_user`` config in ``/etc/cloud/cloud.cfg`` with supplemental config options. This config - will make the default user ``mynewdefault`` and change the user to not - have ``sudo`` rights. + will make the default user ``mynewdefault``, change the user to not + have ``sudo`` rights and allow the Launchpad user ``chad.smith`` to + SSH as ``mynewdefault``. file: cc_users_groups/example7.yaml - comment: > Example 8: Avoid creating any ``default_user``. diff --git a/doc/module-docs/cc_users_groups/example3.yaml b/doc/module-docs/cc_users_groups/example3.yaml index b39665b1b1a..49ef8108d63 100644 --- a/doc/module-docs/cc_users_groups/example3.yaml +++ b/doc/module-docs/cc_users_groups/example3.yaml @@ -1,4 +1,6 @@ #cloud-config users: - name: newsuper + lock_passwd: true + ssh_import_id: [ gh:TheRealFalcon, lp:falcojr ] shell: /bin/bash diff --git a/doc/rtd/reference/datasources/azure.rst b/doc/rtd/reference/datasources/azure.rst index a3c6ffa0d98..61703b9344e 100644 --- a/doc/rtd/reference/datasources/azure.rst +++ b/doc/rtd/reference/datasources/azure.rst @@ -45,6 +45,22 @@ The settings that may be configured are: Boolean to configure secondary IP address(es) for each NIC per IMDS configuration. Default is True. + +* :command:`apply_network_config_set_name` + + Boolean to include ``set-name`` directives in the generated network + configuration, which renames interfaces to ``ethX`` naming. When set to + False, interfaces are matched by MAC address (and optionally driver) + without renaming, and retain kernel-assigned names. Default is True. + + Azure's IMDS does not guarantee the ordering of NICs in the network metadata + response (see `Azure IMDS documentation + `_). + Because cloud-init derives ``ethX`` names from the IMDS response order, + NIC names may change between reboots. Disabling this option avoids that + problem by matching interfaces on MAC address (and optionally driver), + allowing the kernel or udev to assign and retain stable names. + * :command:`data_dir` Path used to read meta-data files and write crawled data. @@ -67,6 +83,7 @@ An example configuration with the default values is provided below: Azure: apply_network_config: true apply_network_config_for_secondary_ips: true + apply_network_config_set_name: true data_dir: /var/lib/waagent disk_aliases: ephemeral0: /dev/disk/cloud/azure_resource diff --git a/doc/rtd/reference/datasources/opennebula.rst b/doc/rtd/reference/datasources/opennebula.rst index f4136c668f8..5171704d48c 100644 --- a/doc/rtd/reference/datasources/opennebula.rst +++ b/doc/rtd/reference/datasources/opennebula.rst @@ -59,6 +59,7 @@ the OpenNebula documentation. :: DNS + SEARCH_DOMAIN ETH_IP ETH_NETWORK ETH_MASK @@ -72,8 +73,19 @@ the OpenNebula documentation. ETH_IP6_ULA ETH_IP6_PREFIX_LENGTH ETH_IP6_GATEWAY + ETH_ROUTES -Static `network configuration`_. +Static `network configuration`_. ``DNS`` and ``SEARCH_DOMAIN`` are global +values applied to every interface. Per-interface ``ETH_DNS`` and +``ETH_SEARCH_DOMAIN`` (defined in `context-linux`_) take precedence; +duplicate entries across both levels are suppressed. + +.. _context-linux: https://github.com/OpenNebula/one-apps/blob/v7.0.0/context-linux/src/etc/one-context.d/loc-10-network.d/functions#L463-L466 + +``ETH_ROUTES`` is a comma-separated list of static routes in the form +``NETWORK via GATEWAY``. For example:: + + ETH0_ROUTES="10.0.0.0/8 via 192.168.1.1, 172.16.0.0/12 via 192.168.1.254" :: @@ -146,5 +158,5 @@ Example VM's context section .. _OpenNebula: http://opennebula.org/ .. _contextualization overview: http://opennebula.org/documentation:documentation:context_overview .. _contextualizing VMs: http://opennebula.org/documentation:documentation:cong -.. _network configuration: https://docs.opennebula.io/ +.. _network configuration: https://docs.opennebula.io/7.2/product/operation_references/configuration_references/template/#context-section .. _iso9660: https://en.wikipedia.org/wiki/ISO_9660 diff --git a/doc/rtd/reference/yaml_examples/user_groups.rst b/doc/rtd/reference/yaml_examples/user_groups.rst index c1b2b74cd4e..2b1a0bb115c 100644 --- a/doc/rtd/reference/yaml_examples/user_groups.rst +++ b/doc/rtd/reference/yaml_examples/user_groups.rst @@ -35,9 +35,14 @@ and ``'sys'``, and the empty group ``cloud-users``. Add users to the system ======================= -Users are added after groups. Note that most of these configuration options -will not be honored if the user already exists. The following options are -exceptions and can be applied to already-existing users: +Users are added after groups. If you provide a ``users`` list and still want +cloud-init to create the default distribution user, keep ``default`` as the +first item in the list. Omitting ``default`` skips creation of the default user +and can prevent cloud-provided SSH keys from being installed for that user. + +Note that most of these configuration options will not be honored if the user +already exists. The following options are exceptions and can be applied to +already-existing users: - ``plain_text_passwd`` - ``hashed_passwd`` diff --git a/meson.build b/meson.build index 1068a0a2a85..0e06d8de292 100644 --- a/meson.build +++ b/meson.build @@ -221,6 +221,27 @@ elif init_system == 'sysvinit_freebsd' endforeach # Enable cloud-init on reboot meson.add_install_script('sh', '-c', '/usr/sbin/sysrc cloudinit_enable=YES') +elif init_system == 'sysvinit_openbsd' + rcd_templates = run_command(find, 'sysvinit/openbsd', '-type', 'f', check: true) + foreach template : rcd_templates.stdout().strip().split('\n') + custom_target( + input: template, + output: '@BASENAME@', + command: [ + render_tmpl, + '@INPUT@', + meson.current_build_dir() / '@OUTPUT@', + ], + install: true, + install_dir: sysconfdir / 'rc.d', + install_mode: 'r-xr-xr-x', + install_tag: 'sysvinit', + ) + endforeach + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudinitlocal') + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudinit') + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudconfig') + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudfinal') endif custom_target( diff --git a/meson_options.txt b/meson_options.txt index 8802c8ef3ba..dcb0198afa5 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -1,4 +1,4 @@ -option('init_system', type: 'combo', value: 'systemd', choices: ['systemd', 'sysvinit_openrc', 'sysvinit_freebsd'], description: 'Set target init system.') +option('init_system', type: 'combo', value: 'systemd', choices: ['systemd', 'sysvinit_openrc', 'sysvinit_freebsd', 'sysvinit_openbsd'], description: 'Set target init system.') option('distro_templates', type: 'array', value: [], description: 'Distro template files to install. WARNING: Templates may change in the future. If using this option, be sure to check new releases for template file changes.') option('disable_sshd_keygen', type: 'boolean', value: false, description: 'Provide systemd service to disable sshd-keygen if present to avoid races with cloud-init.') option('bash_completion', type: 'boolean', value: true, description: 'Bash completion for cloud-init.') diff --git a/packages/brpm b/packages/brpm index 7a1cf3d1d96..e24db2b7a92 100755 --- a/packages/brpm +++ b/packages/brpm @@ -29,7 +29,7 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) - from cloudinit import subp, templater, util + from cloudinit import subp, templater, temp_utils, util # Subdirectories of the ~/rpmbuild dir @@ -83,7 +83,7 @@ def read_version_from_meson(): [ "meson", "setup", - "builddir", + builddir, "-Dinit_system=systemd", "-Ddisable_sshd_keygen=true", ] diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index 2d7b7947307..5af11a7b732 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -99,6 +99,25 @@ "sudo" ] }, + "openbsd": { + "renames" : { + "pyserial" : "py3-serial", + "pyyaml" : "py3-yaml", + "pytest" : "py3-test", + "pytest-cov" : "py3-test-cov", + "pytest-mock" : "py3-test-mock", + "pytest-xdist": "py3-test-xdist" + }, + "build-requires" : [ + "bash", + "bash-completion", + "meson" + ], + "requires" : [ + "e2fsprogs", + "sudo--" + ] + }, "suse" : { "renames" : { "jinja2" : "python3-Jinja2", diff --git a/pyproject.toml b/pyproject.toml index b9e25e083c4..8f08187ca8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,6 @@ no_implicit_optional = true # See GH-5445 [[tool.mypy.overrides]] module = [ - "cloudinit.cmd.devel.make_mime", - "cloudinit.cmd.devel.net_convert", - "cloudinit.cmd.main", "cloudinit.config.cc_apt_configure", "cloudinit.config.cc_ca_certs", "cloudinit.config.cc_growpart", @@ -55,7 +52,6 @@ module = [ "cloudinit.distros.bsd", "cloudinit.distros.opensuse", "cloudinit.distros.parsers.hostname", - "cloudinit.distros.parsers.hosts", "cloudinit.distros.parsers.resolv_conf", "cloudinit.distros.ug_util", "cloudinit.helpers", @@ -78,7 +74,6 @@ module = [ "cloudinit.sources.DataSourceHetzner", "cloudinit.sources.DataSourceNoCloud", "cloudinit.sources.DataSourceOVF", - "cloudinit.sources.DataSourceOpenNebula", "cloudinit.sources.DataSourceOpenStack", "cloudinit.sources.DataSourceOracle", "cloudinit.sources.DataSourceRbxCloud", @@ -118,7 +113,6 @@ module = [ "tests.unittests.config.test_cc_zypper_add_repo", "tests.unittests.config.test_modules", "tests.unittests.config.test_schema", - "tests.unittests.distros.test_hosts", "tests.unittests.distros.test_ifconfig", "tests.unittests.distros.test_netbsd", "tests.unittests.distros.test_netconfig", @@ -146,7 +140,6 @@ module = [ "tests.unittests.sources.test_gce", "tests.unittests.sources.test_init", "tests.unittests.sources.test_nocloud", - "tests.unittests.sources.test_opennebula", "tests.unittests.sources.test_openstack", "tests.unittests.sources.test_oracle", "tests.unittests.sources.test_scaleway", @@ -211,6 +204,7 @@ lint.ignore = [ "D403", # docstring: capitalized first line "E731", # Do not assign a `lambda` expression, use a `def` ] +extend-include = ["*read-dependencies", "*bddeb", "*brpm"] [tool.ruff.lint.pydocstyle] convention = "pep257" diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index e55ece6d276..888bad535ea 100644 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -20,7 +20,7 @@ CLOUD_SYSTEM_TARGET="/usr/lib/systemd/system/cloud-init.target" {% else %} CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target" {% endif %} -{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", +{% if variant in ["almalinux", "azurelinux", "centos", "cloudlinux", "eurolinux", "fedora", "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", "TencentOS", "virtuozzo"] %} dsidentify="/usr/libexec/cloud-init/ds-identify" {% elif variant == "benchmark" %} diff --git a/systemd/cloud-init-local.service.tmpl b/systemd/cloud-init-local.service.tmpl index e88b15ca246..ca6dbe0a159 100644 --- a/systemd/cloud-init-local.service.tmpl +++ b/systemd/cloud-init-local.service.tmpl @@ -2,14 +2,14 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Local Stage (pre-network) -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=network-pre.target After=hv_kvp_daemon.service Before=network-pre.target Before=shutdown.target -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} Before=firewalld.target {% endif %} {% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} @@ -22,7 +22,7 @@ ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled [Service] Type=oneshot -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} ExecStartPre=/sbin/restorecon /run/cloud-init {% endif %} # This service is a shim which preserves systemd ordering while allowing a diff --git a/systemd/cloud-init-main.service.tmpl b/systemd/cloud-init-main.service.tmpl index 2ca6220c24c..a38b2e817cf 100644 --- a/systemd/cloud-init-main.service.tmpl +++ b/systemd/cloud-init-main.service.tmpl @@ -8,10 +8,10 @@ # https://www.freedesktop.org/software/systemd/man/latest/systemd-remount-fs.service.html [Unit] Description=Cloud-init: Single Process -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} Requires=dbus.socket After=dbus.socket {% endif %} @@ -31,7 +31,7 @@ ExecStart=/usr/bin/cloud-init --all-stages KillMode=process TasksMax=infinity TimeoutStartSec=infinity -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} ExecStartPre=/sbin/restorecon /run/cloud-init {% endif %} diff --git a/systemd/cloud-init-network.service.tmpl b/systemd/cloud-init-network.service.tmpl index c284024be76..58a36f5dfe0 100644 --- a/systemd/cloud-init-network.service.tmpl +++ b/systemd/cloud-init-network.service.tmpl @@ -2,7 +2,7 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Network Stage -{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} +{% if variant not in ["almalinux", "azurelinux", "cloudlinux", "photon", "rhel"] %} DefaultDependencies=no {% endif %} Wants=cloud-init-local.service @@ -15,7 +15,7 @@ After=systemd-networkd-wait-online.service {% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} After=networking.service {% endif %} -{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", +{% if variant in ["almalinux", "azurelinux", "centos", "cloudlinux", "eurolinux", "fedora", "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", "suse", "TencentOS", "virtuozzo"] %} After=NetworkManager.service diff --git a/templates/hosts.azurelinux.tmpl b/templates/hosts.azurelinux.tmpl index 8e3c23f6f12..9e64e26916e 100644 --- a/templates/hosts.azurelinux.tmpl +++ b/templates/hosts.azurelinux.tmpl @@ -19,4 +19,5 @@ you need to add the following to config: # The following lines are desirable for IPv6 capable hosts ::1 {{fqdn}} {{hostname}} +::1 localhost.localdomain localhost ::1 localhost6.localdomain6 localhost6 diff --git a/test-requirements.txt b/test-requirements.txt index 9467f3d9328..4d89bf76946 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,6 +15,7 @@ jsonschema responses packaging passlib +pyfakefs # This one is currently used only by the CloudSigma and SmartOS datasources. # If these datasources are removed, this is no longer needed. diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index e72213a1002..3ccb06e104b 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -39,8 +39,24 @@ DISTRO_TO_USERNAME = { "ubuntu": "ubuntu", + "rhel": "cloud-user", + "centos": "cloud-user", } +# Platform specific username overrides: (distro, platform) -> username +# Only list entries where the platform differs from the distro default. +DISTRO_PLATFORM_TO_USERNAME = { + ("rhel", "ec2"): integration_settings.LAUNCH_USERNAME or "ec2-user", + ("rhel", "azure"): integration_settings.LAUNCH_USERNAME or "azureuser", +} + + +def get_launch_username(os: str, platform: str) -> str: + key = (os, platform) + if key in DISTRO_PLATFORM_TO_USERNAME: + return DISTRO_PLATFORM_TO_USERNAME[key] + return DISTRO_TO_USERNAME[os] + def _get_ubuntu_series() -> list: """Use distro-info-data's ubuntu.csv to get a list of Ubuntu series""" @@ -135,7 +151,9 @@ def launch( default_launch_kwargs = { "image_id": self.image_id, "user_data": user_data, - "username": DISTRO_TO_USERNAME[CURRENT_RELEASE.os], + "username": get_launch_username( + CURRENT_RELEASE.os, self.datasource + ), } if self.settings.INSTANCE_TYPE: default_launch_kwargs["instance_type"] = ( @@ -259,9 +277,10 @@ def _get_initial_image(self, **kwargs) -> str: class AzureCloud(IntegrationCloud): datasource = "azure" cloud_instance: Azure + username = get_launch_username(CURRENT_RELEASE.os, datasource) def _get_cloud_instance(self) -> Azure: - return Azure(tag="azure-integration-test") + return Azure(tag="azure-integration-test", username=self.username) def _get_initial_image(self, **kwargs) -> str: return super()._get_initial_image( diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index cb4bae3a84e..6164b4e7332 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -56,6 +56,10 @@ # creating a new one. The exact contents will be platform dependent EXISTING_INSTANCE_ID: Optional[str] = None +# Username to use when launching the instance. +# If not set, the default username for the platform will be used. +LAUNCH_USERNAME: Optional[str] = None + ################################################################## # IMAGE GENERATION SETTINGS ################################################################## diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 78daf954ebd..e4f7e708b1a 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -18,10 +18,12 @@ CURRENT_RELEASE, IS_UBUNTU, MANTIC, + QUESTING, ) from tests.integration_tests.util import ( get_feature_flag_value, verify_clean_boot, + wait_for_cloud_init, ) logger = logging.getLogger(__name__) @@ -522,11 +524,16 @@ def test_apt_proxy(client: IntegrationInstance): r"software-properties-common', 'gnupg)" ) -REMOVE_GPG_USERDATA = """ +GPG_PACKAGES = "gpg software-properties-common" +# On Ubuntu Resolute and newer, gpg-from-sq can replace the gpg metapackage +# when gpg is removed. Remove other packages which rdepend on gpg to avoid +# gpg-from-sq being installed as an alternative to gpg. +GPG_PACKAGES_SQ = f"{GPG_PACKAGES} python3-software-properties libgpgme45" + +REMOVE_GPG_USERDATA_TMPL = """ #cloud-config runcmd: - - DEBIAN_FRONTEND=noninteractive apt-get remove gpg -y - - DEBIAN_FRONTEND=noninteractive apt-get remove software-properties-common -y + - DEBIAN_FRONTEND=noninteractive apt-get remove {packages} -y """ @@ -569,9 +576,11 @@ def test_install_missing_deps(session_cloud: IntegrationCloud): 'software-properties-common' are installed successfully. """ # Two stage install: First stage: remove gpg noninteractively from image - instance1 = session_cloud.launch( - user_data=_do_oci_customization(REMOVE_GPG_USERDATA) - ) + if CURRENT_RELEASE <= QUESTING: + userdata = REMOVE_GPG_USERDATA_TMPL.format(packages=GPG_PACKAGES) + else: + userdata = REMOVE_GPG_USERDATA_TMPL.format(packages=GPG_PACKAGES_SQ) + instance1 = session_cloud.launch(user_data=_do_oci_customization(userdata)) # look for r"un gpg" using regex ('un' means uninstalled) for package in ["gpg", "software-properties-common"]: @@ -597,6 +606,7 @@ def test_install_missing_deps(session_cloud: IntegrationCloud): user_data=INSTALL_ANY_MISSING_RECOMMENDED_DEPENDENCIES, launch_kwargs={"image_id": snapshot_id}, ) as minimal_client: + wait_for_cloud_init(minimal_client) log = minimal_client.read_from_file("/var/log/cloud-init.log") assert re.search(RE_GPG_SW_PROPERTIES_INSTALLED, log) gpg_installed = re.search( diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 857023780d1..286aa5a704a 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -26,7 +26,12 @@ OS_IMAGE_TYPE, PLATFORM, ) -from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, NOBLE +from tests.integration_tests.releases import ( + CURRENT_RELEASE, + IS_RHEL, + IS_UBUNTU, + NOBLE, +) from tests.integration_tests.util import ( get_feature_flag_value, get_inactive_modules, @@ -37,7 +42,7 @@ verify_ordered_items_in_text, ) -USER_DATA = """\ +USER_DATA_UBUNTU = """\ #cloud-config users: - default @@ -72,7 +77,7 @@ input(type="imtcp" port="514") $template RemoteLogs,"/var/spool/rsyslog/cloudinit.log" *.* ?RemoteLogs - & ~ + & stop remotes: me: "127.0.0.1" runcmd: @@ -80,15 +85,70 @@ - echo '💩' > /var/tmp/unicode_data - # - - logger "My test log" + - logger --server localhost --tcp --port 514 "My test log" snap: commands: - snap install hello-world ssh_import_id: - - lp:smoser + - gh:blackboxsw + +timezone: Europe/Madrid +""" + +USER_DATA_RHEL = """\ +#cloud-config +users: +- default +- name: craig + sudo: false # make sure craig doesn't get elevated perms +final_message: | + This is my final message! + $version + $timestamp + $datasource + $uptime +locale: en_GB.UTF-8 +locale_configfile: /etc/locale.conf +package_update: true +random_seed: + data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' + encoding: raw + file: /root/seed +rsyslog: + configs: + - "*.* @@127.0.0.1" + - filename: 0-basic-config.conf + content: | + module(load="imtcp") + input(type="imtcp" port="514") + $template RemoteLogs,"/var/log/rsyslog-cloudinit.log" + *.* ?RemoteLogs + & ~ + remotes: + me: "127.0.0.1" +runcmd: + - echo 'hello world' > /var/tmp/runcmd_output + - echo '💩' > /var/tmp/unicode_data + + - # + - logger "My test log" timezone: Europe/Madrid """ +# Update this dict with proper user data to support new distros +USER_DATA_BY_DISTRO = { + "ubuntu": USER_DATA_UBUNTU, + "rhel": USER_DATA_RHEL, + "centos": USER_DATA_RHEL, +} + +if CURRENT_RELEASE.os not in USER_DATA_BY_DISTRO: + raise KeyError( + f"No USER_DATA for distro {CURRENT_RELEASE.os!r}. " + f"Add an entry to USER_DATA_BY_DISTRO for this distro." + ) + +USER_DATA = USER_DATA_BY_DISTRO[CURRENT_RELEASE.os] @pytest.mark.ci @@ -163,6 +223,7 @@ def test_deprecated_message(self, class_client: IntegrationInstance): ignore_warnings=True, ) + @pytest.mark.skipif(IS_RHEL, reason="rhel does not support ntp module") def test_ntp_with_apt(self, class_client: IntegrationInstance): """LP #1628337. @@ -175,6 +236,9 @@ def test_ntp_with_apt(self, class_client: IntegrationInstance): assert "W: Some index files failed to download" not in log assert "E: Unable to locate package ntp" not in log + @pytest.mark.skipif( + IS_RHEL, reason="rhel does not enable byobu by default" + ) def test_byobu(self, class_client: IntegrationInstance): """Test byobu configured as enabled by default.""" client = class_client @@ -183,9 +247,13 @@ def test_byobu(self, class_client: IntegrationInstance): def test_configured_locale(self, class_client: IntegrationInstance): """Test locale can be configured correctly.""" client = class_client - default_locale = client.read_from_file("/etc/default/locale") + default_locale_file = ( + "/etc/locale.conf" if IS_RHEL else "/etc/default/locale" + ) + default_locale = client.read_from_file(default_locale_file) assert "LANG=en_GB.UTF-8" in default_locale - + if IS_RHEL: + return locale_a = client.execute("locale -a") locale_gen = client.execute("grep -v '^#' /etc/locale.gen | uniq") if OS_IMAGE_TYPE == "minimal": @@ -214,16 +282,21 @@ def test_random_seed_data(self, class_client: IntegrationInstance): def test_rsyslog(self, class_client: IntegrationInstance): """Test rsyslog is configured correctly when applicable.""" + # /var/spool/rsylog is not created on rhel by default + log_file = ( + "/var/log/rsyslog-cloudinit.log" + if IS_RHEL + else "/var/spool/rsyslog/cloudinit.log" + ) if class_client.execute("command -v rsyslogd").ok: - assert "My test log" in class_client.read_from_file( - "/var/spool/rsyslog/cloudinit.log" - ) + assert "My test log" in class_client.read_from_file(log_file) def test_runcmd(self, class_client: IntegrationInstance): """Test runcmd works as expected""" client = class_client assert "hello world" == client.read_from_file("/var/tmp/runcmd_output") + @pytest.mark.skipif(IS_RHEL, reason="rhel does not support snap module") def test_snap(self, class_client: IntegrationInstance): """Integration test for the snap module. @@ -276,19 +349,32 @@ def test_no_problems(self, class_client: IntegrationInstance): verify_clean_boot( client, ignore_deprecations=True, require_warnings=require_warnings ) - requested_modules = { - "apt_configure", - "byobu", - "final_message", - "locale", - "ntp", - "seed_random", - "rsyslog", - "runcmd", - "snap", - "ssh_import_id", - "timezone", - } + # remove modules that are not supported on rhel + requested_modules = ( + { + "byobu", + "final_message", + "locale", + "seed_random", + "rsyslog", + "runcmd", + "timezone", + } + if IS_RHEL + else { + "apt_configure", + "byobu", + "final_message", + "locale", + "ntp", + "seed_random", + "rsyslog", + "runcmd", + "snap", + "ssh_import_id", + "timezone", + } + ) inactive_modules = get_inactive_modules(log) assert not requested_modules.intersection(inactive_modules), ( f"Expected active modules:" @@ -578,6 +664,7 @@ def test_networkd_wait_online(self, class_client: IntegrationInstance): @pytest.mark.user_data(USER_DATA) class TestCombinedNoCI: @retry(tries=30, delay=1) + @pytest.mark.skipif(IS_RHEL, reason="rhel skips ssh_import_id module") def test_ssh_import_id(self, class_client: IntegrationInstance): """Integration test for the ssh_import_id module. @@ -591,4 +678,4 @@ def test_ssh_import_id(self, class_client: IntegrationInstance): client = class_client ssh_output = client.read_from_file("/home/ubuntu/.ssh/authorized_keys") - assert "# ssh-import-id lp:smoser" in ssh_output + assert "# ssh-import-id gh:blackboxsw" in ssh_output diff --git a/tests/integration_tests/modules/test_keys_to_console.py b/tests/integration_tests/modules/test_keys_to_console.py index b8e74f57ba8..2f53fd7f4df 100644 --- a/tests/integration_tests/modules/test_keys_to_console.py +++ b/tests/integration_tests/modules/test_keys_to_console.py @@ -12,7 +12,7 @@ from tests.integration_tests.util import ( HAS_CONSOLE_LOG, get_console_log, - get_syslog_or_console, + get_journal_syslog, ) BLACKLIST_USER_DATA = """\ @@ -52,16 +52,14 @@ class TestKeysToConsoleBlacklist: @pytest.mark.parametrize("key_type", ["ECDSA"]) def test_excluded_keys(self, class_client, key_type): - assert "({})".format(key_type) not in get_syslog_or_console( - class_client - ) + assert "({})".format(key_type) not in get_journal_syslog(class_client) # retry decorator here because it can take some time to be reflected - # in syslog + # in the journal @retry(tries=60, delay=1) @pytest.mark.parametrize("key_type", ["ED25519", "RSA"]) def test_included_keys(self, class_client, key_type): - assert "({})".format(key_type) in get_syslog_or_console(class_client) + assert "({})".format(key_type) in get_journal_syslog(class_client) @pytest.mark.user_data(BLACKLIST_ALL_KEYS_USER_DATA) @@ -75,12 +73,12 @@ class TestAllKeysToConsoleBlacklist: """ def test_header_excluded(self, class_client): - assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) def test_footer_excluded(self, class_client): - assert "END SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "END SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) @@ -95,17 +93,15 @@ class TestKeysToConsoleDisabled: @pytest.mark.parametrize("key_type", ["ECDSA", "ED25519", "RSA"]) def test_keys_excluded(self, class_client, key_type): - assert "({})".format(key_type) not in get_syslog_or_console( - class_client - ) + assert "({})".format(key_type) not in get_journal_syslog(class_client) def test_header_excluded(self, class_client): - assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) def test_footer_excluded(self, class_client): - assert "END SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "END SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py index ec6df783250..e04783520e3 100644 --- a/tests/integration_tests/modules/test_set_password.py +++ b/tests/integration_tests/modules/test_set_password.py @@ -14,7 +14,10 @@ from tests.integration_tests.decorators import retry from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU -from tests.integration_tests.util import get_console_log +from tests.integration_tests.util import ( + fetch_and_parse_etc_shadow, + get_console_log, +) COMMON_USER_DATA = """\ #cloud-config @@ -38,6 +41,9 @@ # sha256 gojanego passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." lock_passwd: false + - name: sally + # sha256 gosallygo + passwd: "$5$bA$KBMTe8lXf0e8lE4f4hYPU0h6h0HzQX4vHpnq6xHn9Q2" - name: "mikey" lock_passwd: false """ @@ -94,41 +100,33 @@ class Mixin: """Shared test definitions.""" - def _fetch_and_parse_etc_shadow(self, class_client): - """Fetch /etc/shadow and parse it into Python data structures - - Returns: ({user: password}, [duplicate, users]) - """ - shadow_content = class_client.read_from_file("/etc/shadow") - users = {} - dupes = [] - for line in shadow_content.splitlines(): - user, encpw = line.split(":")[0:2] - if user in users: - dupes.append(user) - users[user] = encpw - return users, dupes - def test_no_duplicate_users_in_shadow(self, class_client): """Confirm that set_passwords has not added duplicate shadow entries""" - _, dupes = self._fetch_and_parse_etc_shadow(class_client) + _, dupes = fetch_and_parse_etc_shadow(class_client) assert [] == dupes def test_password_in_users_dict_set_correctly(self, class_client): """Test that the password specified in the users dict is set.""" - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) assert USERS_PASSWD_VALUES["jane"] == shadow_users["jane"] + def test_hashed_password_without_lock_passwd_override_is_locked( + self, class_client + ): + """Hashed passwords are locked when lock_passwd is not set.""" + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) + assert f"!{USERS_PASSWD_VALUES['sally']}" == shadow_users["sally"] + def test_password_in_chpasswd_list_set_correctly(self, class_client): """Test that a chpasswd password overrides one in the users dict.""" - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) mikey_hash = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89" assert mikey_hash == shadow_users["mikey"] def test_random_passwords_set_correctly(self, class_client): """Test that RANDOM chpasswd entries replace users dict passwords.""" - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) # These should have been changed assert shadow_users["harry"] != USERS_PASSWD_VALUES["harry"] @@ -170,7 +168,7 @@ def test_explicit_password_set_correctly(self, class_client): ) if minor_version > 12: pytest.xfail("Instance under test doesn't have 'crypt' in stdlib") - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) fmt_and_salt = shadow_users["tom"].rsplit("$", 1)[0] diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py index d55cba91e70..8cc68ae7880 100644 --- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py +++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py @@ -19,7 +19,7 @@ OS_IMAGE_TYPE, PLATFORM, ) -from tests.integration_tests.util import HAS_CONSOLE_LOG, get_syslog_or_console +from tests.integration_tests.util import HAS_CONSOLE_LOG, get_journal_syslog USER_DATA_SSH_AUTHKEY_DISABLE = """\ #cloud-config @@ -55,10 +55,10 @@ def test_ssh_authkey_fingerprints_disable(self, client): reason=f"No console_log available for minimal images on {PLATFORM}", ) def test_ssh_authkey_fingerprints_enable(self, client): - syslog_output = get_syslog_or_console(client) - assert re.search(r"256 SHA256:.*(ECDSA)", syslog_output) is not None - assert re.search(r"256 SHA256:.*(ED25519)", syslog_output) is not None - assert re.search(r"2048 SHA256:.*(RSA)", syslog_output) is None + log_output = get_journal_syslog(client) + assert re.search(r"256 SHA256:.*(ECDSA)", log_output) is not None + assert re.search(r"256 SHA256:.*(ED25519)", log_output) is not None + assert re.search(r"2048 SHA256:.*(RSA)", log_output) is None @pytest.mark.user_data( diff --git a/tests/integration_tests/modules/test_ubuntu_pro.py b/tests/integration_tests/modules/test_ubuntu_pro.py index f955e6eb60f..93f1e51ab45 100644 --- a/tests/integration_tests/modules/test_ubuntu_pro.py +++ b/tests/integration_tests/modules/test_ubuntu_pro.py @@ -231,7 +231,6 @@ def maybe_install_cloud_init(session_cloud: IntegrationCloud): client.install_new_cloud_init(source) session_cloud.snapshot_id = client.snapshot() - client.destroy() return {"image_id": session_cloud.snapshot_id} @@ -251,8 +250,6 @@ def test_custom_services(self, session_cloud: IntegrationCloud): user_data=AUTO_ATTACH_CUSTOM_SERVICES, launch_kwargs=launch_kwargs, ) as client: - log = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log) verify_clean_boot(client) assert_ua_service_noop(client) assert is_attached(client) diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 55813220fc2..7edb5956048 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -10,13 +10,17 @@ import pytest from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.integration_settings import CLOUD_INIT_SOURCE from tests.integration_tests.releases import ( CURRENT_RELEASE, IS_UBUNTU, JAMMY, NOBLE, ) -from tests.integration_tests.util import verify_clean_boot +from tests.integration_tests.util import ( + fetch_and_parse_etc_shadow, + verify_clean_boot, +) USER_DATA = """\ #cloud-config @@ -115,6 +119,10 @@ def test_users_groups(self, regex, getent_args, class_client): ) ) + @pytest.mark.skipif( + not IS_UBUNTU, + reason="Warning expectations are Ubuntu-specific", + ) def test_initial_warnings(self, class_client): """Check for initial warnings.""" warnings = ( @@ -134,6 +142,10 @@ def test_user_root_in_secret(self, class_client): groups = groups_str.split() assert "secret" in groups + @pytest.mark.skipif( + not IS_UBUNTU, + reason="Password unlock warning behavior differs across distros", + ) def test_nopassword_unlock_warnings(self, class_client): """Verify warnings for empty passwords for new and existing users.""" # Fake admin clearing and unlocking and empty unlocked password foobar @@ -161,8 +173,8 @@ def test_nopassword_unlock_warnings(self, class_client): @pytest.mark.user_data(USER_DATA) @pytest.mark.skipif( - CURRENT_RELEASE < JAMMY, - reason="Requires version of sudo not available in older releases", + IS_UBUNTU and CURRENT_RELEASE < JAMMY, + reason="Requires version of sudo not available in older Ubuntu releases", ) def test_sudoers_includedir(client: IntegrationInstance): """Ensure we don't add additional #includedir to sudoers. @@ -192,3 +204,62 @@ def test_sudoers_includedir(client: IntegrationInstance): "/etc/sudoers.d/90-cloud-init-users" ).splitlines()[1:] assert sudoers_content_before == sudoers_content_after + + +USER_DATA_OVERRIDE = """\ +#cloud-config +users: + - default + - name: ubuntu + shell: /bin/sh + lock_passwd: false + hashed_passwd: $5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 +""" + + +@pytest.mark.ci +@pytest.mark.skipif(not IS_UBUNTU, reason="Test is Ubuntu specific") +@pytest.mark.user_data(USER_DATA_OVERRIDE) +def test_default_user_settings_override(client: IntegrationInstance): + """ + Test that the default user settings are correctly overridden. + """ + # Check shell + shell_set = ( + client.execute(["getent", "passwd", "ubuntu"]) + .stdout.strip() + .split(":")[-1] + ) + if CLOUD_INIT_SOURCE in ["NONE", "IN_PLACE"]: + assert ( + "/bin/sh" == shell_set + ), "Shell setting not overriden even though the user is new" + else: + assert ( + "/bin/bash" == shell_set + ), "Shell setting overriden even though user already exists" + # Check password is not locked + passwd_status = client.execute(["passwd", "-S", "ubuntu"]).stdout + assert re.search(r"^ubuntu\s+P\b", passwd_status) + expected_passwd_hash = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89" + parsed_shadow, _ = fetch_and_parse_etc_shadow(client) + assert parsed_shadow["ubuntu"] == expected_passwd_hash + + +@pytest.mark.skipif(not IS_UBUNTU, reason="Test is Ubuntu specific") +def test_default_user_settings(client: IntegrationInstance): + """ + This test serves as a "negative control" for + test_default_user_settings_override, confirming the default + user settings are as expected when not overridden by user-data. + """ + # Check shel + shell_set = ( + client.execute(["getent", "passwd", "ubuntu"]) + .stdout.strip() + .split(":")[-1] + ) + assert "/bin/bash" == shell_set + # Check password is not locked + passwd_status = client.execute(["passwd", "-S", "ubuntu"]).stdout + assert re.search(r"^ubuntu\s+L\b", passwd_status) diff --git a/tests/integration_tests/releases.py b/tests/integration_tests/releases.py index 3c3c1fd3df0..25ca759f8e3 100644 --- a/tests/integration_tests/releases.py +++ b/tests/integration_tests/releases.py @@ -105,3 +105,7 @@ def from_os_image( CURRENT_RELEASE = Release.from_os_image() IS_UBUNTU = CURRENT_RELEASE.os == "ubuntu" +IS_RHEL = CURRENT_RELEASE.os in ( + "rhel", + "centos", +) # will add other RHEL-like distros later diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 6e86b677552..0906d3dbb10 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -10,16 +10,13 @@ from functools import lru_cache from itertools import chain from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Set, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union import pytest from cloudinit.subp import subp from tests.integration_tests.decorators import retry -from tests.integration_tests.integration_settings import ( - OS_IMAGE_TYPE, - PLATFORM, -) +from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, NOBLE LOG = logging.getLogger("integration_testing.util") @@ -137,6 +134,11 @@ def append_or_create_list( ignore_warnings = append_or_create_list( ignore_warnings, "No lease found; using default endpoint" ) + # Pro images sometimes hit a single 404 in early provisioning + ignore_warnings = append_or_create_list( + ignore_warnings, + "Polling IMDS failed attempt 1 with exception: UrlError('404", + ) elif "lxd_vm" == PLATFORM: # Ubuntu lxd storage ignore_warnings = append_or_create_list( @@ -578,13 +580,15 @@ def get_console_log(client: "IntegrationInstance"): return console_log -@retry(tries=5, delay=1) # Retry on get_console_log failures -def get_syslog_or_console(client: "IntegrationInstance") -> str: - """minimal OS_IMAGE_TYPE does not contain rsyslog""" - if OS_IMAGE_TYPE == "minimal" and HAS_CONSOLE_LOG: - return get_console_log(client) - else: - return client.read_from_file("/var/log/syslog") +@retry(tries=5, delay=1) # Retry on transient journalctl failures +def get_journal_syslog(client: "IntegrationInstance") -> str: + """Syslog events are categorized _TRANSPORT=syslog from systemd v205.""" + # Prefer syslog transport categorized messages over presence of + # /var/log/syslog as systemd v255 introduced systemd-executor + # which sandboxes unit processes resulting in direct writes to + # /dev/console being logged directly to journal binary instead + # of mirrored as rsyslog events. + return client.execute(["journalctl", "_TRANSPORT=syslog", "-b", "0"]) @lru_cache() @@ -692,3 +696,21 @@ def get_datetime_from_string( ) ) return converted_datetime + + +def fetch_and_parse_etc_shadow( + client: "IntegrationInstance", +) -> Tuple[Dict[str, str], List[str]]: + """Fetch /etc/shadow and parse it into Python data structures + + Returns: ({user: password}, [duplicate, users]) + """ + shadow_content = client.read_from_file("/etc/shadow") + users = {} + dupes = [] + for line in shadow_content.splitlines(): + user, encpw = line.split(":")[0:2] + if user in users: + dupes.append(user) + users[user] = encpw + return users, dupes diff --git a/tests/unittests/analyze/test_boot.py b/tests/unittests/analyze/test_boot.py index ffe147e5110..9e89ef8867e 100644 --- a/tests/unittests/analyze/test_boot.py +++ b/tests/unittests/analyze/test_boot.py @@ -1,4 +1,5 @@ import os +from unittest import mock import pytest @@ -11,7 +12,6 @@ dist_check_timestamp, gather_timestamps_using_systemd, ) -from tests.unittests.helpers import mock err_code = (FAIL_CODE, -1, -1, -1) @@ -160,7 +160,7 @@ def test_container_no_ci_log_line(self, m_is_container, m_subp): finish_code = analyze_boot(name_default, args) self.remove_dummy_file(path, log_path) - assert FAIL_CODE == finish_code + assert 1 == finish_code @mock.patch("cloudinit.util.is_container", return_value=True) @mock.patch("cloudinit.subp.subp", return_value=("U=1000000", None)) @@ -190,7 +190,7 @@ def test_container_ci_log_line(self, m_is_container, m_subp, m_get, m_g): finish_code = analyze_boot(name_default, args) self.remove_dummy_file(path, log_path) - assert CONTAINER_CODE == finish_code + assert 0 == finish_code @mock.patch("cloudinit.analyze.show.SystemctlReader") @pytest.mark.parametrize( diff --git a/tests/unittests/analyze/test_dump.py b/tests/unittests/analyze/test_dump.py index ebf717e088c..45bf1c743e4 100644 --- a/tests/unittests/analyze/test_dump.py +++ b/tests/unittests/analyze/test_dump.py @@ -4,6 +4,7 @@ from contextlib import suppress from datetime import datetime, timezone from textwrap import dedent +from unittest import mock import pytest @@ -14,7 +15,6 @@ parse_timestamp, ) from cloudinit.util import write_file -from tests.unittests.helpers import mock class TestParseTimestamp: diff --git a/tests/unittests/cmd/devel/test_net_convert.py b/tests/unittests/cmd/devel/test_net_convert.py index 9328150d5bc..584741dbe3b 100644 --- a/tests/unittests/cmd/devel/test_net_convert.py +++ b/tests/unittests/cmd/devel/test_net_convert.py @@ -1,13 +1,13 @@ # This file is part of cloud-init. See LICENSE file for license information. import itertools +from unittest import mock import pytest import yaml from cloudinit.cmd.devel import net_convert from cloudinit.distros.debian import NETWORK_FILE_HEADER -from tests.unittests.helpers import mock M_PATH = "cloudinit.cmd.devel.net_convert." diff --git a/tests/unittests/cmd/devel/test_render.py b/tests/unittests/cmd/devel/test_render.py index 90557dcb49b..d6abb28aa39 100644 --- a/tests/unittests/cmd/devel/test_render.py +++ b/tests/unittests/cmd/devel/test_render.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from io import StringIO +from unittest import mock import pytest @@ -8,7 +9,6 @@ from cloudinit.helpers import Paths from cloudinit.templater import JinjaSyntaxParsingException from cloudinit.util import ensure_dir, write_file -from tests.unittests.helpers import mock, skipUnlessJinja M_PATH = "cloudinit.cmd.devel.render." @@ -87,7 +87,6 @@ def test_root_uses_sensitive_instance_data(self, m_paths, tmpdir): assert render.render_template(user_data, None, False) == 0 assert "rendering: jinja worked" in m_stdout.getvalue() - @skipUnlessJinja() def test_renders_instance_data_vars_in_template(self, caplog, tmpdir): """If user_data file is a jinja template render instance-data vars.""" user_data = tmpdir.join("user-data") @@ -103,7 +102,6 @@ def test_renders_instance_data_vars_in_template(self, caplog, tmpdir): ) assert "rendering: jinja worked" == m_stdout.getvalue() - @skipUnlessJinja() def test_render_warns_and_gives_up_on_invalid_jinja_operation( self, caplog, tmpdir ): @@ -119,7 +117,6 @@ def test_render_warns_and_gives_up_on_invalid_jinja_operation( ' "my_var"?' % user_data ) in caplog.text - @skipUnlessJinja() def test_jinja_load_error(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "##template: jinja\nrendering: {{ my-var }}") @@ -130,7 +127,6 @@ def test_jinja_load_error(self, caplog, tmpdir): "Cannot render from instance data due to exception" in caplog.text ) - @skipUnlessJinja() def test_not_jinja_error(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "{{ my-var }}") @@ -141,7 +137,6 @@ def test_not_jinja_error(self, caplog, tmpdir): "Cannot render from instance data due to exception" in caplog.text ) - @skipUnlessJinja() def test_no_user_data(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "##template: jinja") @@ -150,7 +145,6 @@ def test_no_user_data(self, caplog, tmpdir): render.render_template(user_data, instance_data, False) assert "Unable to render user-data file" in caplog.text - @skipUnlessJinja() def test_invalid_jinja_syntax(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "##template: jinja\nrendering: {{ my_var } }") diff --git a/tests/unittests/cmd/test_cloud_id.py b/tests/unittests/cmd/test_cloud_id.py index d01c6efa283..671a945de62 100644 --- a/tests/unittests/cmd/test_cloud_id.py +++ b/tests/unittests/cmd/test_cloud_id.py @@ -2,12 +2,13 @@ """Tests for cloud-id command line utility.""" +from unittest import mock + import pytest from cloudinit import atomic_helper from cloudinit.cmd import cloud_id, status from cloudinit.helpers import Paths -from tests.unittests.helpers import mock M_PATH = "cloudinit.cmd.cloud_id." diff --git a/tests/unittests/cmd/test_main.py b/tests/unittests/cmd/test_main.py index bcf93bec395..1dd09bc6a38 100644 --- a/tests/unittests/cmd/test_main.py +++ b/tests/unittests/cmd/test_main.py @@ -116,7 +116,7 @@ def test_main_init_run_net_runs_modules( supplemental_config_file.write( EXTRA_CLOUD_CONFIG.format(tmpdir=tmpdir) ) - files = [open(supplemental_config_file)] + files = [supplemental_config_file] else: files = None cmdargs = MyArgs( diff --git a/tests/unittests/cmd/test_query.py b/tests/unittests/cmd/test_query.py index d552f8bc399..9f22c504ee8 100644 --- a/tests/unittests/cmd/test_query.py +++ b/tests/unittests/cmd/test_query.py @@ -8,6 +8,7 @@ from io import BytesIO from pathlib import Path from textwrap import dedent +from unittest import mock import pytest @@ -17,7 +18,6 @@ from cloudinit.sources import REDACT_SENSITIVE_VALUE from cloudinit.templater import JinjaSyntaxParsingException from cloudinit.util import write_file -from tests.unittests.helpers import mock M_PATH = "cloudinit.cmd.query." @@ -258,6 +258,7 @@ def test_handle_args_root_fallsback_to_instance_data(self, caplog, tmpdir): (_gzip_data(b"ud"), "ud", _gzip_data(b"vd"), "vd"), (_gzip_data("ud".encode("utf-8")), "ud", _gzip_data(b"vd"), "vd"), ), + ids=("plain-text", "bytes", "gzip-bytes", "gzip-encoded-bytes"), ) def test_handle_args_root_processes_user_data( self, ud_src, ud_expected, vd_src, vd_expected, capsys, tmpdir diff --git a/tests/unittests/cmd/test_status.py b/tests/unittests/cmd/test_status.py index 022e4034caa..d75e5735958 100644 --- a/tests/unittests/cmd/test_status.py +++ b/tests/unittests/cmd/test_status.py @@ -937,7 +937,6 @@ def common_mocks(self, mocker): # Because of this I'm only testing SubState combinations seen # in real-world testing (or using "any" string if we dont care). ("activating", "enabled", "start", "123", False), - ("activating", "enabled", "start", "123", False), ("active", "enabled-runtime", "exited", "0", False), ("active", "enabled", "exited", "0", False), ("active", "enabled", "running", "345", False), diff --git a/tests/unittests/config/test_apt_source_v3.py b/tests/unittests/config/test_apt_source_v3.py index 2e766d77ed5..b7c76ded350 100644 --- a/tests/unittests/config/test_apt_source_v3.py +++ b/tests/unittests/config/test_apt_source_v3.py @@ -927,20 +927,32 @@ def test_apt_v3_url_resolvable(self): # former tests can leave this set (or not if the test is ran directly) # do a hard reset to ensure a stable result util._DNS_REDIRECT_IP = None + badnames = ( + "does-not-exist.example.com.", + "example.invalid.", + "__cloud_init_expected_not_found__", + ) bad = [(None, None, None, "badname", ["10.3.2.1"])] good = [(None, None, None, "goodname", ["10.2.3.4"])] with mock.patch.object( - socket, "getaddrinfo", side_effect=[bad, bad, bad, good, good] + socket, "getaddrinfo", side_effect=[good, bad, bad, bad] ) as mocksock: ret = util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") - ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu") - mocksock.assert_any_call( - "does-not-exist.example.com.", None, 0, 0, 1, 2 - ) - mocksock.assert_any_call("example.invalid.", None, 0, 0, 1, 2) + for badname in badnames: + mocksock.assert_any_call(badname, None, 0, 0, 1, 2) mocksock.assert_any_call("us.archive.ubuntu.com", None) - assert ret is True + + # IP addresses skip DNS checks entirely + with mock.patch.object(socket, "getaddrinfo") as mocksock: + ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu") + mocksock.assert_not_called() + # Verify badnames were NOT checked for IP addresses + for badname in badnames: + assert ( + mock.call(badname, None, 0, 0, 1, 2) + not in mocksock.call_args_list + ) assert ret2 is True # side effect need only bad ret after initial call @@ -952,6 +964,15 @@ def test_apt_v3_url_resolvable(self): mocksock.assert_has_calls(calls) assert ret3 is False + # Test unresolvable hostname + with mock.patch.object( + socket, "getaddrinfo", side_effect=[bad] + ) as mocksock: + ret4 = util.is_resolvable_url("http://instance.:3336") + calls = [call("instance.", None)] + mocksock.assert_has_calls(calls) + assert ret4 is False + def test_apt_v3_disable_suites(self): """test_disable_suites - disable_suites with many configurations""" release = "xenial" diff --git a/tests/unittests/config/test_cc_ansible.py b/tests/unittests/config/test_cc_ansible.py index 3f3353b25d7..9c6a6bca1e7 100644 --- a/tests/unittests/config/test_cc_ansible.py +++ b/tests/unittests/config/test_cc_ansible.py @@ -281,7 +281,7 @@ def test_schema_validation_deprecations(self, config, error_msg): } }, "'playbook_name' is a required property", - id="require-url-dict", + id="require-playbook-name-dict", ), param( CFG_MINIMAL_LIST, @@ -312,7 +312,7 @@ def test_schema_validation_deprecations(self, config, error_msg): param( CFG_CTRL, None, - id="ctrl-keys", + id="ctrl-keys-list", ), param( { @@ -355,7 +355,7 @@ def test_schema_validation_deprecations(self, config, error_msg): } }, "'playbook_name' is a required property", - id="require-url-list", + id="require-playbook-name-list", ), ), ) diff --git a/tests/unittests/config/test_cc_apt_pipelining.py b/tests/unittests/config/test_cc_apt_pipelining.py index 2085a81f3df..6fb2926aa31 100644 --- a/tests/unittests/config/test_cc_apt_pipelining.py +++ b/tests/unittests/config/test_cc_apt_pipelining.py @@ -3,6 +3,7 @@ """Tests cc_apt_pipelining handler""" import re +from unittest import mock import pytest @@ -12,7 +13,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema class TestAptPipelining: diff --git a/tests/unittests/config/test_cc_disable_ec2_metadata.py b/tests/unittests/config/test_cc_disable_ec2_metadata.py index 55a23060a26..03afcf0ca18 100644 --- a/tests/unittests/config/test_cc_disable_ec2_metadata.py +++ b/tests/unittests/config/test_cc_disable_ec2_metadata.py @@ -3,6 +3,8 @@ """Tests cc_disable_ec2_metadata handler""" +from unittest import mock + import pytest import cloudinit.config.cc_disable_ec2_metadata as ec2_meta @@ -11,7 +13,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema DISABLE_CFG = {"disable_ec2_metadata": "true"} diff --git a/tests/unittests/config/test_cc_disk_setup.py b/tests/unittests/config/test_cc_disk_setup.py index b81a3a97e92..4951848a199 100644 --- a/tests/unittests/config/test_cc_disk_setup.py +++ b/tests/unittests/config/test_cc_disk_setup.py @@ -4,6 +4,7 @@ import random import tempfile from contextlib import ExitStack +from unittest import mock import pytest @@ -13,7 +14,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema class TestIsDiskUsed: @@ -137,6 +138,25 @@ def test_simple1_gpt(self, *args): "gpt", "/dev/xvdb1", [(100, Linux_GUID)] ) + @mock.patch( + "cloudinit.config.cc_disk_setup.subp.subp", + return_value=( + "", + "/dev/sdb: does not contain a recognized partition table", + ), + ) + def test_empty_disk_no_partition_table(self, m_subp): + """Test that empty disk (no partition table) returns empty list.""" + result = cc_disk_setup.check_partition_gpt_layout_sfdisk( + "/dev/sdb", [] + ) + assert result == [] + m_subp.assert_called_once_with( + ["sfdisk", "-l", "-J", "/dev/sdb"], + update_env={"LANG": "C"}, + rcs=[0, 1], + ) + class TestUpdateFsSetupDevices: def test_regression_1634678(self): diff --git a/tests/unittests/config/test_cc_keyboard.py b/tests/unittests/config/test_cc_keyboard.py index d0731a54075..39ae14eea61 100644 --- a/tests/unittests/config/test_cc_keyboard.py +++ b/tests/unittests/config/test_cc_keyboard.py @@ -2,7 +2,6 @@ """Tests cc_keyboard module""" -import os import re from unittest import mock @@ -14,7 +13,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import populate_dir, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -78,7 +77,7 @@ def test_schema_validation(self, config, error_msg): validate_cloudconfig_schema(config, schema, strict=True) -@pytest.mark.usefixtures("fake_filesystem") +@pytest.mark.usefixtures("fake_fs") class TestKeyboard: @mock.patch("cloudinit.distros.Distro.uses_systemd") @mock.patch("cloudinit.distros.subp.subp") @@ -122,7 +121,7 @@ def test_debian_linux_cmd(self, m_subp, m_write_file): ) @mock.patch("cloudinit.distros.subp.subp") - def test_alpine_linux_cmd(self, m_subp, tmpdir): + def test_alpine_linux_cmd(self, m_subp, fake_fs): """Alpine Linux runs setup-keymap""" cfg = {"keyboard": {"layout": "us", "variant": "us"}} layout = "us" @@ -133,14 +132,13 @@ def test_alpine_linux_cmd(self, m_subp, tmpdir): keymap_dir = "/usr/share/bkeymaps/%s" % "us" keymap_file = "%s/%s.bmap.gz" % (keymap_dir, "us") - os.makedirs(tmpdir.join(keymap_dir)) - populate_dir(str(tmpdir), {keymap_file: "# Test\n"}) + fake_fs.create_file(keymap_file) cc_keyboard.handle("cc_keyboard", cfg, cloud, []) m_subp.assert_called_once_with(["setup-keymap", layout, variant]) @mock.patch("cloudinit.distros.subp.subp") - def test_alpine_linux_ignore_model(self, m_subp, caplog, tmpdir): + def test_alpine_linux_ignore_model(self, m_subp, caplog, fake_fs): """Alpine Linux ignores model setting""" cfg = { "keyboard": { @@ -155,8 +153,7 @@ def test_alpine_linux_ignore_model(self, m_subp, caplog, tmpdir): keymap_dir = "/usr/share/bkeymaps/%s" % "us" keymap_file = "%s/%s.bmap.gz" % (keymap_dir, "us") - os.makedirs(tmpdir.join(keymap_dir)) - populate_dir(str(tmpdir), {keymap_file: "# Test\n"}) + fake_fs.create_file(keymap_file) cc_keyboard.handle("cc_keyboard", cfg, cloud, []) assert "Keyboard model is ignored for Alpine Linux." in caplog.text diff --git a/tests/unittests/config/test_cc_keys_to_console.py b/tests/unittests/config/test_cc_keys_to_console.py index caa2fa7bba1..64ca6367e40 100644 --- a/tests/unittests/config/test_cc_keys_to_console.py +++ b/tests/unittests/config/test_cc_keys_to_console.py @@ -1,6 +1,7 @@ """Tests for cc_keys_to_console.""" import re +from unittest import mock import pytest @@ -10,7 +11,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema class TestHandle: diff --git a/tests/unittests/config/test_cc_landscape.py b/tests/unittests/config/test_cc_landscape.py index 559702a33a8..07edc426718 100644 --- a/tests/unittests/config/test_cc_landscape.py +++ b/tests/unittests/config/test_cc_landscape.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +from unittest import mock import pytest @@ -10,7 +11,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema, wrap_and_call +from tests.unittests.helpers import skipUnlessJsonSchema, wrap_and_call from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) diff --git a/tests/unittests/config/test_cc_locale.py b/tests/unittests/config/test_cc_locale.py index fb564323adc..22df4fb3f25 100644 --- a/tests/unittests/config/test_cc_locale.py +++ b/tests/unittests/config/test_cc_locale.py @@ -1,7 +1,3 @@ -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Juerg Haefliger -# # This file is part of cloud-init. See LICENSE file for license information. import logging from io import BytesIO @@ -48,7 +44,8 @@ def test_set_locale_arch(self): contents = util.load_text_file(cc.distro.locale_gen_fn) assert "%s UTF-8" % locale in contents m_subp.assert_called_with( - ["localectl", "set-locale", locale], capture=False + ["localectl", "set-locale", locale], + capture=False, ) @pytest.mark.parametrize( @@ -122,6 +119,11 @@ def test_locale_update_config_if_different_than_default(self, tmpdir): "--locale-file=%s" % locale_conf.strpath, "LANG=C.UTF-8", ], + update_env={ + "LANG": "C.UTF-8", + "LANGUAGE": "C.UTF-8", + "LC_ALL": "C.UTF-8", + }, capture=False, ) m_which.assert_called_once_with("update-locale") diff --git a/tests/unittests/config/test_cc_mounts.py b/tests/unittests/config/test_cc_mounts.py index 549c59cf0cf..80dbf5c6e11 100644 --- a/tests/unittests/config/test_cc_mounts.py +++ b/tests/unittests/config/test_cc_mounts.py @@ -30,107 +30,87 @@ class TestSanitizeDevname: - def _touch(self, path, new_root): - path = os.path.join(new_root, path.lstrip("/")) - basedir = os.path.dirname(path) - if not os.path.exists(basedir): - os.makedirs(basedir) - open(path, "a").close() - - def _makedirs(self, directory, new_root): - directory = os.path.join(new_root, directory.lstrip("/")) - if not os.path.exists(directory): - os.makedirs(directory) - def mock_existence_of_disk(self, disk_path, new_root): - self._touch(disk_path, new_root) - self._makedirs( - os.path.join("/sys/block", disk_path.split("/")[-1]), new_root - ) + # E.g., /sys/block/sda + path = "/sys/block/" + disk_path.split("/")[-1] + new_root.create_dir(path) def mock_existence_of_partition( self, disk_path, partition_number, new_root ): self.mock_existence_of_disk(disk_path, new_root) - self._touch(disk_path + str(partition_number), new_root) + # E.g., /dev/sda1 + dev_path = disk_path + str(partition_number) + new_root.create_file(dev_path) disk_name = disk_path.split("/")[-1] - self._makedirs( - os.path.join( - "/sys/block", disk_name, disk_name + str(partition_number) - ), - new_root, + # E.g., /sys/block/sda/sda1 + block_path = os.path.join( + "/sys/block", disk_name, disk_name + str(partition_number) ) + new_root.create_dir(block_path) - def test_existent_full_disk_path_is_returned(self, fake_filesystem): + def test_existent_full_disk_path_is_returned(self, fake_fs): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert disk_path == cc_mounts.sanitize_devname( disk_path, lambda x: None ) - def test_existent_disk_name_returns_full_path(self, fake_filesystem): + def test_existent_disk_name_returns_full_path(self, fake_fs): disk_name = "sda" disk_path = "/dev/" + disk_name - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert disk_path == cc_mounts.sanitize_devname( disk_name, lambda x: None ) - def test_existent_meta_disk_is_returned(self, fake_filesystem): + def test_existent_meta_disk_is_returned(self, fake_fs): actual_disk_path = "/dev/sda" - self.mock_existence_of_disk(actual_disk_path, fake_filesystem) + self.mock_existence_of_disk(actual_disk_path, fake_fs) assert actual_disk_path == cc_mounts.sanitize_devname( "ephemeral0", lambda x: actual_disk_path, ) - def test_existent_meta_partition_is_returned(self, fake_filesystem): + def test_existent_meta_partition_is_returned(self, fake_fs): disk_name, partition_part = "/dev/sda", "1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0.1", lambda x: disk_name, ) - def test_existent_meta_partition_with_p_is_returned(self, fake_filesystem): + def test_existent_meta_partition_with_p_is_returned(self, fake_fs): disk_name, partition_part = "/dev/sda", "p1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0.1", lambda x: disk_name, ) def test_first_partition_returned_if_existent_disk_is_partitioned( - self, fake_filesystem + self, fake_fs ): disk_name, partition_part = "/dev/sda", "1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0", lambda x: disk_name, ) - def test_nth_partition_returned_if_requested(self, fake_filesystem): + def test_nth_partition_returned_if_requested(self, fake_fs): disk_name, partition_part = "/dev/sda", "3" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0.3", lambda x: disk_name, ) - def test_transformer_returning_none_returns_none(self, fake_filesystem): + def test_transformer_returning_none_returns_none(self, fake_fs): assert ( cc_mounts.sanitize_devname( "ephemeral0", @@ -139,7 +119,7 @@ def test_transformer_returning_none_returns_none(self, fake_filesystem): is None ) - def test_missing_device_returns_none(self, fake_filesystem): + def test_missing_device_returns_none(self, fake_fs): assert ( cc_mounts.sanitize_devname( "/dev/sda", @@ -148,9 +128,9 @@ def test_missing_device_returns_none(self, fake_filesystem): is None ) - def test_missing_sys_returns_none(self, fake_filesystem): + def test_missing_sys_returns_none(self, fake_fs): disk_path = "/dev/sda" - self._makedirs(disk_path, fake_filesystem) + fake_fs.create_dir(disk_path) assert ( cc_mounts.sanitize_devname( disk_path, @@ -159,11 +139,9 @@ def test_missing_sys_returns_none(self, fake_filesystem): is None ) - def test_existent_disk_but_missing_partition_returns_none( - self, fake_filesystem - ): + def test_existent_disk_but_missing_partition_returns_none(self, fake_fs): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert ( cc_mounts.sanitize_devname( "ephemeral0.1", @@ -172,16 +150,16 @@ def test_existent_disk_but_missing_partition_returns_none( is None ) - def test_network_device_returns_network_device(self, fake_filesystem): + def test_network_device_returns_network_device(self, fake_fs): disk_path = "netdevice:/path" assert disk_path == cc_mounts.sanitize_devname( disk_path, None, ) - def test_device_aliases_remapping(self, fake_filesystem): + def test_device_aliases_remapping(self, fake_fs): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert disk_path == cc_mounts.sanitize_devname( "mydata", lambda x: None, {"mydata": disk_path} ) @@ -189,13 +167,10 @@ def test_device_aliases_remapping(self, fake_filesystem): class TestSwapFileCreation: @pytest.fixture(autouse=True) - def setup(self, mocker, fake_filesystem: str): - self.new_root = fake_filesystem - self.swap_path = os.path.join(fake_filesystem, "swap.img") - fstab_path = os.path.join(fake_filesystem, "etc/fstab") - self._makedirs("/etc") + def setup(self, mocker, fake_fs): + self.swap_path = "/swap.img" + fake_fs.create_dir("/etc") - self.m_fstab = mocker.patch(f"{M_PATH}FSTAB_PATH", fstab_path) self.m_subp = mocker.patch(f"{M_PATH}subp.subp") self.m_mounts = mocker.patch( f"{M_PATH}util.mounts", @@ -220,11 +195,6 @@ def setup(self, mocker, fake_filesystem: str): } } - def _makedirs(self, directory): - directory = os.path.join(self.new_root, directory.lstrip("/")) - if not os.path.exists(directory): - os.makedirs(directory) - def device_name_to_device(self, path): if path == "swap": return self.swap_path @@ -324,13 +294,9 @@ class TestFstabHandling: swap_path = "/dev/sdb1" @pytest.fixture(autouse=True) - def setup(self, mocker, fake_filesystem: str): - self.new_root = fake_filesystem + def setup(self, mocker, fake_fs): + fake_fs.create_dir("/etc") - self.fstab_path = os.path.join(self.new_root, "etc/fstab") - self._makedirs("/etc") - - self.m_fstab = mocker.patch(f"{M_PATH}FSTAB_PATH", self.fstab_path) self.m_subp = mocker.patch(f"{M_PATH}subp.subp") self.m_mounts = mocker.patch( f"{M_PATH}util.mounts", @@ -351,11 +317,6 @@ def setup(self, mocker, fake_filesystem: str): self.mock_log = mock.Mock() self.mock_cloud.device_name_to_device = self.device_name_to_device - def _makedirs(self, directory): - directory = os.path.join(self.new_root, directory.lstrip("/")) - if not os.path.exists(directory): - os.makedirs(directory) - def device_name_to_device(self, path): if path == "swap": return self.swap_path diff --git a/tests/unittests/config/test_cc_ntp.py b/tests/unittests/config/test_cc_ntp.py index aea0c40243c..5cfe9f302bc 100644 --- a/tests/unittests/config/test_cc_ntp.py +++ b/tests/unittests/config/test_cc_ntp.py @@ -5,6 +5,7 @@ import shutil from os.path import dirname from typing import Any, Dict, List +from unittest import mock import pytest @@ -15,7 +16,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud NTP_TEMPLATE = """\ diff --git a/tests/unittests/config/test_cc_phone_home.py b/tests/unittests/config/test_cc_phone_home.py index 77428fcedc1..d44f365cff2 100644 --- a/tests/unittests/config/test_cc_phone_home.py +++ b/tests/unittests/config/test_cc_phone_home.py @@ -54,7 +54,9 @@ def test_no_url(self, m_readurl, caplog): (0, -1), (1, 0), (2, 1), - ("2", 1), + # override parametrized id to differentiate str "2" from int 2 in + # former test GH pytest-dev/pytest#14650. + pytest.param("2", 1, id="retries-as-int-str"), ("two", 9), (None, 9), ({}, 9), diff --git a/tests/unittests/config/test_cc_power_state_change.py b/tests/unittests/config/test_cc_power_state_change.py index 37901baaab7..6c159b3bfd4 100644 --- a/tests/unittests/config/test_cc_power_state_change.py +++ b/tests/unittests/config/test_cc_power_state_change.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import sys +from unittest import mock import pytest @@ -11,7 +12,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema @pytest.fixture diff --git a/tests/unittests/config/test_cc_puppet.py b/tests/unittests/config/test_cc_puppet.py index 22911542d2a..a3f04fb8fe7 100644 --- a/tests/unittests/config/test_cc_puppet.py +++ b/tests/unittests/config/test_cc_puppet.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import textwrap +from unittest import mock import pytest import responses @@ -13,7 +14,7 @@ ) from cloudinit.distros import PackageInstallerError from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -72,11 +73,11 @@ def test_enable_fallback_on_failure(self, m_subp): assert expected_calls == m_subp.call_args_list -@pytest.mark.usefixtures("fake_filesystem") +@pytest.mark.usefixtures("fake_fs") @mock.patch("cloudinit.config.cc_puppet._manage_puppet_services") class TestPuppetHandle: - CONF = "puppet.conf" - CSR_ATTRIBUTES_PATH = "csr_attributes.yaml" + CONF = "/etc/puppet/puppet.conf" + CSR_ATTRIBUTES_PATH = "/etc/puppet/csr_attributes.yaml" def test_skips_missing_puppet_key_in_cloudconfig( self, m_man_puppet, caplog @@ -237,21 +238,14 @@ def test_puppet_config_installs_puppet_version(self, m_subp, _): mock.call([["puppet-agent", "3.8"]]) ] == cloud.distro.install_packages.call_args_list - @mock.patch("cloudinit.config.cc_puppet.get_config_value") @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) - def test_puppet_config_updates_puppet_conf( - self, m_subp, m_default, m_man_puppet - ): + def test_puppet_config_updates_puppet_conf(self, m_subp, m_man_puppet): """When 'conf' is provided update values in PUPPET_CONF_PATH.""" - def _fake_get_config_value(puppet_bin, setting): - return self.CONF - - m_default.side_effect = _fake_get_config_value - cfg = { "puppet": { - "conf": {"agent": {"server": "puppetserver.example.org"}} + "conf": {"agent": {"server": "puppetserver.example.org"}}, + "conf_file": self.CONF, } } util.write_file(self.CONF, "[agent]\nserver = origpuppet\nother = 3") diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index cdfe97c6a70..b32695743d3 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest import cloudinit.config.cc_raspberry_pi as cc_rpi @@ -15,7 +17,7 @@ validate_cloudconfig_schema, ) from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud M_PATH = "cloudinit.config.cc_raspberry_pi." @@ -111,7 +113,7 @@ def test_configure_usb_gadget_enable(self, m_subp): with mock.patch("os.path.exists", return_value=True): cc_rpi.configure_usb_gadget(True) m_subp.assert_called_once_with( - [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=15 + [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=30 ) @mock.patch("cloudinit.subp.subp") @@ -150,7 +152,7 @@ def test_configure_usb_gadget_script_failure(self, m_subp, caplog): # Subprocess should have been invoked once m_subp.assert_called_once_with( - [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=15 + [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=30 ) # Error log should contain failure message diff --git a/tests/unittests/config/test_cc_rh_subscription.py b/tests/unittests/config/test_cc_rh_subscription.py index a9b95072afc..b8323ed6449 100644 --- a/tests/unittests/config/test_cc_rh_subscription.py +++ b/tests/unittests/config/test_cc_rh_subscription.py @@ -116,7 +116,7 @@ def test_update_repos_disable_with_none( mock.call(["repos", "--enable=repo1"]) ] - def test_full_registration(self, m_sman_cli, caplog): + def test_full_registration(self, m_sman_cli, caplog, mocker): """ Registration with auto_attach, service_level, adding pools, enabling and disabling yum repos and setting release_version @@ -147,7 +147,7 @@ def test_full_registration(self, m_sman_cli, caplog): # to avoid deleting the actual cache files # (triggered by the presence of the release_version key) # on the host running the tests - mock.patch("shutil.rmtree") + mocker.patch("shutil.rmtree") cc_rh_subscription.handle(NAME, self.CONFIG_FULL, None, []) assert m_sman_cli.call_count == 10 diff --git a/tests/unittests/config/test_cc_set_passwords.py b/tests/unittests/config/test_cc_set_passwords.py index a706917d731..ad112c95fbe 100644 --- a/tests/unittests/config/test_cc_set_passwords.py +++ b/tests/unittests/config/test_cc_set_passwords.py @@ -581,8 +581,6 @@ def _get_str_class_num(self, str): (3, ValueError), (4, 4), (5, 4), - (5, 4), - (6, 4), (20, 4), ], ) diff --git a/tests/unittests/config/test_cc_ssh_import_id.py b/tests/unittests/config/test_cc_ssh_import_id.py index 2d5608a6464..2daffdd9997 100644 --- a/tests/unittests/config/test_cc_ssh_import_id.py +++ b/tests/unittests/config/test_cc_ssh_import_id.py @@ -6,11 +6,12 @@ import pytest from cloudinit.config import cc_ssh_import_id +from cloudinit.subp import ProcessExecutionError from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) -MODPATH = "cloudinit.config.cc_ssh_import_ids." +MODPATH = "cloudinit.config.cc_ssh_import_id." class TestIsKeyInNestedDict: @@ -69,7 +70,7 @@ class TestHandleSshImportIDs: ({"ssh_import_id": ["bobkey"]}, "ssh-import-id is not installed"), ), ) - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "subp.which") def test_skip_inapplicable_configs(self, m_which, cfg, log, caplog): """Skip config without ssh_import_id""" m_which.return_value = None @@ -77,9 +78,9 @@ def test_skip_inapplicable_configs(self, m_which, cfg, log, caplog): cc_ssh_import_id.handle("name", cfg, cloud, []) assert log in caplog.text - @mock.patch("cloudinit.ssh_util.pwd.getpwnam") - @mock.patch("cloudinit.config.cc_ssh_import_id.subp.subp") - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.subp") + @mock.patch(MODPATH + "subp.which") def test_use_sudo(self, m_which, m_subp, m_getpwnam): """Check that sudo is available and use that""" m_which.return_value = "/usr/bin/ssh-import-id" @@ -98,9 +99,9 @@ def test_use_sudo(self, m_which, m_subp, m_getpwnam): capture=False, ) - @mock.patch("cloudinit.ssh_util.pwd.getpwnam") - @mock.patch("cloudinit.config.cc_ssh_import_id.subp.subp") - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.subp") + @mock.patch(MODPATH + "subp.which") def test_use_doas(self, m_which, m_subp, m_getpwnam): """Check that doas is available and use that""" m_which.side_effect = [None, "/usr/bin/doas"] @@ -111,9 +112,9 @@ def test_use_doas(self, m_which, m_subp, m_getpwnam): ["doas", "-u", user, "ssh-import-id"] + ids, capture=False ) - @mock.patch("cloudinit.ssh_util.pwd.getpwnam") - @mock.patch("cloudinit.config.cc_ssh_import_id.subp.subp") - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.subp") + @mock.patch(MODPATH + "subp.which") def test_use_neither_sudo_nor_doas( self, m_which, m_subp, m_getpwnam, caplog ): @@ -125,3 +126,45 @@ def test_use_neither_sudo_nor_doas( assert ( "Neither sudo nor doas available! Unable to import SSH ids" ) in caplog.text + + @mock.patch(MODPATH + "time.sleep") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.which") + def test_retry_once_on_exit_code_1( + self, m_which, m_getpwnam, m_sleep, mocker + ): + """Only attempt one retry when ssh-import-id exits with code 1.""" + m_subp = mocker.patch( + "cloudinit.config.cc_ssh_import_id.subp.subp", + side_effect=[ + ProcessExecutionError(exit_code=1, stderr="try1"), + ProcessExecutionError(exit_code=1, stderr="try2"), + ], + ) + m_which.return_value = "/usr/bin/ssh-import-id" + with pytest.raises( + ProcessExecutionError, + match=r"(?s)Unexpected error while running command.*try2", + ): + cc_ssh_import_id.import_ssh_ids(["waffle"], "bob") + + assert m_subp.call_count == 2 + assert m_sleep.call_count == 1 + + @mock.patch(MODPATH + "time.sleep") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.which") + def test_retry_with_success(self, m_which, m_getpwnam, m_sleep, mocker): + """Retry succeeds on ssh-import-id with a retry.""" + m_subp = mocker.patch( + "cloudinit.config.cc_ssh_import_id.subp.subp", + side_effect=[ + ProcessExecutionError(exit_code=1, stderr="try1"), + None, + ], + ) + m_which.return_value = "/usr/bin/ssh-import-id" + cc_ssh_import_id.import_ssh_ids(["waffle"], "bob") + + assert m_subp.call_count == 2 + assert m_sleep.call_count == 1 diff --git a/tests/unittests/config/test_cc_ubuntu_drivers.py b/tests/unittests/config/test_cc_ubuntu_drivers.py index 02854585ba5..8344dd1ab71 100644 --- a/tests/unittests/config/test_cc_ubuntu_drivers.py +++ b/tests/unittests/config/test_cc_ubuntu_drivers.py @@ -5,6 +5,7 @@ import os import re from typing import Any, Dict +from unittest import mock import pytest @@ -15,7 +16,7 @@ validate_cloudconfig_schema, ) from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" diff --git a/tests/unittests/config/test_cc_update_etc_hosts.py b/tests/unittests/config/test_cc_update_etc_hosts.py index ebf6f599e1e..48f985cc217 100644 --- a/tests/unittests/config/test_cc_update_etc_hosts.py +++ b/tests/unittests/config/test_cc_update_etc_hosts.py @@ -1,13 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging -import os import re -import shutil import pytest +from pyfakefs.fake_filesystem import FakeFilesystem -from cloudinit import cloud, distros, helpers, util +from cloudinit import util from cloudinit.config import cc_update_etc_hosts from cloudinit.config.schema import ( SchemaValidationError, @@ -16,44 +15,34 @@ ) from tests.helpers import cloud_init_project_dir from tests.unittests import helpers as t_help +from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) -@pytest.fixture(autouse=True) -def with_templates(tmp_path, fake_filesystem_hook): - shutil.copytree( - str(cloud_init_project_dir("templates")), - str(tmp_path / "templates"), - dirs_exist_ok=True, +@pytest.fixture +def with_templates(fake_fs: FakeFilesystem): + fake_fs.add_real_directory( + cloud_init_project_dir("templates"), + target_path="/etc/cloud/templates", + read_only=True, ) -@pytest.mark.usefixtures("fake_filesystem") +@pytest.mark.usefixtures("with_templates") class TestHostsFile: - def _fetch_distro(self, kind): - cls = distros.fetch(kind) - paths = helpers.Paths({}) - return cls(kind, {}, paths) - - def test_write_etc_hosts_suse_localhost(self, tmp_path): + def test_write_etc_hosts_suse_localhost(self, fake_fs: FakeFilesystem): cfg = { "manage_etc_hosts": "localhost", "hostname": "cloud-init.test.us", } - os.makedirs(tmp_path / "etc/") + hosts_path = "/etc/hosts" hosts_content = "192.168.1.1 blah.blah.us blah\n" - etc_hosts = str(tmp_path / "etc/hosts") - fout = open(etc_hosts, "w") - fout.write(hosts_content) - fout.close() - distro = self._fetch_distro("sles") - distro.hosts_fn = etc_hosts - paths = helpers.Paths({}) - ds = None - cc = cloud.Cloud(ds, paths, {}, distro, None) + fake_fs.create_file(hosts_path, contents=hosts_content) + cc = get_cloud("sles") + cc.distro.hosts_fn = hosts_path cc_update_etc_hosts.handle("test", cfg, cc, []) - contents = util.load_text_file(etc_hosts) + contents = util.load_text_file(hosts_path) assert ( "127.0.1.1\tcloud-init.test.us\tcloud-init" in contents ), "No entry for 127.0.1.1 in etc/hosts" @@ -61,22 +50,14 @@ def test_write_etc_hosts_suse_localhost(self, tmp_path): "192.168.1.1\tblah.blah.us\tblah" in contents ), "Default etc/hosts content modified" - @t_help.skipUnlessJinja() - def test_write_etc_hosts_suse_template(self, tmp_path): + def test_write_etc_hosts_suse_template(self, fake_fs): cfg = { "manage_etc_hosts": "template", "hostname": "cloud-init.test.us", } - shutil.copytree( - tmp_path / "templates", str(tmp_path / "etc/cloud/templates") - ) - distro = self._fetch_distro("sles") - paths = helpers.Paths({}) - paths.template_tpl = str(tmp_path / "etc/cloud/templates/%s.tmpl") - ds = None - cc = cloud.Cloud(ds, paths, {}, distro, None) + cc = get_cloud("sles") cc_update_etc_hosts.handle("test", cfg, cc, []) - contents = util.load_text_file(tmp_path / "etc/hosts") + contents = util.load_text_file("/etc/hosts") assert ( "127.0.1.1 cloud-init.test.us cloud-init" in contents ), "No entry for 127.0.1.1 in etc/hosts" @@ -116,7 +97,6 @@ class TestUpdateEtcHosts: ), ], ) - @t_help.skipUnlessJsonSchema() def test_schema_validation(self, config, expectation): with expectation: validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_wireguard.py b/tests/unittests/config/test_cc_wireguard.py index 6702eb9385a..b87851aaa7e 100644 --- a/tests/unittests/config/test_cc_wireguard.py +++ b/tests/unittests/config/test_cc_wireguard.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +from unittest import mock import pytest @@ -10,7 +11,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema NL = "\n" # Module path used in mocks diff --git a/tests/unittests/config/test_modules.py b/tests/unittests/config/test_modules.py index d986442e250..5dc542b64fe 100644 --- a/tests/unittests/config/test_modules.py +++ b/tests/unittests/config/test_modules.py @@ -6,6 +6,7 @@ import logging from pathlib import Path from typing import List +from unittest import mock import pytest @@ -16,7 +17,6 @@ from cloudinit.settings import FREQUENCIES from cloudinit.stages import Init from tests.helpers import cloud_init_project_dir -from tests.unittests.helpers import mock M_PATH = "cloudinit.config.modules." diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 797403bc39f..cc87d2ce825 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -451,8 +451,8 @@ def test_validateconfig_schema_non_strict_emits_warnings(self, caplog): """When strict is False validate_cloudconfig_schema emits warnings.""" schema = {"properties": {"p1": {"type": "string"}}} validate_cloudconfig_schema({"p1": -1}, schema=schema, strict=False) - assert ( - caplog.record_tuples and len(caplog.record_tuples) == 1 + assert caplog.record_tuples and ( + len(caplog.record_tuples) == 1 or len(caplog.record_tuples) == 2 ), caplog.record_tuples [(module, log_level, log_msg)] = caplog.record_tuples assert "cloudinit.config.schema" == module @@ -1758,7 +1758,7 @@ class TestNetworkSchema: SchemaType.NETWORK_CONFIG_V1, does_not_raise(), "", - id="config_key_required", + id="config_key_empty_valid", ), pytest.param( { diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index 07d50ce1a20..76be658b384 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -6,6 +6,7 @@ from unittest import mock import pytest +from pyfakefs.fake_filesystem import FakeFilesystem from cloudinit import ( atomic_helper, @@ -119,7 +120,9 @@ def fake_filesystem_hook(): @pytest.fixture def fake_filesystem(mocker, tmpdir, fake_filesystem_hook): - """Mocks fs functions to operate under `tmpdir` + """This fixture is DEPRECATED for new tests. Use fake_fs instead. + + Mocks fs functions to operate under `tmpdir`. This fixture is sorted after fix_cloud_init_hook to allow fixtures sorted before fake_cloud_init_hook to access the real filesystem. @@ -148,6 +151,14 @@ def fake_filesystem(mocker, tmpdir, fake_filesystem_hook): yield str(tmpdir) +@pytest.fixture +def fake_fs(fs: FakeFilesystem): + """Mocks fs and pathlib functions to operate under a fake filesystem. + + See https://pytest-pyfakefs.readthedocs.io""" + yield fs + + @pytest.fixture(scope="session", autouse=True) def disable_sysfs_net(tmpdir_factory): """Avoid tests which read the underlying host's /syc/class/net.""" diff --git a/tests/unittests/distros/test_bsd_utils.py b/tests/unittests/distros/test_bsd_utils.py index f5a7efd8fc2..44351762f93 100644 --- a/tests/unittests/distros/test_bsd_utils.py +++ b/tests/unittests/distros/test_bsd_utils.py @@ -1,8 +1,9 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest import cloudinit.distros.bsd_utils as bsd_utils -from tests.unittests.helpers import mock RC_FILE = """ if something; then diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 095924b106b..a875472cd3d 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -750,8 +750,9 @@ def test_create_user_with_ssh_redirect_user_no_cloud_keys( dist.create_user(USER, ssh_redirect_user="someuser") assert caplog.records[1].levelname in ["WARNING", "DEPRECATED"] assert ( - "Unable to disable SSH logins for foo_user given " - "ssh_redirect_user: someuser. No cloud public-keys present.\n" + "Unable to disable SSH logins for foo_user." + " ssh_redirect_user was set to redirect logins to" + " someuser, but no cloud public-keys are present.\n" ) in caplog.text m_setup_user_keys.assert_not_called() diff --git a/tests/unittests/distros/test_debian.py b/tests/unittests/distros/test_debian.py index 9634d185ce2..1262615d3a4 100644 --- a/tests/unittests/distros/test_debian.py +++ b/tests/unittests/distros/test_debian.py @@ -67,7 +67,7 @@ def test_rerun_if_different(self, distro, m_subp, caplog): util.write_file(LOCALE_PATH, "LANG=fr_FR.UTF-8", omode="w") distro.apply_locale(locale, out_fn=LOCALE_PATH) assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", @@ -103,17 +103,20 @@ def test_rerun_if_no_file( with mock.patch.object(distro, "install_packages") as m_install: distro.apply_locale(locale, out_fn=LOCALE_PATH) assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", f"LANG={locale}", ], ] == [p[0][0] for p in m_subp.call_args_list] - assert [ - mock.call("locale-gen"), - mock.call("update-locale"), - ] == m_which.call_args_list + calls = [c.args[0] for c in m_which.call_args_list] + # collapse consecutive duplicates from _ensure_tool re-check + uniq: list[str] = [] + for name in calls: + if not uniq or uniq[-1] != name: + uniq.append(name) + assert uniq == ["locale-gen", "update-locale"] if install_pkgs: m_install.assert_called_with(install_pkgs) else: @@ -125,7 +128,7 @@ def test_rerun_on_unset_system_locale(self, distro, m_subp, caplog): util.write_file(LOCALE_PATH, "LANG=", omode="w") distro.apply_locale(locale, out_fn=LOCALE_PATH) assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", @@ -148,7 +151,7 @@ def test_rerun_on_mismatched_keys(self, distro, m_subp): util.write_file(LOCALE_PATH, "LANG=", omode="w") distro.apply_locale(locale, out_fn=LOCALE_PATH, keyname="LC_ALL") assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", @@ -170,3 +173,69 @@ def test_falseish_locale_raises_valueerror(self, distro, m_subp): ): distro.apply_locale("") m_subp.assert_not_called() + + +@pytest.mark.usefixtures("fake_filesystem") +class TestLookupSupportedI18nValue: + """Test _lookup_supported_i18n_value function.""" + + def test_no_match_constructs_default(self, mocker): + """When no match is found in SUPPORTED, construct a default line.""" + from cloudinit.distros.debian import _lookup_supported_i18n_value + + # Mock SUPPORTED file to be empty + mocker.patch( + "cloudinit.distros.debian.util.load_text_file", + side_effect=OSError("File not found"), + ) + + # Request a locale that won't be found + result = _lookup_supported_i18n_value("xyz_XY.UTF-8") + assert result == "xyz_XY.UTF-8 UTF-8" + + # Without charset, should default to UTF-8 + result = _lookup_supported_i18n_value("abc_AB") + assert result == "abc_AB.UTF-8 UTF-8" + + # With explicit charset + result = _lookup_supported_i18n_value("def_DE.ISO-8859-1") + assert result == "def_DE.ISO-8859-1 ISO-8859-1" + + def test_formats_with_charset_and_modifier(self, mocker): + """Test various locale formats: prefix, charset, and modifier.""" + from cloudinit.distros.debian import _lookup_supported_i18n_value + + # Mock SUPPORTED file with various locale formats + supported_content = """# Supported locales +en_US.UTF-8 UTF-8 +fi_FI.ISO-8859-1 ISO-8859-1 +fi_FI.UTF-8 UTF-8 +it_IT@euro ISO-8859-15 +ca_ES@valencia.UTF-8 UTF-8 +de_DE.UTF-8 UTF-8 + fr_FR.UTF-8 UTF-8 +""" + mocker.patch( + "cloudinit.distros.debian.util.load_text_file", + return_value=supported_content, + ) + + # Test modifier without explicit charset: it_IT@euro + result = _lookup_supported_i18n_value("it_IT@euro") + assert result == "it_IT@euro ISO-8859-15" + + # Test charset without modifier: fi_FI.ISO-8859-1 + result = _lookup_supported_i18n_value("fi_FI.ISO-8859-1") + assert result == "fi_FI.ISO-8859-1 ISO-8859-1" + + # Test both charset and modifier: ca_ES@valencia.UTF-8 + result = _lookup_supported_i18n_value("ca_ES@valencia.UTF-8") + assert result == "ca_ES@valencia.UTF-8 UTF-8" + + # Test bare locale preferring UTF-8 + result = _lookup_supported_i18n_value("fi_FI") + assert result == "fi_FI.UTF-8 UTF-8" + + # Test that lines with leading whitespace are matched (then stripped) + result = _lookup_supported_i18n_value("fr_FR.UTF-8") + assert result == "fr_FR.UTF-8 UTF-8" diff --git a/tests/unittests/distros/test_hosts.py b/tests/unittests/distros/test_hosts.py index 7fd5abf2bd9..30c00d9c08a 100644 --- a/tests/unittests/distros/test_hosts.py +++ b/tests/unittests/distros/test_hosts.py @@ -21,8 +21,8 @@ def test_parse(self): ["foo.mydomain.org", "foo"], ["bar.mydomain.org", "bar"], ] - eh = str(eh) - assert eh.startswith("# Example") + eh_str = str(eh) + assert eh_str.startswith("# Example") def test_add(self): eh = hosts.HostsConf(BASE_ETC) diff --git a/tests/unittests/distros/test_raspberry_pi_os.py b/tests/unittests/distros/test_raspberry_pi_os.py index 2ef36acf117..ba54cee2f10 100644 --- a/tests/unittests/distros/test_raspberry_pi_os.py +++ b/tests/unittests/distros/test_raspberry_pi_os.py @@ -1,10 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +from unittest import mock from cloudinit.distros import fetch from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock M_PATH = "cloudinit.distros.raspberry_pi_os." diff --git a/tests/unittests/distros/test_user_data_normalize.py b/tests/unittests/distros/test_user_data_normalize.py index a1a77d1a0e9..925a5a93855 100644 --- a/tests/unittests/distros/test_user_data_normalize.py +++ b/tests/unittests/distros/test_user_data_normalize.py @@ -196,6 +196,30 @@ def test_users_dict_default_additional(self): assert users["bob"]["blah"] is True assert users["bob"]["default"] is True + def test_users_dict_override_default_attribute(self): + distro = self._make_distro("ubuntu", bcfg) + ug_cfg = { + "users": ["default", {"name": "bob", "lock_passwd": False}], + } + users, _ = self._norm(ug_cfg, distro) + + assert "bob" in users + assert "name" not in users["bob"] + + for key, val in bcfg.items(): + if key == "lock_passwd": + # Assert that the default user config is True + assert val is True + # Assert that the resolved value + # matches the passed config: False + assert users["bob"][key] is False + elif key == "groups": + assert users["bob"][key] == ",".join(val) + elif key != "name": + assert users["bob"][key] == val + + assert users["bob"]["default"] is True + def test_users_dict_extract(self): distro = self._make_distro("ubuntu", bcfg) ug_cfg = { diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 429b999d339..63aeae955b1 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -10,15 +10,12 @@ import unittest from contextlib import contextmanager from unittest import mock -from unittest.util import strclass from urllib.parse import urlsplit, urlunsplit import pytest import responses from cloudinit import distros, helpers, settings, util -from cloudinit.helpers import Paths -from cloudinit.templater import JINJA_AVAILABLE from tests.helpers import cloud_init_project_dir try: @@ -114,42 +111,6 @@ def random_string(length=8): ) -# Note: The use of this class and unittests.TestCase is discouraged. Use pytest -# instead. See development docs on testing. -class TestCase(unittest.TestCase): - def reset_global_state(self): - """Reset any global state to its original settings. - - cloudinit caches some values in cloudinit.util. Unit tests that - involved those cached paths were then subject to failure if the order - of invocation changed (LP: #1703697). - - This function resets any of these global state variables to their - initial state. - - In the future this should really be done with some registry that - can then be cleaned in a more obvious way. - """ - util._DNS_REDIRECT_IP = None - - def setUp(self): - super(TestCase, self).setUp() - self.reset_global_state() - - def shortDescription(self): - return strclass(self.__class__) + "." + self._testMethodName - - def add_patch(self, target, attr, *args, **kwargs): - """Patches specified target object and sets it as attr on test - instance also schedules cleanup""" - if "autospec" not in kwargs: - kwargs["autospec"] = True - m = mock.patch(target, *args, **kwargs) - p = m.start() - self.addCleanup(m.stop) - setattr(self, attr, p) - - def replicate_test_root(example_root, target_root): real_root = resourceLocation() real_root = os.path.join(real_root, "roots", example_root) @@ -195,24 +156,6 @@ def _ensure_url_default_path(url): ) -def get_mock_paths(temp_dir): - class MockPaths(Paths): - def __init__(self, path_cfgs: dict, ds=None): - super().__init__(path_cfgs=path_cfgs, ds=ds) - - self.cloud_dir: str = path_cfgs.get( - "cloud_dir", f"{temp_dir}/var/lib/cloud" - ) - self.run_dir: str = path_cfgs.get( - "run_dir", f"{temp_dir}/run/cloud/" - ) - self.template_dir: str = path_cfgs.get( - "templates_dir", f"{temp_dir}/etc/cloud/templates/" - ) - - return MockPaths - - def populate_dir(path, files): if not os.path.exists(path): os.makedirs(path) @@ -340,13 +283,6 @@ def skipUnlessJsonSchema(): ) -def skipUnlessJinja(): - return pytest.mark.skipif( - not JINJA_AVAILABLE, reason="No jinja dependency present." - ) - - -@skipUnlessJinja() def skipUnlessJinjaVersionGreaterThan(version=(0, 0, 0)): import jinja2 @@ -356,27 +292,6 @@ def skipUnlessJinjaVersionGreaterThan(version=(0, 0, 0)): ) -def skipIfJinja(): - return pytest.mark.skipif( - JINJA_AVAILABLE, reason="Jinja dependency present." - ) - - -# older versions of mock do not have the useful 'assert_not_called' -if not hasattr(mock.Mock, "assert_not_called"): - - def __mock_assert_not_called(mmock): - if mmock.call_count != 0: - msg = ( - "[citest] Expected '%s' to not have been called. " - "Called %s times." - % (mmock._mock_name or "mock", mmock.call_count) - ) - raise AssertionError(msg) - - mock.Mock.assert_not_called = __mock_assert_not_called # type: ignore - - @contextmanager def does_not_raise(): """Context manager to parametrize tests raising and not raising exceptions diff --git a/tests/unittests/net/test_dhcp.py b/tests/unittests/net/test_dhcp.py index ec1b8ef83ac..c21f858a390 100644 --- a/tests/unittests/net/test_dhcp.py +++ b/tests/unittests/net/test_dhcp.py @@ -20,8 +20,11 @@ NoDHCPLeaseInterfaceError, NoDHCPLeaseMissingDhclientError, Udhcpc, + find_correct_device_nmcli, maybe_perform_dhcp_discovery, + network_manager_load_leases, networkd_load_leases, + run_nmcli, ) from cloudinit.net.ephemeral import EphemeralDHCPv4 from cloudinit.subp import SubpResult @@ -839,7 +842,6 @@ def test_ephemeral_dhcp_setup_network_if_url_connectivity( @pytest.mark.parametrize( "error_class", [ - NoDHCPLeaseInterfaceError, NoDHCPLeaseInterfaceError, NoDHCPLeaseMissingDhclientError, ], @@ -1392,3 +1394,84 @@ def test_none_and_missing_fallback(self): with pytest.raises(NoDHCPLeaseInterfaceError): distro = mock.Mock(fallback_interface=None) maybe_perform_dhcp_discovery(distro, None) + + +class TestNMDhcpLeases: + def test_find_correct_connected_device_first_match(self): + with mock.patch( + "cloudinit.net.dhcp.run_nmcli", + return_value=dedent( + """ + ens150:ethernet:unavailable: + p2p-dev-wlp:wifi-p2p:disconnected: + ens160:ethernet:connected:Wired connection 2 + ens256:ethernet:connected:Wired connection 3 + lo:loopback:connected (externally):lo + """ + ), + ): + ret = find_correct_device_nmcli() + assert ret == "ens160" + + def test_network_manager_load_leases(self): + expected_return = { + "broadcast_address": "172.16.127.255", + "dhcp_client_identifier": "01:00:0c:29:bf:c5:56", + "dhcp_lease_time": "1800", + "dhcp_server_identifier": "172.16.127.254", + } + with mock.patch( + "cloudinit.net.dhcp.run_nmcli", + return_value=dedent( + """ + DHCP4.OPTION[1]: broadcast_address = 172.16.127.255 + DHCP4.OPTION[2]: dhcp_client_identifier = 01:00:0c:29:bf:c5:56 + DHCP4.OPTION[3]: dhcp_lease_time = 1800 + DHCP4.OPTION[4]: dhcp_server_identifier = 172.16.127.254 + """ + ), + ): + ret = network_manager_load_leases("ens10") + assert ret == expected_return + + @mock.patch("cloudinit.net.dhcp.subp.which", return_value="/usr/bin/nmcli") + @mock.patch("cloudinit.net.dhcp.subp.subp") + def test_run_nmcli(self, m_subp, m_which): + expected_return = dedent( + """ + DHCP4.OPTION[1]: broadcast_address = 172.16.127.255 + DHCP4.OPTION[2]: dhcp_client_identifier = 01:00:0c:29:bf:c5:56 + DHCP4.OPTION[3]: dhcp_lease_time = 1800 + DHCP4.OPTION[4]: dhcp_server_identifier = 172.16.127.254 + """ + ) + m_subp.return_value = (expected_return, "") + ret = run_nmcli(["show"]) + assert ret == expected_return + + @mock.patch("cloudinit.net.dhcp.subp.which", return_value=None) + def test_run_nmcli_missing_nmcli(self, m_which): + """verify that absence of nmcli binary can result in + raising NoDHCPLeaseMissingDhclientError""" + + with pytest.raises(NoDHCPLeaseMissingDhclientError): + run_nmcli(["show"]) + + @mock.patch("cloudinit.net.dhcp.subp.which", return_value="/usr/bin/nmcli") + @mock.patch("cloudinit.net.dhcp.subp.subp") + def test_run_nmcli_subp_err(self, m_subp, m_which): + """verify that when subp.ProcessExecutionError is raised while + running nmcli, it results in run_nmcli raising NoDHCPLeaseError""" + + m_subp.side_effect = subp.ProcessExecutionError(exit_code=-5) + + with pytest.raises(NoDHCPLeaseError): + run_nmcli(["--fields", "all", "device", "show"]) + + m_subp.assert_has_calls( + [ + mock.call( + ["/usr/bin/nmcli", "--fields", "all", "device", "show"], + ), + ] + ) diff --git a/tests/unittests/net/test_init.py b/tests/unittests/net/test_init.py index 6aed3ecfa18..7a02bd4524d 100644 --- a/tests/unittests/net/test_init.py +++ b/tests/unittests/net/test_init.py @@ -1731,6 +1731,13 @@ class TestIsIpAddress: (lambda _: ipaddress.IPv6Address("2001:db8::"), True), (lambda _: ipaddress.IPv6Address("2001:db8::/48"), False), ), + ids=( + "value-error", + "ipv4-address", + "ipv4-network", + "ipv6-address", + "ipv6-network", + ), ) def test_is_ip_address(self, ip_address_side_effect, expected_return): with mock.patch( diff --git a/tests/unittests/net/test_net_rendering.py b/tests/unittests/net/test_net_rendering.py index 7698462b3bc..ded5795ec1e 100644 --- a/tests/unittests/net/test_net_rendering.py +++ b/tests/unittests/net/test_net_rendering.py @@ -28,6 +28,7 @@ import glob from enum import Flag, auto from pathlib import Path +from unittest import mock import pytest import yaml @@ -36,7 +37,6 @@ from cloudinit.net.network_manager import Renderer as NetworkManagerRenderer from cloudinit.net.network_state import NetworkState, parse_net_config_data from cloudinit.net.networkd import Renderer as NetworkdRenderer -from tests.unittests.helpers import mock ARTIFACT_DIR = Path(__file__).parent.absolute() / "artifacts" diff --git a/tests/unittests/sources/azure/test_errors.py b/tests/unittests/sources/azure/test_errors.py index d9dcf13d555..61d16faa45c 100644 --- a/tests/unittests/sources/azure/test_errors.py +++ b/tests/unittests/sources/azure/test_errors.py @@ -211,6 +211,15 @@ def test_imds_metadata_parsing_exception(): assert error.supporting_data["exception"] == repr(exception) +def test_import_error(): + exception = ImportError("No module named 'foobar'", name="foobar") + + error = errors.ReportableErrorImportError(error=exception) + + assert error.reason == "error importing foobar library" + assert error.supporting_data["error"] == repr(exception) + + def test_ovf_parsing_exception(): error = None try: @@ -257,6 +266,7 @@ def test_unhandled_exception(): "None", None, ], + ids=["running", "string-none", "none-value"], ) def test_imds_invalid_metadata(value): key = "compute" diff --git a/tests/unittests/sources/helpers/test_akamai.py b/tests/unittests/sources/helpers/test_akamai.py index 3cc0e9190b3..9ca83c8ccf0 100644 --- a/tests/unittests/sources/helpers/test_akamai.py +++ b/tests/unittests/sources/helpers/test_akamai.py @@ -1,9 +1,9 @@ from typing import Any, Dict +from unittest import mock import pytest from cloudinit.sources.helpers.akamai import get_dmi_config, is_on_akamai -from tests.unittests.helpers import mock class TestAkamaiHelper: diff --git a/tests/unittests/sources/helpers/test_netlink.py b/tests/unittests/sources/helpers/test_netlink.py index 208ca7831d3..28197b49d90 100644 --- a/tests/unittests/sources/helpers/test_netlink.py +++ b/tests/unittests/sources/helpers/test_netlink.py @@ -5,6 +5,7 @@ import codecs import socket import struct +from unittest import mock import pytest @@ -31,7 +32,6 @@ wait_for_nic_attach_event, wait_for_nic_detach_event, ) -from tests.unittests.helpers import mock def int_to_bytes(i): diff --git a/tests/unittests/sources/test___init__.py b/tests/unittests/sources/test___init__.py index 2c214aeed66..67924d6a517 100644 --- a/tests/unittests/sources/test___init__.py +++ b/tests/unittests/sources/test___init__.py @@ -1,8 +1,9 @@ +from unittest import mock + import pytest from cloudinit import sources from cloudinit.sources import DataSourceOpenStack as ds -from tests.unittests.helpers import mock openstack_ds_name = ds.DataSourceOpenStack.dsname.lower() diff --git a/tests/unittests/sources/test_akamai.py b/tests/unittests/sources/test_akamai.py index 2a68099753a..05ad49847c3 100644 --- a/tests/unittests/sources/test_akamai.py +++ b/tests/unittests/sources/test_akamai.py @@ -1,5 +1,6 @@ from contextlib import suppress from typing import Any, Dict, List, Optional, Union +from unittest import mock import pytest @@ -8,7 +9,6 @@ DataSourceAkamaiLocal, MetadataAvailabilityResult, ) -from tests.unittests.helpers import mock class TestDataSourceAkamai: diff --git a/tests/unittests/sources/test_altcloud.py b/tests/unittests/sources/test_altcloud.py index 0a42602c6ab..24ef12ee7e5 100644 --- a/tests/unittests/sources/test_altcloud.py +++ b/tests/unittests/sources/test_altcloud.py @@ -12,12 +12,12 @@ import os import shutil +from unittest import mock import pytest import cloudinit.sources.DataSourceAltCloud as dsac from cloudinit import subp, util -from tests.unittests.helpers import mock OS_UNAME_ORIG = getattr(os, "uname") diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index 67373e1ef1e..807acdbdd71 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -1,16 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. # pylint: disable=attribute-defined-outside-init +import builtins import copy import datetime import json import logging import os import stat +import sys import xml.etree.ElementTree as ET from pathlib import Path -import passlib.hash +try: + import passlib.hash +except ImportError: + passlib = None # type: ignore import pytest import requests @@ -979,11 +984,98 @@ def test_parsing_scenarios( ): assert ( dsaz.generate_network_config_from_instance_network_metadata( - metadata, apply_network_config_for_secondary_ips=ip_config + metadata, + apply_network_config_for_secondary_ips=ip_config, + apply_network_config_set_name=True, ) == expected ) + @pytest.mark.parametrize( + "set_name,expected", + [ + ( + True, + { + "ethernets": { + "eth0": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": True, + "dhcp6-overrides": {"route-metric": 100}, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + "set-name": "eth0", + }, + "eth1": { + "dhcp4": True, + "dhcp4-overrides": { + "route-metric": 200, + "use-dns": False, + }, + "dhcp6": False, + "match": {"macaddress": "22:0d:3a:04:75:98"}, + "set-name": "eth1", + }, + }, + "version": 2, + }, + ), + ( + False, + { + "ethernets": { + "enx000d3a047598": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": True, + "dhcp6-overrides": {"route-metric": 100}, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + }, + "enx220d3a047598": { + "dhcp4": True, + "dhcp4-overrides": { + "route-metric": 200, + "use-dns": False, + }, + "dhcp6": False, + "match": {"macaddress": "22:0d:3a:04:75:98"}, + }, + }, + "version": 2, + }, + ), + ], + ) + def test_set_name_config(self, mock_get_interfaces, set_name, expected): + """Verify set-name with two NICs (primary with IPv6, secondary).""" + two_nic_metadata = { + "interface": [ + { + "macAddress": "000D3A047598", + "ipv6": { + "subnet": [{"prefix": "64", "address": "fd00::"}], + "ipAddress": [{"privateIpAddress": "fd00::4"}], + }, + "ipv4": { + "subnet": [{"prefix": "24", "address": "10.0.0.0"}], + "ipAddress": [ + { + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "104.46.124.81", + } + ], + }, + }, + SECONDARY_INTERFACE, + ] + } + result = dsaz.generate_network_config_from_instance_network_metadata( + two_nic_metadata, + apply_network_config_for_secondary_ips=True, + apply_network_config_set_name=set_name, + ) + assert result == expected + class TestNetworkConfig: fallback_config = { @@ -999,22 +1091,45 @@ class TestNetworkConfig: ], } - def test_single_ipv4_nic_configuration( - self, azure_ds, mock_get_interfaces - ): - """Network config emits dhcp on single nic with ipv4""" - expected = { - "ethernets": { - "eth0": { - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": False, - "match": {"macaddress": "00:0d:3a:04:75:98"}, - "set-name": "eth0", + @pytest.mark.parametrize( + "set_name,expected", + [ + ( + True, + { + "ethernets": { + "eth0": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": False, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + "set-name": "eth0", + }, + }, + "version": 2, }, - }, - "version": 2, - } + ), + ( + False, + { + "ethernets": { + "enx000d3a047598": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": False, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + }, + }, + "version": 2, + }, + ), + ], + ) + def test_network_config( + self, azure_ds, mock_get_interfaces, set_name, expected + ): + """Verify network_config via ds_cfg for set-name enabled/disabled.""" + azure_ds.ds_cfg["apply_network_config_set_name"] = set_name azure_ds._metadata_imds = NETWORK_METADATA assert azure_ds.network_config == expected @@ -1732,12 +1847,13 @@ def test_username_used(self, get_ds): assert "ssh_pwauth" not in dsrc.cfg + @pytest.mark.skipif(passlib is None, reason="passlib not installed") def test_password_given(self, get_ds, mocker): # The crypt module has platform-specific behavior and the purpose of # this test isn't to verify the differences between crypt and passlib, # so hardcode passlib usage as crypt is deprecated. mocker.patch.object( - dsaz, "blowfish_hash", passlib.hash.sha512_crypt.hash + dsaz, "hash_password", passlib.hash.sha512_crypt.hash ) data = { "ovfcontent": construct_ovf_env( @@ -2434,6 +2550,19 @@ def test_wb_invalid_ovf_env_xml_calls_read_azure_ovf(self, tmp_path): == cm.value.reason ) + def test_import_error_from_failed_import(self): + """Attempt to import a module that is not present""" + try: + import nonexistent_module_that_will_never_exist # type: ignore[import-not-found] # noqa: F401 # isort:skip + except ImportError as error: + reportable_error = errors.ReportableErrorImportError(error=error) + + assert ( + reportable_error.reason == "error importing " + "nonexistent_module_that_will_never_exist library" + ) + assert reportable_error.supporting_data["error"] == repr(error) + class TestReadAzureOvf: def test_invalid_xml_raises_non_azure_ds(self): @@ -5212,6 +5341,213 @@ def test_os_disk_pps(self, mock_sleep, subp_side_effect): assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + @pytest.mark.parametrize("pps_type", ["None", "Running", "Savable"]) + def test_skip_ready_report(self, pps_type): + """Verify ready report is skipped when configured to.""" + self.azure_ds.ds_cfg["experimental_skip_ready_report"] = True + + is_pps = pps_type in ("Running", "Savable") + + imds_md_source = copy.deepcopy(self.imds_md) + imds_md_source["extended"]["compute"]["ppsType"] = pps_type + + nl_sock = mock.MagicMock() + self.mock_netlink.create_bound_netlink_socket.return_value = nl_sock + if pps_type == "Savable": + self.mock_netlink.wait_for_nic_detach_event.return_value = "eth9" + self.mock_netlink.wait_for_nic_attach_event.return_value = ( + "ethAttached1" + ) + + if is_pps: + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(imds_md_source).encode()), + mock.MagicMock( + contents=construct_ovf_env( + provision_guest_proxy_agent=False + ).encode() + ), + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] + else: + ovf = construct_ovf_env(provision_guest_proxy_agent=False) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] + + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._check_and_get_data() + + assert self.mock_subp_subp.mock_calls == [] + + # Verify IMDS calls. + if is_pps: + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + exception_cb=mock.ANY, + headers_cb=imds.headers_cb, + infinite=True, + log_req_resp=True, + timeout=30, + ), + mock.call( + "http://169.254.169.254/metadata/reprovisiondata?" + "api-version=2019-06-01", + exception_cb=mock.ANY, + headers_cb=imds.headers_cb, + log_req_resp=False, + infinite=True, + timeout=30, + ), + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + exception_cb=mock.ANY, + headers_cb=imds.headers_cb, + infinite=True, + log_req_resp=True, + timeout=30, + ), + ] + else: + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + timeout=30, + headers_cb=imds.headers_cb, + exception_cb=mock.ANY, + infinite=True, + log_req_resp=True, + ), + ] + + # Verify DHCP setup. + if pps_type == "Running": + assert ( + self.mock_wrapping_setup_ephemeral_networking.mock_calls + == [ + mock.call(timeout_minutes=20), + mock.call(timeout_minutes=5), + ] + ) + assert ( + self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls + == [ + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + ] + ) + elif pps_type == "Savable": + assert ( + self.mock_wrapping_setup_ephemeral_networking.mock_calls + == [ + mock.call(timeout_minutes=20), + mock.call( + iface="ethAttached1", + timeout_minutes=20, + report_failure_if_not_primary=False, + ), + ] + ) + assert ( + self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls + == [ + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + mock.call( + self.azure_ds.distro, + "ethAttached1", + dsaz.dhcp_log_cb, + ), + ] + ) + else: + assert ( + self.mock_wrapping_setup_ephemeral_networking.mock_calls + == [mock.call(timeout_minutes=20)] + ) + assert ( + self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls + == [ + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + ] + ) + + assert self.azure_ds._wireserver_endpoint == "10.11.12.13" + assert self.azure_ds._is_ephemeral_networking_up() is False + + # Verify DMI usage. + assert self.mock_dmi_read_dmi_data.mock_calls == [ + mock.call("chassis-asset-tag"), + mock.call("system-uuid"), + ] + assert ( + self.azure_ds.metadata["instance-id"] + == "50109936-ef07-47fe-ac82-890c853f60d5" + ) + + # Verify IMDS metadata. + assert self.azure_ds.metadata["imds"] == self.imds_md + + # PPS types still report ready once (source), no-PPS skips entirely. + if is_pps: + assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ + mock.call( + endpoint="10.11.12.13", + distro=self.azure_ds.distro, + iso_dev="/dev/sr0", + pubkey_info=None, + ), + ] + else: + assert self.mock_azure_get_metadata_from_fabric.mock_calls == [] + + # Verify netlink operations. + if pps_type == "Running": + assert self.mock_netlink.mock_calls == [ + mock.call.create_bound_netlink_socket(), + mock.call.wait_for_media_disconnect_connect( + mock.ANY, "ethBoot0" + ), + mock.call.create_bound_netlink_socket().close(), + ] + elif pps_type == "Savable": + assert self.mock_netlink.mock_calls == [ + mock.call.create_bound_netlink_socket(), + mock.call.wait_for_nic_detach_event(nl_sock), + mock.call.wait_for_nic_attach_event(nl_sock, ["ethAttached1"]), + mock.call.create_bound_netlink_socket().close(), + ] + else: + assert self.mock_netlink.mock_calls == [] + + # Verify reported_ready marker cleaned up. + if is_pps: + assert self.wrapped_util_write_file.mock_calls[0] == mock.call( + self.patched_reported_ready_marker_path.as_posix(), + mock.ANY, + ) + else: + assert self.wrapped_util_write_file.mock_calls == [] + assert self.patched_reported_ready_marker_path.exists() is False + + # Verify KVP reports. + assert not self.mock_kvp_report_via_kvp.mock_calls + assert not self.mock_azure_report_failure_to_fabric.mock_calls + expected_kvp_count = 1 if is_pps else 0 + assert ( + len(self.mock_kvp_report_success_to_host.mock_calls) + == expected_kvp_count + ) + assert ( + len(self.mock_report_dmesg_to_kvp.mock_calls) == expected_kvp_count + ) + def test_imds_failure_results_in_provisioning_failure(self): self.mock_readurl.side_effect = url_helper.UrlError( requests.ConnectionError( @@ -5255,6 +5591,84 @@ def test_imds_failure_results_in_provisioning_failure(self): assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 assert not self.mock_kvp_report_success_to_host.mock_calls + @pytest.mark.parametrize( + "flag_enabled", + [False, True], + ) + @pytest.mark.parametrize( + "has_custom_data,custom_data", + [ + (True, None), + (True, "myCustomData"), + (True, ""), + (False, None), + ], + ) + def test_missing_customdata_reporting( + self, + caplog, + flag_enabled, + has_custom_data, + custom_data, + ): + """Test failure reporting behavior based on custom data fields. + + Failure is reported only when + experimental_fail_on_missing_customdata is True, + IMDS reports hasCustomData=True, and OVF has no custom data. + When the flag is not enabled but IMDS reports custom data + should be present, a diagnostic event is logged. + """ + self.azure_ds.ds_cfg["experimental_fail_on_missing_customdata"] = ( + flag_enabled + ) + + imds_md = copy.deepcopy(self.imds_md) + imds_md["extended"]["compute"]["hasCustomData"] = has_custom_data + + ovf = construct_ovf_env( + custom_data=custom_data, + provision_guest_proxy_agent=False, + ) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(imds_md).encode()), + ] + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._check_and_get_data() + + expect_failure = flag_enabled and has_custom_data and not custom_data + if expect_failure: + assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 + assert ( + len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 + ) + assert not self.mock_kvp_report_success_to_host.mock_calls + else: + assert not self.mock_kvp_report_via_kvp.mock_calls + assert not self.mock_azure_report_failure_to_fabric.mock_calls + assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + + if custom_data: + assert self.azure_ds.userdata_raw == custom_data.encode("utf-8") + else: + assert self.azure_ds.userdata_raw == "" + + # Verify diagnostic event for missing custom data when + # the experimental flag is not enabled. + expect_diagnostic = ( + not flag_enabled and has_custom_data and not custom_data + ) + if expect_diagnostic: + assert ( + "Did not find custom data in /dev/sr0, IMDS returned" + " extended.compute.hasCustomData=True" + ) in caplog.text + else: + assert "Did not find custom data in" not in caplog.text + class TestCheckAzureProxyAgent: @pytest.fixture(autouse=True) @@ -5630,14 +6044,6 @@ def test_missing_secondary( assert azure_ds.validate_imds_network_metadata(imds_md) is False -class TestDependencyFallback: - def test_dependency_fallback(self): - """Ensure that crypt/passlib import failover gets exercised on all - Python versions - """ - assert dsaz.encrypt_pass("`") - - class TestQueryVmId: @mock.patch.object( identity, "query_system_uuid", side_effect=["test-system-uuid"] @@ -5700,3 +6106,119 @@ def test_query_vm_id_vm_id_conversion_failure( mock_query_system_uuid.assert_called_once() mock_convert_uuid.assert_called_once_with("test-system-uuid") + + +class TestHasCustomDataFromImds: + """Unit tests for the _hascustomdata_from_imds helper.""" + + @pytest.mark.parametrize( + "imds_data,expected", + [ + ({"extended": {"compute": {"hasCustomData": True}}}, True), + ({"extended": {"compute": {"hasCustomData": False}}}, False), + ({}, None), + ({"extended": {}}, None), + ({"extended": {"compute": {}}}, None), + ], + ) + def test_hascustomdata_from_imds(self, imds_data, expected): + assert dsaz._hascustomdata_from_imds(imds_data) is expected + + +class TestHashPassword: + """Tests for the hash_password function.""" + + def test_dependency_fallback(self): + """Ensure that crypt/passlib import failover gets exercised on all + Python versions + """ + result = dsaz.hash_password("`") + assert result + assert result.startswith("$6$") + + def test_crypt_working(self): + """Test that hash_password uses crypt when available.""" + mock_crypt = mock.MagicMock() + mock_crypt.METHOD_SHA512 = "sha512" + mock_crypt.mksalt.return_value = "$6$saltvalue" + mock_crypt.crypt.return_value = "$6$saltvalue$hashedpassword" + + with mock.patch.dict("sys.modules", {"crypt": mock_crypt}): + result = dsaz.hash_password("testpassword") + + mock_crypt.mksalt.assert_called_once_with("sha512") + mock_crypt.crypt.assert_called_once_with( + "testpassword", "$6$saltvalue" + ) + assert result == "$6$saltvalue$hashedpassword" + + def test_crypt_not_installed_passlib_fallback(self): + """Test that hash_password falls back to passlib when missing crypt.""" + real_import = builtins.__import__ + passlib_available = True + try: + import passlib.hash as _passlib_hash + except ImportError: + passlib_available = False + + if passlib_available: + # passlib is installed; block crypt and let passlib work normally + def mock_import(name, *args, **kwargs): + if name == "crypt": + raise ImportError("No module named 'crypt'") + return real_import(name, *args, **kwargs) + + with mock.patch.object( + builtins, "__import__", side_effect=mock_import + ): + result = dsaz.hash_password("testpassword") + + # Verify we got a valid SHA-512 hash from passlib + assert result.startswith("$6$") + assert _passlib_hash.sha512_crypt.verify("testpassword", result) + else: + # passlib is not installed; mock it to return a known hash + mock_passlib_hash = mock.MagicMock() + mock_passlib_hash.sha512_crypt.hash.return_value = ( + "$6$mocksalt$mockedhash" + ) + + def mock_import(name, *args, **kwargs): + if name == "crypt": + raise ImportError("No module named 'crypt'") + if name == "passlib.hash": + mod = mock.MagicMock() + mod.hash = mock_passlib_hash + sys.modules["passlib"] = mod + sys.modules["passlib.hash"] = mock_passlib_hash + return mod + return real_import(name, *args, **kwargs) + + with mock.patch.object( + builtins, "__import__", side_effect=mock_import + ): + result = dsaz.hash_password("testpassword") + + assert result == "$6$mocksalt$mockedhash" + mock_passlib_hash.sha512_crypt.hash.assert_called_once_with( + "testpassword" + ) + + def test_crypt_and_passlib_unavailable_raises_error(self): + """Test that hash_password raises ReportableErrorImportError.""" + real_import = builtins.__import__ + + def mock_import(name, *args, **kwargs): + if name == "crypt": + raise ImportError("No module named 'crypt'") + if name == "passlib.hash": + raise ImportError("No module named 'passlib'", name="passlib") + return real_import(name, *args, **kwargs) + + with mock.patch.object( + builtins, "__import__", side_effect=mock_import + ): + with pytest.raises(errors.ReportableErrorImportError) as exc_info: + dsaz.hash_password("testpassword") + + assert "passlib" in exc_info.value.reason diff --git a/tests/unittests/sources/test_bigstep.py b/tests/unittests/sources/test_bigstep.py index 100b6fc63a5..0427dee6712 100644 --- a/tests/unittests/sources/test_bigstep.py +++ b/tests/unittests/sources/test_bigstep.py @@ -1,12 +1,12 @@ import json import os +from unittest import mock import pytest import responses from cloudinit import helpers from cloudinit.sources import DataSourceBigstep as bigstep -from tests.unittests.helpers import mock M_PATH = "cloudinit.sources.DataSourceBigstep." diff --git a/tests/unittests/sources/test_cloudstack.py b/tests/unittests/sources/test_cloudstack.py index 90c56291f5b..42544e42e05 100644 --- a/tests/unittests/sources/test_cloudstack.py +++ b/tests/unittests/sources/test_cloudstack.py @@ -2,6 +2,7 @@ # pylint: disable=attribute-defined-outside-init from socket import gaierror from textwrap import dedent +from unittest import mock from unittest.mock import patch import pytest @@ -16,7 +17,6 @@ get_data_server, get_vr_address, ) -from tests.unittests.helpers import mock from tests.unittests.util import MockDistro SOURCES_PATH = "cloudinit.sources" @@ -67,6 +67,7 @@ def setup(self, mocker, tmp_path): self.hostname = "vm-hostname" self.networkd_domainname = "networkd.local" self.isc_dhclient_domainname = "dhclient.local" + self.nm_domainname = "nm.local" get_hostname_parent = mock.MagicMock( return_value=DataSourceHostname(self.hostname, True) @@ -108,6 +109,10 @@ def test_get_domainname_isc_dhclient(self, cloudstack_ds, mocker): DHCP_MOD_PATH + ".networkd_get_option_from_leases", get_networkd_domain, ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) with patch( MOD_PATH + ".util.load_text_file", @@ -141,6 +146,201 @@ def test_get_domainname_isc_dhclient(self, cloudstack_ds, mocker): result = cloudstack_ds._get_domainname() assert self.isc_dhclient_domainname == result + def test_get_domainname_network_manager(self, cloudstack_ds, mocker): + """ + Test if DataSourceCloudStack._get_domainname() + gets domain name from nmcli lease information + """ + nmcliop = f"DHCP4.OPTION[5]: domain_name = {self.nm_domainname}" + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".find_correct_device_nmcli", + return_value="ens120", + ) + + with patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + DHCP4.OPTION[1]: broadcast_address = 172.16.127.255 + DHCP4.OPTION[2]: dhcp_client_identifier = 01:00:0c:29:bf:c5:56 + DHCP4.OPTION[3]: dhcp_lease_time = 1800 + DHCP4.OPTION[4]: dhcp_server_identifier = 172.16.127.254 + """ + + nmcliop + + """ + DHCP4.OPTION[6]: domain_name_servers = 172.16.127.2 + DHCP4.OPTION[7]: expiry = 1775029195 + DHCP4.OPTION[8]: ip_address = 172.16.127.135 + DHCP4.OPTION[9]: next_server = 172.16.127.254 + """ + ), + ): + result = cloudstack_ds._get_domainname() + assert self.nm_domainname == result + + def test_get_domainname_nm_multi_conn_nic(self, cloudstack_ds, mocker): + """ + Test if DataSourceCloudStack._get_domainname() + can handle multi connected nic environments. + """ + domain_name = f"{self.nm_domainname}" + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + ens160:ethernet:connected:Wired connection 1 + ens256:ethernet:connected:Wired connection 2 + lo:loopback:connected (externally):lo + """ + ), + ) + + with patch( + DHCP_MOD_PATH + ".network_manager_load_leases", + return_value={"domain_name": domain_name}, + ): + result = cloudstack_ds._get_domainname() + assert self.nm_domainname == result + + def test_get_domainname_nm_single_conn_nic(self, cloudstack_ds, mocker): + """ + Test if DataSourceCloudStack._get_domainname() + can handle one connected nic environments. + """ + domain_name = f"{self.nm_domainname}" + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + ens160:ethernet:connected:Wired connection 1 + lo:loopback:connected (externally):lo + ens256:ethernet:unavailable: + """ + ), + ) + + with patch( + DHCP_MOD_PATH + ".network_manager_load_leases", + return_value={"domain_name": domain_name}, + ): + result = cloudstack_ds._get_domainname() + assert self.nm_domainname == result + + def test_get_domainname_nm_no_conn_nic( + self, cloudstack_ds, mocker, caplog + ): + """ + Test if DataSourceCloudStack._get_domainname() + can handle one connected nic environments. + """ + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + lo:loopback:connected (externally):lo + ens256:ethernet:unavailable: + """ + ), + ) + + result = cloudstack_ds._get_domainname() + assert "Could not obtain FQDN from NM leases" in caplog.text + assert ( + "No domain name found in any DHCP lease; returning empty" + in caplog.text + ) + assert result == "" + + def test_get_domainname_nm_nodhcp_lease_err( + self, cloudstack_ds, mocker, caplog + ): + """ + Test if DataSourceCloudStack._get_domainname() + can handle NoDHCPLeaseError. + """ + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".network_manager_get_option_from_leases", + side_effect=NoDHCPLeaseError, + ) + + cloudstack_ds._get_domainname() + assert "Could not obtain FQDN from NM leases" in caplog.text + def test_get_hostname_non_fqdn(self, cloudstack_ds): """ Test get_hostname() method implementation @@ -186,6 +386,12 @@ def test_get_hostname_fqdn_fallback(self, cloudstack_ds, mocker): get_networkd_domain, ) + get_nm_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".network_manager_get_option_from_leases", + get_nm_domain, + ) + mocker.patch( "cloudinit.distros.net.find_fallback_nic", return_value="eth0", @@ -286,9 +492,17 @@ def test_data_server_from_dns( MOD_PATH + ".dhcp.networkd_get_option_from_leases", return_value="10.1.37.132", ) +@mock.patch( + MOD_PATH + ".dhcp.network_manager_get_option_from_leases", + return_value="10.1.37.135", +) class TestGetVrAddress: def test_get_vr_addr_from_dns( - self, m_networkd_option_from_leases, m_get_data_server, caplog + self, + m_nm_get_option_from_leases, + m_networkd_option_from_leases, + m_get_data_server, + caplog, ): """cloud-init first obtains data-server if resolved by DNS""" assert "10.1.37.131" == get_vr_address(MockDistro()) @@ -299,7 +513,12 @@ def test_get_vr_addr_from_dns( assert 0 == m_networkd_option_from_leases.call_count def test_get_vr_addr_from_networkd_leases( - self, m_networkd_option_from_leases, m_get_data_server, mocker, caplog + self, + m_nm_get_option_from_leases, + m_networkd_option_from_leases, + m_get_data_server, + mocker, + caplog, ): """When no DNS for data-server use networkd dhcp-server-identifier""" mocker.patch(MOD_PATH + ".get_data_server", return_value=None) @@ -310,6 +529,36 @@ def test_get_vr_addr_from_networkd_leases( ) m_networkd_option_from_leases.assert_called_once_with("SERVER_ADDRESS") + def test_get_vr_addr_from_network_manager_leases( + self, + m_nm_get_option_from_leases, + m_networkd_option_from_leases, + m_get_data_server, + mocker, + caplog, + ): + """When no DNS for data-server or networkd or IscDhclient, + use dhcp_server_identifier from network manager lease""" + mocker.patch(MOD_PATH + ".get_data_server", return_value=None) + mocker.patch( + MOD_PATH + ".dhcp.networkd_get_option_from_leases", + return_value=None, + ) + mocker.patch( + DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", + return_value=None, + ) + assert "10.1.37.135" == get_vr_address(MockDistro()) + m_which = mocker.patch( + "cloudinit.net.dhcp.subp.which", + return_value=None, + ) + assert "10.1.37.135" == get_vr_address(MockDistro()) + m_which.assert_called_once_with("dhclient") + m_nm_get_option_from_leases.assert_called_with( + "dhcp_server_identifier" + ) + @pytest.mark.usefixtures("dhclient_exists") @mock.patch(MOD_PATH + ".dmi.read_dmi_data", return_value=CLOUD_STACK_DMI_NAME) @@ -331,6 +580,10 @@ def setup(self, mocker, tmp_path): "dhcp-server-identifier": "168.63.129.16", }, ) + mocker.patch( + DHCP_MOD_PATH + ".network_manager_get_option_from_leases", + return_value=None, + ) get_newest_lease_file_from_distro = mock.MagicMock(return_value=None) mocker.patch( DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", @@ -496,11 +749,11 @@ def test_local_datasource_fails_ephemeral_dhcp( MOD_PATH + ".EphemeralIPNetwork", autospec=True, ) - @mock.patch(MOD_PATH + ".net.find_fallback_nic") - # @mock.patch(MOD_PATH + ".get_vr_address", return_value="10.1.37.131") + @mock.patch(MOD_PATH + ".net.find_fallback_nic", return_value="enp0s1") + @mock.patch(MOD_PATH + ".get_vr_address", return_value="10.1.37.131") def test_local_datasource_success( self, - # m_get_vr_address, + m_get_vr_address, m_find_fallback_nic, m_dhcp, m_wait_for_mds, @@ -514,7 +767,6 @@ def test_local_datasource_success( {}, distro, helpers.Paths({"run_dir": tmpdir}) ) - m_find_fallback_nic.return_value = "enp0s1" m_dhcp.return_value.__enter__.side_effect = (None,) m_wait_for_mds.return_value = (True,) m_get_userdata.return_value = "ud" diff --git a/tests/unittests/sources/test_configdrive.py b/tests/unittests/sources/test_configdrive.py index 43ea8be3d4d..0f3e0c76439 100644 --- a/tests/unittests/sources/test_configdrive.py +++ b/tests/unittests/sources/test_configdrive.py @@ -4,6 +4,7 @@ import os from contextlib import ExitStack from copy import copy, deepcopy +from unittest import mock import pytest @@ -11,7 +12,7 @@ from cloudinit.net import eni, network_state from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack -from tests.unittests.helpers import mock, populate_dir +from tests.unittests.helpers import populate_dir PUBKEY = "ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n" EC2_META = { diff --git a/tests/unittests/sources/test_digitalocean.py b/tests/unittests/sources/test_digitalocean.py index b5bc526ed37..587c767f7e2 100644 --- a/tests/unittests/sources/test_digitalocean.py +++ b/tests/unittests/sources/test_digitalocean.py @@ -8,13 +8,13 @@ # pylint: disable=attribute-defined-outside-init import json +from unittest import mock import pytest from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from cloudinit.sources.helpers import digitalocean -from tests.unittests.helpers import mock DO_MULTIPLE_KEYS = [ "ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py index 0cb990c1a31..27a9a034c8f 100644 --- a/tests/unittests/sources/test_ec2.py +++ b/tests/unittests/sources/test_ec2.py @@ -1411,7 +1411,7 @@ class TestBuildNicOrder: "0a:0d:dd:44:cd:7b": 1, "0a:f7:8d:96:f2:a2": 2, }, - id="no-device-number-info-subset-sort-by-nic-name", + id="no-device-number-info-extra-mac-sort-by-nic-name", ), ], ) diff --git a/tests/unittests/sources/test_exoscale.py b/tests/unittests/sources/test_exoscale.py index 08cbeeb913d..667f42cd85e 100644 --- a/tests/unittests/sources/test_exoscale.py +++ b/tests/unittests/sources/test_exoscale.py @@ -3,6 +3,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. import os +from unittest import mock import requests import responses @@ -16,7 +17,6 @@ get_password, read_metadata, ) -from tests.unittests.helpers import mock TEST_PASSWORD_URL = "{}:{}/{}/".format( METADATA_URL, PASSWORD_SERVER_PORT, API_VERSION diff --git a/tests/unittests/sources/test_hetzner.py b/tests/unittests/sources/test_hetzner.py index af0a893cd0f..d7fe082028f 100644 --- a/tests/unittests/sources/test_hetzner.py +++ b/tests/unittests/sources/test_hetzner.py @@ -4,11 +4,12 @@ # # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest from cloudinit import settings, util from cloudinit.sources import DataSourceHetzner -from tests.unittests.helpers import mock METADATA = b""" hostname: cloudinit-test diff --git a/tests/unittests/sources/test_ibmcloud.py b/tests/unittests/sources/test_ibmcloud.py index f6b81908edf..c604ac6b922 100644 --- a/tests/unittests/sources/test_ibmcloud.py +++ b/tests/unittests/sources/test_ibmcloud.py @@ -5,6 +5,7 @@ import copy import json from textwrap import dedent +from unittest import mock import pytest @@ -12,8 +13,6 @@ from cloudinit.sources import DataSourceIBMCloud as ibm from tests.unittests import helpers as test_helpers -mock = test_helpers.mock - D_PATH = "cloudinit.sources.DataSourceIBMCloud." diff --git a/tests/unittests/sources/test_nocloud.py b/tests/unittests/sources/test_nocloud.py index a401c6292ed..383160e52b3 100644 --- a/tests/unittests/sources/test_nocloud.py +++ b/tests/unittests/sources/test_nocloud.py @@ -2,6 +2,7 @@ import os import textwrap +from unittest import mock import pytest import yaml @@ -11,7 +12,7 @@ DataSourceNoCloudNet, parse_cmdline_data, ) -from tests.unittests.helpers import mock, populate_dir +from tests.unittests.helpers import populate_dir @pytest.fixture(autouse=True) diff --git a/tests/unittests/sources/test_nwcs.py b/tests/unittests/sources/test_nwcs.py index c208c875974..219f322b950 100644 --- a/tests/unittests/sources/test_nwcs.py +++ b/tests/unittests/sources/test_nwcs.py @@ -1,10 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest from cloudinit import settings, util from cloudinit.sources import DataSourceNWCS -from tests.unittests.helpers import mock METADATA = util.load_yaml( """ diff --git a/tests/unittests/sources/test_opennebula.py b/tests/unittests/sources/test_opennebula.py index c6f4066abbe..1c7e48691dd 100644 --- a/tests/unittests/sources/test_opennebula.py +++ b/tests/unittests/sources/test_opennebula.py @@ -3,12 +3,13 @@ import os import pwd +from unittest import mock import pytest -from cloudinit import atomic_helper, util +from cloudinit import atomic_helper from cloudinit.sources import DataSourceOpenNebula as ds -from tests.unittests.helpers import mock, populate_dir +from tests.unittests.helpers import populate_dir TEST_VARS = { "VAR1": "single", @@ -364,7 +365,6 @@ def my_devs_with(criteria): }.get(criteria, []) m_find_devs_with.side_effect = my_devs_with - util.find_devs_with = my_devs_with assert ["/dev/sdb", "/dev/sr0", "/dev/vdb"] == ds.find_candidate_devs() @@ -570,16 +570,6 @@ def test_get_field_emptycontext(self): val = net.get_field("eth9", "dummy") assert None is val - def test_get_field_nonecontext(self): - """ - Verify get_field('device', 'name') returns None if context value is - None. - """ - context = {"ETH9_DUMMY": None} - net = ds.OpenNebulaNetwork(context, mock.Mock()) - val = net.get_field("eth9", "dummy") - assert None is val - @mock.patch(DS_PATH + ".get_physical_nics_by_mac") def test_gen_conf_gateway(self, m_get_phys_by_mac): """Test rendering with/without IPv4 gateway""" @@ -799,7 +789,7 @@ def test_gen_conf_mtu(self, m_get_phys_by_mac): "version": 2, "ethernets": { nic: { - "mtu": "1280", + "mtu": 1280, "match": {"macaddress": MACADDR}, "addresses": [IP_BY_MACADDR + "/" + IP4_PREFIX], } @@ -908,7 +898,7 @@ def test_eth0_v4v6_override(self): "addresses": ["1.2.3.6", "1.2.3.7", "1.2.3.8"], "search": ["example.com", "example.org"], }, - "mtu": "1280", + "mtu": 1280, } }, } @@ -969,7 +959,7 @@ def test_multiple_nics(self): "addresses": ["1.2.3.6", "1.2.3.7", "1.2.3.8"], "search": ["example.com"], }, - "mtu": "1280", + "mtu": 1280, }, "enp0s25": { "match": {"macaddress": MAC_1}, @@ -985,6 +975,154 @@ def test_multiple_nics(self): assert expected == net.gen_conf() + @pytest.mark.parametrize( + "context,expected_search", + [ + pytest.param( + {"SEARCH_DOMAIN": "global.example.com global.example.org"}, + ["global.example.com", "global.example.org"], + id="global_only", + ), + pytest.param( + { + "ETH0_SEARCH_DOMAIN": "iface.example.com", + "SEARCH_DOMAIN": "global.example.com", + }, + ["iface.example.com", "global.example.com"], + id="per_interface_and_global", + ), + pytest.param( + {"ETH0_SEARCH_DOMAIN": "iface.example.com"}, + ["iface.example.com"], + id="per_interface_only", + ), + pytest.param( + { + "ETH0_SEARCH_DOMAIN": "shared.example.com", + # extra precedes shared in global; shared must still come + # first because per-interface ordering takes precedence + "SEARCH_DOMAIN": "extra.example.com shared.example.com", + }, + ["shared.example.com", "extra.example.com"], + id="dedup_iface_order_preferred", + ), + ], + ) + def test_get_nameservers_search_domain(self, context, expected_search): + """get_nameservers merges and deduplicates SEARCH_DOMAIN correctly.""" + net = ds.OpenNebulaNetwork(context, mock.Mock()) + val = net.get_nameservers("eth0") + assert val["search"] == expected_search + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_global_search_domain(self, m_get_phys_by_mac): + """gen_conf includes global SEARCH_DOMAIN in nameservers.search.""" + context = { + "ETH0_MAC": MACADDR, + "SEARCH_DOMAIN": "global.example.com", + } + for nic in self.system_nics: + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context, mock.Mock()) + conf = net.gen_conf() + assert conf["ethernets"][nic]["nameservers"]["search"] == [ + "global.example.com" + ] + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_global_search_domain_multiple_nics( + self, m_get_phys_by_mac + ): + """Global SEARCH_DOMAIN appears on every NIC.""" + MAC_1 = "02:00:0a:12:01:01" + MAC_2 = "02:00:0a:12:01:02" + context = { + "ETH0_MAC": MAC_1, + "ETH1_MAC": MAC_2, + "SEARCH_DOMAIN": "global.example.com", + } + net = ds.OpenNebulaNetwork( + context, + mock.Mock(), + system_nics_by_mac={MAC_1: "eth0", MAC_2: "eth1"}, + ) + conf = net.gen_conf() + for nic in ("eth0", "eth1"): + assert ( + "global.example.com" + in conf["ethernets"][nic]["nameservers"]["search"] + ) + + # ------------------------------------------------------------------ # + # ETHx_ROUTES # + # ------------------------------------------------------------------ # + + @pytest.mark.parametrize( + "context,expected", + [ + pytest.param({}, [], id="absent"), + pytest.param({"ETH0_ROUTES": ""}, [], id="empty_string"), + pytest.param( + {"ETH0_ROUTES": "10.0.0.0/8 via 192.168.1.1"}, + [{"to": "10.0.0.0/8", "via": "192.168.1.1"}], + id="single_entry", + ), + pytest.param( + { + "ETH0_ROUTES": ( + "10.0.0.0/8 via 192.168.1.1," + " 172.16.0.0/12 via 192.168.1.254" + ) + }, + [ + {"to": "10.0.0.0/8", "via": "192.168.1.1"}, + {"to": "172.16.0.0/12", "via": "192.168.1.254"}, + ], + id="multiple_comma_separated_entries", + ), + pytest.param( + {"ETH0_ROUTES": "bad-entry, 10.0.0.0/8 via 192.168.1.1"}, + [{"to": "10.0.0.0/8", "via": "192.168.1.1"}], + id="malformed_entry_skipped", + ), + ], + ) + def test_get_routes(self, context, expected): + net = ds.OpenNebulaNetwork(context, mock.Mock()) + assert net.get_routes("eth0") == expected + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_routes(self, m_get_phys_by_mac): + """Routes from ETHx_ROUTES appear in gen_conf() output.""" + self.maxDiff = None + context = { + "ETH0_MAC": "02:00:0a:12:01:01", + "ETH0_IP": "10.0.0.5", + "ETH0_MASK": "255.255.255.0", + "ETH0_GATEWAY": "10.0.0.1", + "ETH0_ROUTES": ( + "192.168.0.0/16 via 10.0.0.1, 172.16.0.0/12 via 10.0.0.1" + ), + } + for nic in self.system_nics: + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context, mock.Mock()) + conf = net.gen_conf() + routes = conf["ethernets"][nic].get("routes", []) + assert {"to": "192.168.0.0/16", "via": "10.0.0.1"} in routes + assert {"to": "172.16.0.0/12", "via": "10.0.0.1"} in routes + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_no_routes_key_when_absent(self, m_get_phys_by_mac): + """gen_conf() does not emit 'routes' key when ETHx_ROUTES is unset.""" + context = { + "ETH0_MAC": "02:00:0a:12:01:01", + } + m_get_phys_by_mac.return_value = {MACADDR: "eth0"} + net = ds.OpenNebulaNetwork(context, mock.Mock()) + conf = net.gen_conf() + assert "routes" not in conf["ethernets"]["eth0"] + class TestParseShellConfig: @pytest.mark.allow_subp_for("bash", "sh") diff --git a/tests/unittests/sources/test_ovf.py b/tests/unittests/sources/test_ovf.py index 5b771df1220..1d348bc378b 100644 --- a/tests/unittests/sources/test_ovf.py +++ b/tests/unittests/sources/test_ovf.py @@ -9,12 +9,12 @@ import os from collections import OrderedDict from textwrap import dedent +from unittest import mock import pytest from cloudinit import subp, util from cloudinit.sources import DataSourceOVF as dsovf -from tests.unittests.helpers import mock MPATH = "cloudinit.sources.DataSourceOVF." diff --git a/tests/unittests/sources/test_scaleway.py b/tests/unittests/sources/test_scaleway.py index 57983be08de..50d76f94831 100644 --- a/tests/unittests/sources/test_scaleway.py +++ b/tests/unittests/sources/test_scaleway.py @@ -3,6 +3,7 @@ import json import socket +from unittest import mock from urllib.parse import SplitResult, urlsplit import pytest @@ -13,7 +14,7 @@ from cloudinit import settings from cloudinit.distros import ubuntu from cloudinit.sources import DataSourceScaleway -from tests.unittests.helpers import mock, responses_assert_call_count +from tests.unittests.helpers import responses_assert_call_count class DataResponses: diff --git a/tests/unittests/sources/test_smartos.py b/tests/unittests/sources/test_smartos.py index e25e7818189..b1bc03d4c02 100644 --- a/tests/unittests/sources/test_smartos.py +++ b/tests/unittests/sources/test_smartos.py @@ -24,6 +24,7 @@ import uuid from binascii import crc32 from collections import namedtuple +from unittest import mock import pytest import serial @@ -44,7 +45,6 @@ ) from cloudinit.subp import ProcessExecutionError, subp, which from cloudinit.util import write_file -from tests.unittests.helpers import mock DSMOS = "cloudinit.sources.DataSourceSmartOS" SDC_NICS = json.loads( diff --git a/tests/unittests/sources/test_vmware.py b/tests/unittests/sources/test_vmware.py index e4bd639b04d..12b2ec14332 100644 --- a/tests/unittests/sources/test_vmware.py +++ b/tests/unittests/sources/test_vmware.py @@ -11,6 +11,7 @@ from contextlib import ExitStack from logging import DEBUG from textwrap import dedent +from unittest import mock import pytest @@ -19,7 +20,7 @@ from cloudinit.sources import DataSourceVMware from cloudinit.sources.helpers.vmware.imc import guestcust_util from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, populate_dir, wrap_and_call +from tests.unittests.helpers import populate_dir, wrap_and_call MPATH = "cloudinit.sources.DataSourceVMware." PRODUCT_NAME_FILE_PATH = "/sys/class/dmi/id/product_name" diff --git a/tests/unittests/sources/test_wsl.py b/tests/unittests/sources/test_wsl.py index a5300ff8af0..75c7fb5b524 100644 --- a/tests/unittests/sources/test_wsl.py +++ b/tests/unittests/sources/test_wsl.py @@ -144,6 +144,19 @@ def test_cmd_exe_no_win_mounts(self, m_mounts, m_os_access): with pytest.raises(IOError): wsl.cmd_executable() + @mock.patch("cloudinit.sources.DataSourceWSL.cmd_executable") + @mock.patch("cloudinit.util.subp.subp") + def test_find_home_raises(self, m_subp, m_cmd): + # The value really doesn't matter. + m_cmd.return_value = PurePath("/mnt/c/cmd.exe") + m_subp.return_value = util.subp.SubpResult( + "I am UTF-8 🦄 !".encode("utf-8"), "\r\n".encode("utf-8") + ) + # Checking for ValueError instead of UnicodeDecodeError because + # that's what we catch at the call sites. + with pytest.raises(ValueError): + wsl.find_home() + @pytest.mark.parametrize( "linux_distro_value,files", ( diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 43e38fd3d96..e9fbf5897cf 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -3,12 +3,12 @@ import logging import os from types import SimpleNamespace +from unittest import mock import pytest from cloudinit import handlers, helpers, settings, url_helper, util from cloudinit.cmd import main -from tests.unittests.helpers import mock @pytest.fixture diff --git a/tests/unittests/test_apport.py b/tests/unittests/test_apport.py index 7bbf40d6f68..c3fedcaf36e 100644 --- a/tests/unittests/test_apport.py +++ b/tests/unittests/test_apport.py @@ -1,11 +1,11 @@ import os import sys from importlib import reload +from unittest import mock import pytest from cloudinit import apport -from tests.unittests.helpers import mock M_PATH = "cloudinit.apport." diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 14edb44cde2..cd545479729 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -6,6 +6,7 @@ import errno import os import re +from unittest import mock import pytest @@ -25,7 +26,6 @@ path_map, ) from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE -from tests.unittests.helpers import mock, skipUnlessJinja from tests.unittests.util import FakeDataSource INSTANCE_DATA_FILE = "instance-data-sensitive.json" @@ -95,7 +95,6 @@ def test_jinja_template_handle_noop_on_content_signals(self, paths): ) m_handle_part.assert_not_called() - @skipUnlessJinja() def test_jinja_template_handle_subhandler_v2_with_clean_payload( self, paths ): @@ -122,7 +121,6 @@ def test_jinja_template_handle_subhandler_v2_with_clean_payload( "data", "!__begin__", "part01", "#!/bin/bash\necho himom", "freq" ) - @skipUnlessJinja() def test_jinja_template_handle_subhandler_v3_with_clean_payload( self, paths ): @@ -208,7 +206,6 @@ def test_jinja_template_handle_errors_on_unreadable_instance_data( "Unexpected file created %s" % script_file ) - @skipUnlessJinja() def test_jinja_template_handle_renders_jinja_content(self, paths, caplog): """When present, render jinja variables from instance data""" script_handler = ShellScriptPartHandler(paths) @@ -237,7 +234,6 @@ def test_jinja_template_handle_renders_jinja_content(self, paths, caplog): ) assert "#!/bin/bash\necho himom" == util.load_text_file(script_file) - @skipUnlessJinja() def test_jinja_template_handle_renders_jinja_content_missing_keys( self, paths, caplog ): @@ -349,7 +345,7 @@ def test_convert_instance_data_decodes_decode_paths(self): class TestRenderJinjaPayload: - @skipUnlessJinja() + def test_render_jinja_payload_logs_jinja_vars_on_debug(self, caplog): """When debug is True, log jinja variables available.""" payload = ( @@ -372,7 +368,6 @@ def test_render_jinja_payload_logs_jinja_vars_on_debug(self, caplog): ) assert re.match(expected_log, caplog.text, re.DOTALL) - @skipUnlessJinja() def test_render_jinja_payload_replaces_missing_variables_and_warns( self, caplog ): diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 77b1cc244d0..ff06cfe567c 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -7,6 +7,7 @@ import os import sys from collections import namedtuple +from unittest import mock import pytest @@ -14,8 +15,6 @@ from cloudinit.cmd import main as cli from tests.unittests import helpers as test_helpers -mock = test_helpers.mock - M_PATH = "cloudinit.cmd.main." Tmpdir = namedtuple("Tmpdir", ["tmpdir", "link_d", "data_d"]) FakeArgs = namedtuple("FakeArgs", ["action", "local", "mode"]) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index ba5aae5ebb7..d9ef8ef441f 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -663,7 +663,7 @@ def test_wb_print_variables(self, tmp_path): # Launched by os code always has config-2 disk. pytest.param("IBMCloud-config-2", True, id="ibmcloud_os_code"), # Test that Aliyun cloud is identified by product id. - pytest.param("AliYun", True, id="ibmcloud_os_code"), + pytest.param("AliYun", True, id="aliyun_found_by_product_id"), # On Intel, openstack must be identified. pytest.param( "OpenStack", True, id="default_openstack_intel_is_found" diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 5b92ba28041..81c2b8d612c 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2229,13 +2229,7 @@ def test_config_with_explicit_loopback(self): "expected_name,yaml_version", [ ("bond_v1", "yaml"), - pytest.param( - "bond_v2", - "yaml", - marks=pytest.mark.xfail( - reason="Bond MAC address not rendered" - ), - ), + ("bond_v2", "yaml"), ("vlan_v1", "yaml"), ("vlan_v2", "yaml"), ("bridge", "yaml_v1"), @@ -2884,13 +2878,7 @@ def test_config_with_explicit_loopback(self): "expected_name,yaml_name", [ ("bond_v1", "yaml"), - pytest.param( - "bond_v2", - "yaml", - marks=pytest.mark.xfail( - reason="Bond MAC address not rendered" - ), - ), + ("bond_v2", "yaml"), ("vlan_v1", "yaml"), ("vlan_v2", "yaml"), ("bridge", "yaml_v1"), @@ -5141,8 +5129,8 @@ class TestRenderersSelect: ("eni", False, True, False, False, False), # +netplan +ifupdown -sys -nm -networkd selects eni ("eni", True, True, False, False, False), - # +netplan -ifupdown -sys -nm -networkd selects netplan - ("netplan", True, False, False, False, False), + # +netplan -ifupdown -sys +nm -networkd selects netplan + ("netplan", True, False, False, True, False), # +netplan -ifupdown -sys -nm -networkd selects netplan ("netplan", True, False, False, False, False), # -netplan -ifupdown +sys -nm -networkd selects sysconfig diff --git a/tests/unittests/test_netinfo.py b/tests/unittests/test_netinfo.py index 90f78356a9d..35b064678c0 100644 --- a/tests/unittests/test_netinfo.py +++ b/tests/unittests/test_netinfo.py @@ -4,6 +4,7 @@ import json from copy import copy +from unittest import mock import pytest @@ -14,7 +15,7 @@ netdev_pformat, route_pformat, ) -from tests.unittests.helpers import mock, readResource +from tests.unittests.helpers import readResource # Example ifconfig and route output SAMPLE_OLD_IFCONFIG_OUT = readResource("netinfo/old-ifconfig-output") diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index 7f8fc944429..32926da1dd2 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -93,6 +93,7 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): default_user_exceptions = { "amazon": "ec2-user", + "azurelinux": "azureuser", "rhel": "cloud-user", "centos": "cloud-user", "raspberry-pi-os": "pi", diff --git a/tests/unittests/test_stages.py b/tests/unittests/test_stages.py index 85846352be6..b217272b1df 100644 --- a/tests/unittests/test_stages.py +++ b/tests/unittests/test_stages.py @@ -5,6 +5,7 @@ import json import os import stat +from unittest import mock import pytest @@ -13,7 +14,6 @@ from cloudinit.helpers import Paths from cloudinit.sources import DataSource, NetworkConfigSource from cloudinit.util import sym_link, write_file -from tests.unittests.helpers import mock from tests.unittests.util import TEST_INSTANCE_ID, FakeDataSource M_PATH = "cloudinit.stages." diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 83c1ba4fc69..a5aab1f9a03 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -4,16 +4,13 @@ # # This file is part of cloud-init. See LICENSE file for license information. -import logging import textwrap -from unittest import mock import pytest from cloudinit import templater from cloudinit.templater import JinjaSyntaxParsingException from cloudinit.util import load_binary_file, write_file -from tests.unittests import helpers as test_helpers class TestTemplates: @@ -137,26 +134,6 @@ def test_jinja_nonascii_render_from_file(self, tmp_path): result = templater.render_from_file(tmpl_fn, {"name": "bob"}) assert result == self.jinja_utf8_rbob - @test_helpers.skipIfJinja() - def test_jinja_warns_on_missing_dep_and_uses_basic_renderer( - self, caplog, tmp_path - ): - """Test jinja render_from_file will fallback to basic renderer.""" - tmpl_fn = str(tmp_path / "j-render-from-file.template") - write_file( - tmpl_fn, - omode="wb", - content=self.add_header("jinja", self.jinja_utf8).encode("utf-8"), - ) - result = templater.render_from_file(tmpl_fn, {"name": "bob"}) - assert result == self.jinja_utf8.decode() - assert ( - mock.ANY, - logging.WARNING, - "Jinja not available as the selected renderer for desired" - " template, reverting to the basic renderer.", - ) in caplog.record_tuples - def test_jinja_do_extension_render_to_string(self): """Test jinja render_to_string using do extension.""" expected_result = "[1, 2, 3]" diff --git a/tests/unittests/test_upgrade.py b/tests/unittests/test_upgrade.py index 85209409ffc..5cdd3fbe9f4 100644 --- a/tests/unittests/test_upgrade.py +++ b/tests/unittests/test_upgrade.py @@ -294,6 +294,10 @@ def test_pkl_load_defines_all_init_side_effect_attributes( missing_attrs = ds.__dict__.keys() - previous_obj_pkl.__dict__.keys() for attr in missing_attrs: assert attr in expected + missing_ds_cfg_attrs = ( + ds.ds_cfg.keys() - previous_obj_pkl.ds_cfg.keys() + ) + assert set() == missing_ds_cfg_attrs def test_networking_set_on_distro(self, previous_obj_pkl): """We always expect to have ``.networking`` on ``Distro`` objects.""" diff --git a/tests/unittests/test_url_helper.py b/tests/unittests/test_url_helper.py index e441f85ca7f..4bf87f7600b 100644 --- a/tests/unittests/test_url_helper.py +++ b/tests/unittests/test_url_helper.py @@ -283,7 +283,16 @@ def request(cls, **kwargs): class TestReadFileOrUrlParameters: @mock.patch(M_PATH + "readurl") @pytest.mark.parametrize( - "timeout", [1, 1.2, "1", (1, None), (1, 1), (None, None)] + "timeout", + [1, 1.2, "1", (1, None), (1, 1), (None, None)], + ids=[ + "timeout-int", + "timeout-float", + "timeout-str-int", + "timeout-tuple-write-default", + "timeout-tuple-read-write", + "timeout-tuple-none", + ], ) def test_read_file_or_url_passes_params_to_readurl( self, m_readurl, timeout @@ -322,6 +331,17 @@ def test_read_file_or_url_passes_params_to_readurl( ((1, 1), (1, 1)), ((None, None), (None, None)), ], + ids=[ + "negative-int-defaults-to-zero", + "negative-str-defaults-to-zero", + "none-timeout", + "int-timeout", + "float-timeout", + "str-int-timeout", + "tuple-read-timeout-only", + "tuple-read-write-timeout", + "tuple-none-timeout", + ], ) def test_readurl_timeout(self, readurl_timeout, request_timeout): url = "http://hostname/path" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 90206489987..fe24962e19b 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -35,7 +35,7 @@ from cloudinit.sources import DataSourceHostname from cloudinit.subp import SubpResult from tests.unittests import helpers -from tests.unittests.helpers import random_string, skipUnlessJinja +from tests.unittests.helpers import random_string LOG = logging.getLogger(__name__) M_PATH = "cloudinit.util." @@ -409,15 +409,23 @@ OS_RELEASE_AZURELINUX = dedent( """\ - NAME="Microsoft Azure Linux" - VERSION="3.0.20240206" + NAME="Azure Linux" + VERSION="4.0 (Cloud Variant)" + RELEASE_TYPE=stable ID=azurelinux - VERSION_ID="3.0" - PRETTY_NAME="Microsoft Azure Linux 3.0" - ANSI_COLOR="1;34" + ID_LIKE=fedora + VERSION_ID=4.0 + VERSION_CODENAME="" + PRETTY_NAME="Azure Linux 4.0 (Cloud Variant)" + ANSI_COLOR="0;38;2;60;110;180" + LOGO=azurelinux-logo-icon + CPE_NAME="cpe:/o:microsoft:azurelinux:4.0" HOME_URL="https://aka.ms/azurelinux" - BUG_REPORT_URL="https://aka.ms/azurelinux" + DOCUMENTATION_URL="https://aka.ms/azurelinux" SUPPORT_URL="https://aka.ms/azurelinux" + BUG_REPORT_URL="https://aka.ms/azurelinux" + VARIANT="Cloud Variant" + VARIANT_ID=cloud """ ) @@ -470,7 +478,6 @@ def test_read_conf(self, mocker): ) assert util.read_conf("any") == {"a": "b"} - @skipUnlessJinja() def test_read_conf_with_template(self, mocker, caplog): mocker.patch("os.path.exists", return_value=True) mocker.patch( @@ -489,7 +496,6 @@ def test_read_conf_with_template(self, mocker, caplog): "from 'cfg_path'" ) in caplog.text - @skipUnlessJinja() def test_read_conf_with_failed_config_json(self, mocker, caplog): mocker.patch("os.path.exists", return_value=True) mocker.patch( @@ -504,7 +510,6 @@ def test_read_conf_with_failed_config_json(self, mocker, caplog): assert "Failed loading yaml blob" in caplog.text assert conf == {} - @skipUnlessJinja() def test_read_conf_with_failed_instance_data_json(self, mocker, caplog): mocker.patch("os.path.exists", return_value=True) mocker.patch( @@ -527,7 +532,6 @@ def test_read_conf_with_failed_instance_data_json(self, mocker, caplog): "{% if c %} C is present {% else % } C is NOT present {% endif %}", ], ) - @skipUnlessJinja() def test_read_conf_with_config_invalid_jinja_syntax( self, mocker, caplog, template ): @@ -1280,7 +1284,7 @@ def test_get_linux_azurelinux_os_release( m_os_release.return_value = OS_RELEASE_AZURELINUX m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists dist = util.get_linux_distro() - assert ("azurelinux", "3.0", "") == dist + assert ("azurelinux", "4.0", "Cloud Variant") == dist @mock.patch(M_PATH + "load_text_file") def test_get_linux_openmandriva(self, m_os_release, m_path_exists): @@ -3173,6 +3177,12 @@ def test_from_str(self, str_ver, cls_ver): @pytest.mark.allow_dns_lookup class TestResolvable: + @pytest.fixture(autouse=True) + def reset_dns_redirect_ip(self): + util._DNS_REDIRECT_IP = None + yield # Test runs here + util._DNS_REDIRECT_IP = None + @mock.patch.object(util, "_DNS_REDIRECT_IP", return_value=True) @mock.patch.object(util.socket, "getaddrinfo") def test_ips_need_not_be_resolved(self, m_getaddr, m_dns): @@ -3183,8 +3193,45 @@ def test_ips_need_not_be_resolved(self, m_getaddr, m_dns): """ assert util.is_resolvable("http://169.254.169.254/") is True assert util.is_resolvable("http://[fd00:ec2::254]/") is True + assert util.is_resolvable("http://169.254.169.254:80") is True + assert util.is_resolvable("http://[fd00:ec2::254]:80/") is True assert not m_getaddr.called + @mock.patch.object(util.net, "is_ip_address") + @mock.patch.object(util.socket, "getaddrinfo") + def test_hostnames_require_dns_resolution(self, m_getaddr, m_is_ip): + """Hostnames should go through DNS resolution.""" + m_is_ip.return_value = False + + def mock_getaddrinfo(host, port, *args, **kwargs): + badnames = ( + "does-not-exist.example.com.", + "example.invalid.", + "__cloud_init_expected_not_found__", + ) + if host in badnames: + return [(None, None, None, "badname", ("192.0.2.1", 0))] + return [(None, None, None, "example.com", ("10.2.3.4", 0))] + + m_getaddr.side_effect = mock_getaddrinfo + + assert util.is_resolvable("http://example.com/") is True + assert util.is_resolvable("http://example.com/:80") is True + assert m_getaddr.called + + assert m_getaddr.call_args_list[0] == mock.call("example.com", None) + + badnames = ( + "does-not-exist.example.com.", + "example.invalid.", + "__cloud_init_expected_not_found__", + ) + called_hosts = [call[0][0] for call in m_getaddr.call_args_list[1:]] + for badname in badnames: + assert ( + badname in called_hosts + ), f"Expected badname {badname} to be checked" + class TestMaybeB64Decode: """Test the maybe_b64decode helper function.""" diff --git a/tools/Z99-cloud-locale-test.sh b/tools/Z99-cloud-locale-test.sh index e051fa7369a..9ba7716532f 100644 --- a/tools/Z99-cloud-locale-test.sh +++ b/tools/Z99-cloud-locale-test.sh @@ -6,12 +6,12 @@ # (c) 2012, Canonical Group, Ltd. # # This file is part of cloud-init. See LICENSE file for license information. - + # Purpose: Detect invalid locale settings and inform the user # of how to fix them. locale_warn() { - command -v local >/dev/null && local _local="local" || + command -v local > /dev/null && local _local="local" || typeset _local="typeset" $_local bad_names="" bad_lcs="" key="" val="" var="" vars="" bad_kv="" @@ -27,13 +27,14 @@ locale_warn() { # locale: Cannot set LC_SOMETHING to default locale while read -r w1 w2 w3 w4 remain; do case "$w1" in - locale:) bad_names="${bad_names} ${w4}";; + locale:) bad_names="${bad_names} ${w4}" ;; *) key=${w1%%=*} val=${w1#*=} val=${val#\"} val=${val%\"} - vars="${vars} $key=$val";; + vars="${vars} $key=$val" + ;; esac done for bad in $bad_names; do @@ -93,7 +94,7 @@ locale_warn() { printf "_____________________________________________________________________\n\n" # only show the message once - : > ~/.cloud-locale-test.skip 2>/dev/null || : + : > ~/.cloud-locale-test.skip 2> /dev/null || : } [ -f ~/.cloud-locale-test.skip -o -f /var/lib/cloud/instance/locale-check.skip ] || diff --git a/tools/Z99-cloudinit-warnings.sh b/tools/Z99-cloudinit-warnings.sh index 560902350ca..297fbee8380 100644 --- a/tools/Z99-cloudinit-warnings.sh +++ b/tools/Z99-cloudinit-warnings.sh @@ -4,7 +4,7 @@ # Purpose: show user warnings on login. cloud_init_warnings() { - command -v local >/dev/null && local _local="local" || + command -v local > /dev/null && local _local="local" || typeset _local="typeset" $_local warning="" idir="/var/lib/cloud/instance" n=0 $_local warndir="$idir/warnings" @@ -16,7 +16,7 @@ cloud_init_warnings() { for warning in "$warndir"/*; do [ -f "$warning" ] || continue cat "$warning" - n=$((n+1)) + n=$((n + 1)) done [ $n -eq 0 ] && return 0 echo "" diff --git a/tools/benchmark.sh b/tools/benchmark.sh deleted file mode 100755 index c382f374472..00000000000 --- a/tools/benchmark.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/bash - -BIN="$@" -chmod +x "$BIN" - -time for _ in $(seq 1 $ITER); do - "$BIN"; -done diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index d7b3e354062..c5cc6e23662 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -8,7 +8,10 @@ set -eux -fail() { echo "FAILED:" "$@" 1>&2; exit 1; } +fail() { + echo "FAILED:" "$@" 1>&2 + exit 1 +} PYTHON="${PYTHON:-python3}" if [ ! $(which ${PYTHON}) ]; then diff --git a/tools/build-on-netbsd b/tools/build-on-netbsd index b743d591b6e..25db86f66ec 100755 --- a/tools/build-on-netbsd +++ b/tools/build-on-netbsd @@ -1,6 +1,9 @@ #!/bin/sh -fail() { echo "FAILED:" "$@" 1>&2; exit 1; } +fail() { + echo "FAILED:" "$@" 1>&2 + exit 1 +} PYTHON="${PYTHON:-python3}" if [ ! $(which ${PYTHON}) ]; then diff --git a/tools/build-on-openbsd b/tools/build-on-openbsd index 94c66dd1744..1665f6b918c 100755 --- a/tools/build-on-openbsd +++ b/tools/build-on-openbsd @@ -1,51 +1,27 @@ #!/bin/sh +# This script provides a quick and dirty way of building and installing +# cloud-init on OpenBSD. -fail() { echo "FAILED:" "$@" 1>&2; exit 1; } +set -eux + +fail() { + echo "FAILED:" "$@" 1>&2 + exit 1 +} PYTHON=${PYTHON:-python3} -if ! command -v ${PYTHON} >/dev/null 2>&1; then +if ! command -v ${PYTHON} > /dev/null 2>&1; then echo "Please install python first." exit 1 fi # Check dependencies: depschecked=/tmp/c-i.dependencieschecked -pkgs=" - bash - dmidecode - py3-configobj - py3-jinja2 - py3-jsonschema - py3-jsonpatch - py3-jsonpointer - py3-oauthlib - py3-requests - py3-setuptools - py3-serial - py3-yaml - sudo-- - wget -" - -[ -f $depschecked ] || echo "Installing the following packages: $pkgs"; output=$(pkg_add -zI $pkgs 2>&1) - +[ -f "$depschecked" ] || ./tools/read-dependencies --distro openbsd -t || fail "install packages" +touch "$depschecked" -if echo "$output" | grep -q -e "Can't find" -e "Ambiguous"; then - echo "Failed to find or install one or more packages" - echo "Failed Package(s):" - echo "$output" - exit 1 -else - echo Successfully installed packages - touch $depschecked - - python3 setup.py build - python3 setup.py install -O1 --distro openbsd --skip-build --init-system sysvinit_openbsd +# Build the code and install in /usr/local/: +meson setup builddir -Dinit_system=sysvinit_openbsd -Dsysconfdir=/etc +meson install -C builddir - echo "Installation completed." - - rcctl enable cloudinitlocal - rcctl enable cloudinit - rcctl enable cloudconfig - rcctl enable cloudfinal -fi +echo "Installation completed." diff --git a/tools/check_json_format.sh b/tools/check_json_format.sh deleted file mode 100755 index 62f7d6cd74e..00000000000 --- a/tools/check_json_format.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# -# Run python's json.tool and check for changes -# -# requires python 3.9 for --indent -# -file=$1 -before=$(cat "$file") && - python3 -m json.tool --indent 2 "$file" "$file" && - after=$(cat "$file") && - test "$before" = "$after" diff --git a/tools/cloud-init-hotplugd b/tools/cloud-init-hotplugd index eb811d69b10..4df5dd55693 100755 --- a/tools/cloud-init-hotplugd +++ b/tools/cloud-init-hotplugd @@ -14,9 +14,9 @@ PIPE="/run/cloud-init/share/hook-hotplug-cmd" [ -p $PIPE ] || mkfifo -m700 $PIPE while true; do - # shellcheck disable=SC2162 - if read args < $PIPE; then - # shellcheck disable=SC2086 - /usr/bin/cloud-init devel hotplug-hook $args - fi + # shellcheck disable=SC2162 + if read args < $PIPE; then + # shellcheck disable=SC2086 + /usr/bin/cloud-init devel hotplug-hook $args + fi done diff --git a/tools/cloud-init-per b/tools/cloud-init-per index fcd1ea796df..07adf40890f 100755 --- a/tools/cloud-init-per +++ b/tools/cloud-init-per @@ -5,7 +5,7 @@ DATA_PRE="/var/lib/cloud/sem/bootper" INST_PRE="/var/lib/cloud/instance/sem/bootper" Usage() { - cat <&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } +fail() { + [ $# -eq 0 ] || error "$@" + exit 1 +} # support the old 'cloud-init-run-module freq name "execute" cmd arg1' # if < 3 arguments, it will fail below on usage. if [ "${0##*/}" = "cloud-init-run-module" ]; then - if [ $# -le 2 -o "$3" = "execute" ]; then - error "Warning: ${0##*/} is deprecated. Please use cloud-init-per." - freq=$1; name=$2; - [ $# -le 2 ] || shift 3; - set -- "$freq" "$name" "$@" - else - fail "legacy cloud-init-run-module only supported with module 'execute'" - fi + if [ $# -le 2 -o "$3" = "execute" ]; then + error "Warning: ${0##*/} is deprecated. Please use cloud-init-per." + freq=$1 + name=$2 + [ $# -le 2 ] || shift 3 + set -- "$freq" "$name" "$@" + else + fail "legacy cloud-init-run-module only supported with module 'execute'" + fi fi -[ "$1" = "-h" -o "$1" = "--help" ] && { Usage ; exit 0; } -[ $# -ge 3 ] || { Usage 1>&2; exit 1; } +[ "$1" = "-h" -o "$1" = "--help" ] && { + Usage + exit 0 +} +[ $# -ge 3 ] || { + Usage 1>&2 + exit 1 +} freq=$1 name=$(echo $2 | sed 's/-/_/g') -shift 2; +shift 2 [ "${name#*/}" = "${name}" ] || fail "name cannot contain a /" [ "$(id -u)" = "0" ] || fail "must be root" case "$freq" in - once|always) sem="${DATA_PRE}.$name.$freq";; - instance) sem="${INST_PRE}.$name.$freq";; - *) Usage 1>&2; fail "invalid frequency: $freq";; + once | always) sem="${DATA_PRE}.$name.$freq" ;; + instance) sem="${INST_PRE}.$name.$freq" ;; + *) + Usage 1>&2 + fail "invalid frequency: $freq" + ;; esac [ -d "${sem%/*}" ] || mkdir -p "${sem%/*}" || - fail "failed to make directory for ${sem}" + fail "failed to make directory for ${sem}" # Rename legacy sem files with dashes in their names. Do not overwrite existing # sem files to prevent clobbering those which may have been created from calls @@ -63,5 +76,5 @@ sem_legacy=$(echo $sem | sed 's/_/-/g') "$@" ret=$? printf "%s\t%s\n" "$ret" "$(date +%s)" > "$sem" || - fail "failed to write to $sem" + fail "failed to write to $sem" exit $ret diff --git a/tools/ds-identify b/tools/ds-identify index 02c70a62c2a..82f6b036bab 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -143,7 +143,7 @@ DI_EC2_STRICT_ID_DEFAULT="true" _IS_IBM_CLOUD="" error() { - set -- "ERROR:" "$@"; + set -- "ERROR:" "$@" debug 0 "$@" stderr "$@" } @@ -166,7 +166,7 @@ debug() { if [ "$_DI_LOGGED" != "$DI_LOG" ]; then # first time here, open file descriptor for append case "$DI_LOG" in - stderr) :;; + stderr) : ;; ?*/*) if [ ! -d "${DI_LOG%/*}" ]; then mkdir -p "${DI_LOG%/*}" || { @@ -174,12 +174,13 @@ debug() { DI_LOG="stderr" } fi + ;; esac if [ "$DI_LOG" = "stderr" ]; then exec 3>&2 else - ( exec 3>>"$DI_LOG" ) && exec 3>>"$DI_LOG" || { - stderr "ERROR: failed writing to $DI_LOG. logging to stderr."; + (exec 3>> "$DI_LOG") && exec 3>> "$DI_LOG" || { + stderr "ERROR: failed writing to $DI_LOG. logging to stderr." exec 3>&2 DI_LOG="stderr" } @@ -191,71 +192,77 @@ debug() { get_kenv_field() { local sys_field="$1" kenv_field="" val="" - command -v kenv >/dev/null 2>&1 || { + command -v kenv > /dev/null 2>&1 || { warn "No kenv program. Cannot read $sys_field." return 1 } case "$sys_field" in - board_asset_tag) kenv_field="smbios.planar.tag";; - board_vendor) kenv_field='smbios.planar.maker';; - board_name) kenv_field='smbios.planar.product';; - board_serial) kenv_field='smbios.planar.serial';; - board_version) kenv_field='smbios.planar.version';; - bios_date) kenv_field='smbios.bios.reldate';; - bios_vendor) kenv_field='smbios.bios.vendor';; - bios_version) kenv_field='smbios.bios.version';; - chassis_asset_tag) kenv_field='smbios.chassis.tag';; - chassis_vendor) kenv_field='smbios.chassis.maker';; - chassis_serial) kenv_field='smbios.chassis.serial';; - chassis_version) kenv_field='smbios.chassis.version';; - sys_vendor) kenv_field='smbios.system.maker';; - product_name) kenv_field='smbios.system.product';; - product_serial) kenv_field='smbios.system.serial';; - product_uuid) kenv_field='smbios.system.uuid';; - *) error "Unknown field $sys_field. Cannot call kenv." - return 1;; + board_asset_tag) kenv_field="smbios.planar.tag" ;; + board_vendor) kenv_field='smbios.planar.maker' ;; + board_name) kenv_field='smbios.planar.product' ;; + board_serial) kenv_field='smbios.planar.serial' ;; + board_version) kenv_field='smbios.planar.version' ;; + bios_date) kenv_field='smbios.bios.reldate' ;; + bios_vendor) kenv_field='smbios.bios.vendor' ;; + bios_version) kenv_field='smbios.bios.version' ;; + chassis_asset_tag) kenv_field='smbios.chassis.tag' ;; + chassis_vendor) kenv_field='smbios.chassis.maker' ;; + chassis_serial) kenv_field='smbios.chassis.serial' ;; + chassis_version) kenv_field='smbios.chassis.version' ;; + sys_vendor) kenv_field='smbios.system.maker' ;; + product_name) kenv_field='smbios.system.product' ;; + product_serial) kenv_field='smbios.system.serial' ;; + product_uuid) kenv_field='smbios.system.uuid' ;; + *) + error "Unknown field $sys_field. Cannot call kenv." + return 1 + ;; esac - val=$(kenv -q "$kenv_field" 2>/dev/null) || return 1 + val=$(kenv -q "$kenv_field" 2> /dev/null) || return 1 _RET="$val" } get_sysctl_field() { local sys_field="$1" sysctl_field="" val="" - command -v sysctl >/dev/null 2>&1 || { + command -v sysctl > /dev/null 2>&1 || { warn "No sysctl program. Cannot read $sys_field." return 1 } case "$sys_field" in - chassis_vendor) sysctl_field='hw.vendor';; - chassis_serial) sysctl_field='hw.type';; - chassis_version) sysctl_field='hw.uuid';; - sys_vendor) sysctl_field='hw.vendor';; - product_name) sysctl_field='hw.product';; - product_serial) sysctl_field='hw.uuid';; - product_uuid) sysctl_field='hw.uuid';; - *) error "Unknown field $sys_field. Cannot call sysctl." - return 1;; + chassis_vendor) sysctl_field='hw.vendor' ;; + chassis_serial) sysctl_field='hw.type' ;; + chassis_version) sysctl_field='hw.uuid' ;; + sys_vendor) sysctl_field='hw.vendor' ;; + product_name) sysctl_field='hw.product' ;; + product_serial) sysctl_field='hw.uuid' ;; + product_uuid) sysctl_field='hw.uuid' ;; + *) + error "Unknown field $sys_field. Cannot call sysctl." + return 1 + ;; esac - val=$(sysctl -nq "$sysctl_field" 2>/dev/null) || return 1 + val=$(sysctl -nq "$sysctl_field" 2> /dev/null) || return 1 _RET="$val" } dmi_decode() { local sys_field="$1" dmi_field="" val="" - command -v dmidecode >/dev/null 2>&1 || { + command -v dmidecode > /dev/null 2>&1 || { warn "No dmidecode program. Cannot read $sys_field." return 1 } case "$sys_field" in - sys_vendor) dmi_field="system-manufacturer";; - product_name) dmi_field="system-product-name";; - product_uuid) dmi_field="system-uuid";; - product_serial) dmi_field="system-serial-number";; - chassis_asset_tag) dmi_field="chassis-asset-tag";; - *) error "Unknown field $sys_field. Cannot call dmidecode." - return 1;; + sys_vendor) dmi_field="system-manufacturer" ;; + product_name) dmi_field="system-product-name" ;; + product_uuid) dmi_field="system-uuid" ;; + product_serial) dmi_field="system-serial-number" ;; + chassis_asset_tag) dmi_field="chassis-asset-tag" ;; + *) + error "Unknown field $sys_field. Cannot call dmidecode." + return 1 + ;; esac - val=$(dmidecode --quiet "--string=$dmi_field" 2>/dev/null) || return 1 + val=$(dmidecode --quiet "--string=$dmi_field" 2> /dev/null) || return 1 _RET="$val" } @@ -295,7 +302,7 @@ ensure_sane_path() { local t for t in /sbin /usr/sbin /bin /usr/bin; do case ":$PATH:" in - *:$t:*|*:$t/:*) continue;; + *:$t:* | *:$t/:*) continue ;; esac PATH="${PATH:+${PATH}:}$t" done @@ -341,20 +348,28 @@ read_fs_info_linux() { # empty lines in "$@" below. # shellcheck disable=2086 - { IFS="$CR"; set -- $DI_BLKID_EXPORT_OUT; IFS="$oifs"; } + { + IFS="$CR" + set -- $DI_BLKID_EXPORT_OUT + IFS="$oifs" + } for line in "$@"; do case "${line}" in DEVNAME=*) [ -n "$dev" -a "$ftype" = "iso9660" ] && isodevs="${isodevs},${dev}=$label" - ftype=""; dev=""; label=""; - dev=${line#DEVNAME=};; - LABEL=*|LABEL_FATBOOT=*) - label="${line#*=}"; - labels="${labels}${label}${delim}";; - TYPE=*) ftype=${line#TYPE=};; - UUID=*) uuids="${uuids}${line#UUID=}$delim";; + ftype="" + dev="" + label="" + dev=${line#DEVNAME=} + ;; + LABEL=* | LABEL_FATBOOT=*) + label="${line#*=}" + labels="${labels}${label}${delim}" + ;; + TYPE=*) ftype=${line#TYPE=} ;; + UUID=*) uuids="${uuids}${line#UUID=}$delim" ;; esac done [ -n "$dev" -a "$ftype" = "iso9660" ] && @@ -395,7 +410,11 @@ read_fs_info_freebsd() { # iso9660/cidata N/A vtbd2 # shellcheck disable=2086 - { IFS="$CR"; set -- $DI_GEOM_LABEL_STATUS_OUT; IFS="$oifs"; } + { + IFS="$CR" + set -- $DI_GEOM_LABEL_STATUS_OUT + IFS="$oifs" + } for line in "$@"; do # shellcheck disable=2086 @@ -451,16 +470,16 @@ detect_virt() { fi debug 2 "detected $virt via ds-identify" fi - elif command -v virt-what >/dev/null 2>&1; then + elif command -v virt-what > /dev/null 2>&1; then # Map virt-what's names to those systemd-detect-virt that # don't match up. out=$(virt-what 2>&1 | head -n 1) && { case "$out" in - ibm_systemz-zvm) virt="zvm" ;; - hyperv) virt="microsoft" ;; - virtualbox) virt="oracle" ;; - xen-domU) virt="xen" ;; - *) virt="$out" + ibm_systemz-zvm) virt="zvm" ;; + hyperv) virt="microsoft" ;; + virtualbox) virt="oracle" ;; + xen-domU) virt="xen" ;; + *) virt="$out" ;; esac } elif [ "$DI_UNAME_KERNEL_NAME" = "FreeBSD" -o "$DI_UNAME_KERNEL_NAME" = "Dragonfly" ]; then @@ -480,15 +499,15 @@ detect_virt() { # parallels | parallels # bhyve | bhyve # vm-other | generic - out=$(sysctl -qn kern.vm_guest 2>/dev/null) && { + out=$(sysctl -qn kern.vm_guest 2> /dev/null) && { case "$out" in hv) virt="microsoft" ;; vbox) virt="oracle" ;; - generic) virt="vm-other";; - *) virt="$out" + generic) virt="vm-other" ;; + *) virt="$out" ;; esac } - out=$(sysctl -qn security.jail.jailed 2>/dev/null) && { + out=$(sysctl -qn security.jail.jailed 2> /dev/null) && { if [ "$out" = "1" ]; then virt="jail" fi @@ -505,8 +524,8 @@ read_virt() { is_container() { case "${DI_VIRT}" in - container-other|lxc|lxc-libvirt|systemd-nspawn|docker|rkt|jail) return 0;; - *) return 1;; + container-other | lxc | lxc-libvirt | systemd-nspawn | docker | rkt | jail) return 0 ;; + *) return 1 ;; esac } @@ -524,7 +543,7 @@ read_kernel_cmdline() { cmdline=$x fi elif [ -f "$fpath" ]; then - read cmdline <"$fpath" + read cmdline < "$fpath" else cmdline="${UNAVAILABLE}:no-cmdline" fi @@ -609,7 +628,11 @@ parse_yaml_array() { val=${val#"["} val=${val%"]"} # shellcheck disable=2086 - { IFS=","; set -- $val; IFS="$oifs"; } + { + IFS="," + set -- $val + IFS="$oifs" + } for tok in "$@"; do trim "$tok" unquote "$_RET" @@ -655,9 +678,13 @@ read_pid1_product_name() { local oifs="$IFS" out="" tok="" key="" val="" product_name="${UNAVAILABLE}" cached "${DI_PID_1_PRODUCT_NAME}" && return [ -r "${PATH_PROC_1_ENVIRON}" ] || return - out=$(tr '\0' '\n' <"${PATH_PROC_1_ENVIRON}") + out=$(tr '\0' '\n' < "${PATH_PROC_1_ENVIRON}") # shellcheck disable=2086 - { IFS="$CR"; set -- $out; IFS="$oifs"; } + { + IFS="$CR" + set -- $out + IFS="$oifs" + } for tok in "$@"; do key=${tok%%=*} [ "$key" != "$tok" ] || continue @@ -671,7 +698,7 @@ dmi_chassis_asset_tag_matches() { is_container && return 1 # shellcheck disable=2254 case "${DI_DMI_CHASSIS_ASSET_TAG}" in - $1) return 0;; + $1) return 0 ;; esac return 1 } @@ -680,7 +707,7 @@ dmi_product_name_matches() { is_container && return 1 # shellcheck disable=2254 case "${DI_DMI_PRODUCT_NAME}" in - $1) return 0;; + $1) return 0 ;; esac return 1 } @@ -689,7 +716,7 @@ dmi_product_serial_matches() { is_container && return 1 # shellcheck disable=2254 case "${DI_DMI_PRODUCT_SERIAL}" in - $1) return 0;; + $1) return 0 ;; esac return 1 } @@ -701,7 +728,7 @@ dmi_sys_vendor_is() { has_fs_with_uuid() { case ",${DI_FS_UUIDS}," in - *,$1,*) return 0;; + *,$1,*) return 0 ;; esac return 1 } @@ -712,7 +739,7 @@ has_fs_with_label() { local label="" for label in "$@"; do case ",${DI_FS_LABELS}," in - *,$label,*) return 0;; + *,$label,*) return 0 ;; esac done return 1 @@ -768,16 +795,25 @@ probe_floppy() { local fpath=/dev/floppy [ -b "$fpath" ] || - { STATE_FLOPPY_PROBED=1; return 1; } + { + STATE_FLOPPY_PROBED=1 + return 1 + } # Use "-b" option as Busybox modprobe doesn't support long-option - modprobe -b floppy >/dev/null 2>&1 || - { STATE_FLOPPY_PROBED=1; return 1; } + modprobe -b floppy > /dev/null 2>&1 || + { + STATE_FLOPPY_PROBED=1 + return 1 + } # Some Linux distros/non-Linux OSes may not have udev if command -v udevadm; then udevadm settle "--exit-if-exists=$fpath" || - { STATE_FLOPPY_PROBED=1; return 1; } + { + STATE_FLOPPY_PROBED=1 + return 1 + } fi [ -b "$fpath" ] @@ -827,7 +863,11 @@ check_config() { files="$*" fi # shellcheck disable=2086 - { set +f; set -- $files; set -f; } + { + set +f + set -- $files + set -f + } if [ "$1" = "$files" -a ! -f "$1" ]; then return 1 fi @@ -846,11 +886,11 @@ check_config() { # key: [ some_value ] # key : [ "some value" ] # key\t:\t[\tsome_value\t]\t - # + # # The syntax warned about is not valid posix shell, and we are not # attempting to access an index of arrays. Silence it. # shellcheck disable=1087 - out=$(grep "$key[\"\']*[[:space:]]*:" "$@" 2>/dev/null) + out=$(grep "$key[\"\']*[[:space:]]*:" "$@" 2> /dev/null) IFS=${CR} for line in $out; do # drop '# comment' @@ -869,8 +909,8 @@ check_config() { continue fi fi - ret=${line#*: }; - found=$((found+1)) + ret=${line#*: } + found=$((found + 1)) done IFS="$oifs" if [ $found -ne 0 ]; then @@ -959,15 +999,17 @@ dscheck_LXD() { return ${DS_NOT_FOUND} fi # Temporarily enable globbing to walk virtio_ports_path. - set +f; set -- "${virtio_ports_path}/"*; set -f; + set +f + set -- "${virtio_ports_path}/"* + set -f for port_dir in "$@"; do local name_file="${port_dir}/name" port_name [ ! -f "${name_file}" ] && continue - read port_name < "${name_file}" || \ - warn "unable to read file: $name_file" + read port_name < "${name_file}" || + warn "unable to read file: $name_file" # Check for both current and legacy LXD serial names - if [ "${port_name}" = "com.canonical.lxd" ] || \ - [ "${port_name}" = "org.linuxcontainers.lxd" ]; then + if [ "${port_name}" = "com.canonical.lxd" ] || + [ "${port_name}" = "org.linuxcontainers.lxd" ]; then return ${DS_FOUND} fi done @@ -977,7 +1019,7 @@ dscheck_LXD() { dscheck_NoCloud() { local fslabel="cidata CIDATA" d="" case " ${DI_DMI_PRODUCT_SERIAL} " in - *\ ds=nocloud*) return ${DS_FOUND};; + *\ ds=nocloud*) return ${DS_FOUND} ;; esac for d in nocloud nocloud-net; do @@ -1012,7 +1054,9 @@ check_configdrive_v2() { local d="" local vlc_config_drive_path="${PATH_VAR_LIB_CLOUD}/seed/config_drive" for d in /config-drive $vlc_config_drive_path; do - set +f; set -- "$d/openstack/"2???-??-??/meta_data.json; set -f; + set +f + set -- "$d/openstack/"2???-??-??/meta_data.json + set -f [ -f "$1" ] && return ${DS_FOUND} done # at least one cloud (softlayer) seeds config drive with only 'latest'. @@ -1081,13 +1125,15 @@ vmware_guest_customization() { local ppath="plugins/vmsvc/libdeployPkgPlugin.so" for pkg in vmware-tools open-vm-tools; do if [ -f "$pre/$pkg/$ppath" -o -f "${pre}64/$pkg/$ppath" ]; then - found="$pkg"; break; + found="$pkg" + break fi # search in multiarch dir - if [ -f "$pre/$x86/$pkg/$ppath" ] || \ - [ -f "$pre/$aarch/$pkg/$ppath" ] || \ - [ -f "$pre/$i386/$pkg/$ppath" ]; then - found="$pkg"; break; + if [ -f "$pre/$x86/$pkg/$ppath" ] || + [ -f "$pre/$aarch/$pkg/$ppath" ] || + [ -f "$pre/$i386/$pkg/$ppath" ]; then + found="$pkg" + break fi done [ -n "$found" ] || return 1 @@ -1098,8 +1144,8 @@ vmware_guest_customization() { if check_config "$key" && get_value "$key" "$_RET"; then debug 2 "${_RET_fname} set $key to $_RET" case "$_RET" in - 0|false|False) return 0;; - *) return 1;; + 0 | false | False) return 0 ;; + *) return 1 ;; esac fi @@ -1107,11 +1153,11 @@ vmware_guest_customization() { } vmware_has_rpctool() { - command -v vmware-rpctool >/dev/null 2>&1 + command -v vmware-rpctool > /dev/null 2>&1 } vmware_rpctool_guestinfo() { - vmware-rpctool "info-get guestinfo.${1}" 2>/dev/null | grep "[[:alnum:]]" + vmware-rpctool "info-get guestinfo.${1}" 2> /dev/null | grep "[[:alnum:]]" } vmware_rpctool_guestinfo_err() { @@ -1119,11 +1165,11 @@ vmware_rpctool_guestinfo_err() { } vmware_has_vmtoolsd() { - command -v vmtoolsd >/dev/null 2>&1 + command -v vmtoolsd > /dev/null 2>&1 } vmware_vmtoolsd_guestinfo() { - vmtoolsd --cmd "info-get guestinfo.${1}" 2>/dev/null | grep "[[:alnum:]]" + vmtoolsd --cmd "info-get guestinfo.${1}" 2> /dev/null | grep "[[:alnum:]]" } vmware_vmtoolsd_guestinfo_err() { @@ -1167,9 +1213,11 @@ ovf_vmware_transport_guestinfo() { return 1 fi case "$out" in - "/dev/null 2>&1; then + if { vmware_guestinfo_metadata || + vmware_guestinfo_userdata || + vmware_guestinfo_vendordata; } > /dev/null 2>&1; then return "${DS_FOUND}" fi @@ -1708,7 +1781,9 @@ WSL_path() { WSL_run_cmd() { local val="" exepath="$1" shift - _RET=$(/init "$exepath" /c "$@" 2>/dev/null) + # Using the '/u' flag to enforce Unicode (UTF-16 LE), thus we need to decode it afterwards. + # It's more reliable than the default ANSI Code Pages for anything above the ASCII range. + _RET=$(/init "$exepath" /u /c "$@" 2> /dev/null | iconv --from-code UTF-16LE --to-code UTF-8) } WSL_profile_dir() { @@ -1719,10 +1794,12 @@ WSL_profile_dir() { for m in $@; do cmdexe="$m/Windows/System32/cmd.exe" if command -v "$cmdexe" > /dev/null 2>&1; then - # Here WSL's proprietary `/init` is used to start the Windows cmd.exe + # Here WSL's `/init` is used to start the Windows cmd.exe # to output the Windows user profile directory path, which is # held by the environment variable %USERPROFILE%. - WSL_run_cmd "$cmdexe" "echo %USERPROFILE%" + # See https://wsl.dev/technical-documentation/interop/ for more information on how /init + # is used to launch Windows binaries. + WSL_run_cmd "$cmdexe" "echo.%USERPROFILE%" profiledir="${_RET%%[[:cntrl:]]}" if [ -n "$profiledir" ]; then # wslpath is a program supplied by WSL itself that translates Windows and Linux paths, @@ -1863,7 +1940,7 @@ write_result() { pre=" " fi for line in "$@"; do - echo "${pre}$line"; + echo "${pre}$line" done } > "$runcfg" ret=$? @@ -1905,8 +1982,8 @@ found() { fi # if None is not already in the list, then add it last. case " $list " in - *\ None,\ *|*\ None\ ) :;; - *) list=${list:+${list}, None};; + *\ None,\ * | *\ None\ ) : ;; + *) list=${list:+${list}, None} ;; esac write_result "datasource_list: [ $list ]" "$@" return @@ -1915,8 +1992,14 @@ found() { trim() { # trim all whitespace from the string, assign output to _RET local tmp="" cur="$*" - until tmp="${cur#[[:space:]]}"; [ "$tmp" = "$cur" ]; do cur="$tmp"; done - until tmp="${cur%[[:space:]]}"; [ "$tmp" = "$cur" ]; do cur="$tmp"; done + until + tmp="${cur#[[:space:]]}" + [ "$tmp" = "$cur" ] + do cur="$tmp"; done + until + tmp="${cur%[[:space:]]}" + [ "$tmp" = "$cur" ] + do cur="$tmp"; done _RET="$tmp" } @@ -1925,8 +2008,10 @@ unquote() { local quote='"' tick="'" local val="$1" case "$val" in - ${quote}*${quote}|${tick}*${tick}) - val=${val#?}; val=${val%?};; + ${quote}*${quote} | ${tick}*${tick}) + val=${val#?} + val=${val%?} + ;; esac _RET="$val" } @@ -1960,8 +2045,8 @@ _read_config() { fi case "$key" in - datasource) _rc_dsname="$val";; - policy) _rc_policy="$val";; + datasource) _rc_dsname="$val" ;; + policy) _rc_policy="$val" ;; esac done if [ "$keyname" = "_unset" ]; then @@ -1999,10 +2084,10 @@ parse_policy() { local def="" case "$DI_UNAME_MACHINE" in # these have dmi data - i?86|x86_64) def=${DI_DEFAULT_POLICY};; + i?86 | x86_64) def=${DI_DEFAULT_POLICY} ;; # aarch64 has dmi, but not currently used (LP: #1663304) - aarch64) def=${DI_DEFAULT_POLICY_NO_DMI};; - *) def=${DI_DEFAULT_POLICY_NO_DMI};; + aarch64) def=${DI_DEFAULT_POLICY_NO_DMI} ;; + *) def=${DI_DEFAULT_POLICY_NO_DMI} ;; esac local policy="$1" local _def_mode="" _def_report="" _def_found="" _def_maybe="" @@ -2015,23 +2100,30 @@ parse_policy() { local mode="" report="" found="" maybe="" notfound="" local oifs="$IFS" tok="" val="" # shellcheck disable=2086 - { IFS=","; set -- $policy; IFS="$oifs"; } + { + IFS="," + set -- $policy + IFS="$oifs" + } for tok in "$@"; do val=${tok#*=} case "$tok" in - "${DI_ENABLED}"|"${DI_DISABLED}"|search|report) mode=$tok;; - found=all|found=first) found=$val;; - maybe=all|maybe=none) maybe=$val;; - notfound="${DI_ENABLED}"|notfound="${DI_DISABLED}") notfound=$val;; + "${DI_ENABLED}" | "${DI_DISABLED}" | search | report) mode=$tok ;; + found=all | found=first) found=$val ;; + maybe=all | maybe=none) maybe=$val ;; + notfound="${DI_ENABLED}" | notfound="${DI_DISABLED}") notfound=$val ;; found=*) - parse_warn found "$val" "${_def_found}" - found=${_def_found};; + parse_warn found "$val" "${_def_found}" + found=${_def_found} + ;; maybe=*) - parse_warn maybe "$val" "${_def_maybe}" - maybe=${_def_maybe};; + parse_warn maybe "$val" "${_def_maybe}" + maybe=${_def_maybe} + ;; notfound=*) - parse_warn notfound "$val" "${_def_notfound}" - notfound=${_def_notfound};; + parse_warn notfound "$val" "${_def_notfound}" + notfound=${_def_notfound} + ;; esac done report=${report:-${_def_report:-false}} @@ -2060,17 +2152,17 @@ read_config() { # discard anything after the first delimiter val=${val%%;*} case "$key" in - ds) _rc_dsname="$val";; - ci.ds) _rc_dsname="$val";; - ci.datasource) _rc_dsname="$val";; - ci.di.policy) _rc_policy="$val";; + ds) _rc_dsname="$val" ;; + ci.ds) _rc_dsname="$val" ;; + ci.datasource) _rc_dsname="$val" ;; + ci.di.policy) _rc_policy="$val" ;; esac done local _rc_mode _rc_report _rc_found _rc_maybe _rc_notfound parse_policy "${_rc_policy}" debug 1 "policy loaded: mode=${_rc_mode} report=${_rc_report}" \ - "found=${_rc_found} maybe=${_rc_maybe} notfound=${_rc_notfound}" + "found=${_rc_found} maybe=${_rc_maybe} notfound=${_rc_notfound}" DI_MODE=${_rc_mode} DI_ON_FOUND=${_rc_found} DI_ON_MAYBE=${_rc_maybe} @@ -2080,7 +2172,6 @@ read_config() { return $ret } - manual_clean_and_existing() { [ -f "${PATH_VAR_LIB_CLOUD}/instance/manual-clean" ] } @@ -2111,7 +2202,7 @@ set_run_path() { # testing only - NOT use for production code, it is NOT supported get_environment() { if [ -f "$PATH_DI_ENV" ]; then - debug 0 "WARN: loading environment file [${PATH_DI_ENV}]"; + debug 0 "WARN: loading environment file [${PATH_DI_ENV}]" # shellcheck source=/dev/null . "$PATH_DI_ENV" fi @@ -2142,8 +2233,9 @@ _main() { ;; "${DI_ENABLED}") debug 1 "mode=$DI_ENABLED. returning $ret_en" - return $ret_en;; - search|report) :;; + return $ret_en + ;; + search | report) : ;; esac if [ -n "${DI_DSNAME}" ]; then @@ -2161,7 +2253,7 @@ _main() { # shellcheck disable=2086 set -- $DI_DSLIST # if there is only a single entry in $DI_DSLIST - if [ $# -eq 1 ] || [ $# -eq 2 -a "$2" = "None" ] ; then + if [ $# -eq 1 ] || [ $# -eq 2 -a "$2" = "None" ]; then debug 1 "single entry in datasource_list ($DI_DSLIST) use that." if [ $# -eq 1 ]; then write_result "datasource_list: [ $1 ]" @@ -2176,7 +2268,7 @@ _main() { for ds in ${DI_DSLIST}; do dscheck_fn="dscheck_${ds}" debug 2 "Checking for datasource '$ds' via '$dscheck_fn'" - if ! type "$dscheck_fn" >/dev/null 2>&1; then + if ! type "$dscheck_fn" > /dev/null 2>&1; then warn "No check method '$dscheck_fn' for datasource '$ds'" continue fi @@ -2185,14 +2277,16 @@ _main() { ret="$?" case "$ret" in "${DS_FOUND}") - debug 1 "check for '$ds' returned found"; + debug 1 "check for '$ds' returned found" exfound_cfg="${exfound_cfg:+${exfound_cfg}${CR}}${_RET_excfg}" - found="${found} $ds";; + found="${found} $ds" + ;; "${DS_MAYBE}") - debug 1 "check for '$ds' returned maybe"; + debug 1 "check for '$ds' returned maybe" exmaybe_cfg="${exmaybe_cfg:+${exmaybe_cfg}${CR}}${_RET_excfg}" - maybe="${maybe} $ds";; - *) debug 2 "check for '$ds' returned not-found[$ret]";; + maybe="${maybe} $ds" + ;; + *) debug 2 "check for '$ds' returned not-found[$ret]" ;; esac done @@ -2229,17 +2323,21 @@ _main() { case "$DI_MODE:$DI_ON_NOTFOUND" in report:"${DI_DISABLED}") msg="$basemsg Would disable cloud-init [$ret_dis]" - ret=$ret_en;; + ret=$ret_en + ;; report:"${DI_ENABLED}") msg="$basemsg Would enable cloud-init [$ret_en]" - ret=$ret_en;; + ret=$ret_en + ;; search:"${DI_DISABLED}") msg="$basemsg Disabled cloud-init [$ret_dis]" - ret=$ret_dis;; + ret=$ret_dis + ;; search:"${DI_ENABLED}") msg="$basemsg Enabled cloud-init [$ret_en]" - ret=$ret_en;; - *) error "Unexpected result";; + ret=$ret_en + ;; + *) error "Unexpected result" ;; esac debug 1 "$msg" return "$ret" @@ -2258,7 +2356,7 @@ main() { if read ret < "$PATH_RUN_DI_RESULT"; then if [ "$ret" = "0" ] || [ "$ret" = "1" ] || [ "$ret" = "2" ]; then debug 2 "used cached result $ret. pass --force to re-run." - return "$ret"; + return "$ret" fi debug 1 "previous run returned unexpected '$ret'. Re-running." else @@ -2280,11 +2378,12 @@ noop() { get_environment case "${DI_MAIN}" in # builtin DI_MAIN implementations - main|print_info|noop) "${DI_MAIN}" "$@";; + main | print_info | noop) "${DI_MAIN}" "$@" ;; # side-load an alternate implementation # testing only - NOT use for production code, it is NOT supported *) - debug 0 "WARN: side-loading alternate implementation: [${DI_MAIN}]"; - exec "${DI_MAIN}" "$@";; + debug 0 "WARN: side-loading alternate implementation: [${DI_MAIN}]" + exec "${DI_MAIN}" "$@" + ;; esac diff --git a/tools/hook-hotplug b/tools/hook-hotplug index f142d4b9548..485cff19ec5 100755 --- a/tools/hook-hotplug +++ b/tools/hook-hotplug @@ -25,7 +25,7 @@ if ! should_run; then fi # open cloud-init's hotplug-hook fifo rw -exec 3<>$fifo +exec 3<> $fifo env_params=" --subsystem=${SUBSYSTEM} handle --devpath=${DEVPATH} --udevaction=${ACTION}" # write params to cloud-init's hotplug-hook fifo echo "${env_params}" >&3 diff --git a/tools/make-tarball b/tools/make-tarball index 48442d0b080..11dddc26f52 100755 --- a/tools/make-tarball +++ b/tools/make-tarball @@ -8,7 +8,7 @@ cleanup() { trap cleanup EXIT Usage() { - cat <&2; exit 1; } + eval set -- "${getopt_out}" || { + Usage 1>&2 + exit 1 +} long_opt="" orig_opt="" version="" while [ $# -ne 0 ]; do - cur=$1; next=$2 + cur=$1 + next=$2 case "$cur" in - -h|--help) Usage; exit 0;; - -o|--output) output=$next; shift;; - --version) version=$next; shift;; - --long) long_opt="--long";; - --orig-tarball) orig_opt=".orig";; - --) shift; break;; + -h | --help) + Usage + exit 0 + ;; + -o | --output) + output=$next + shift + ;; + --version) + version=$next + shift + ;; + --long) long_opt="--long" ;; + --orig-tarball) orig_opt=".orig" ;; + --) + shift + break + ;; esac - shift; + shift done rev=${1:-HEAD} diff --git a/tools/motd-hook b/tools/motd-hook deleted file mode 100755 index 73d9792c1ab..00000000000 --- a/tools/motd-hook +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# Copyright (C) 2010 Canonical Ltd. -# -# Authors: Scott Moser -# -# This file is part of cloud-init. See LICENSE file for license information. - -# 92-ec2-upgrade-available - update-motd script - -# Determining if updates are available is possibly slow. -# a cronjob runs occasioinally and updates a file with information -# on the latest available release (if newer than current) - -BUILD_FILE=/var/lib/cloud/data/available.build - -[ -s "${BUILD_FILE}" ] || exit 0 - -read suite build_name name serial other < "${BUILD_FILE}" - -cat <&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } -errorrc() { local r=$?; error "$@" "ret=$r"; return $r; } +fail() { + [ $# -eq 0 ] || error "$@" + exit 1 +} +errorrc() { + local r=$? + error "$@" "ret=$r" + return $r +} Usage() { - cat <&2; [ $# -eq 0 ] || error "$@"; return 1; } +bad_Usage() { + Usage 1>&2 + [ $# -eq 0 ] || error "$@" + return 1 +} cleanup() { if [ -n "$CONTAINER" ]; then if [ "$KEEP" = "true" ]; then @@ -60,12 +71,12 @@ cleanup() { } debug() { - local level=${1}; shift; + local level=${1} + shift [ "${level}" -gt "${VERBOSITY}" ] && return error "${@}" } - inside_as() { # inside_as(container_name, user, cmd[, args]) # executes cmd with args inside container as user in users home dir. @@ -80,7 +91,7 @@ inside_as() { stuffed=${stuffed# -- } b64=$(printf "%s\n" "$stuffed" | base64 --wrap=0) inside "$name" su "$user" -c \ - 'cd; eval set -- "$(echo '"$b64"' | base64 --decode)" && exec "$@"'; + 'cd; eval set -- "$(echo '"$b64"' | base64 --decode)" && exec "$@"' } inside_as_cd() { @@ -95,21 +106,22 @@ inside() { $LXC exec "$name" -- "$@" } -inject_cloud_init(){ +inject_cloud_init() { # take current cloud-init git dir and put it inside $name at # ~$user/cloud-init. local name="$1" user="$2" dirty="$3" local dname="cloud-init" gitdir="" commitish="" gitdir=$(git rev-parse --git-dir) || { - errorrc "Failed to get git dir in $PWD"; + errorrc "Failed to get git dir in $PWD" return } local t=${gitdir%/*} case "$t" in - */worktrees) + */worktrees) if [ -f "${t%worktrees}/config" ]; then gitdir="${t%worktrees}" fi + ;; esac # attempt to get branch name. @@ -182,7 +194,10 @@ get_os_info_in() { # prep the container (install very basic dependencies) [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] && return 0 data=$(run_self_inside "$name" os_info) || - { errorrc "Failed to get os-info in container $name"; return; } + { + errorrc "Failed to get os-info in container $name" + return + } eval "$data" && [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] || return debug 1 "determined $name is $OS_NAME/$OS_VERSION" } @@ -204,33 +219,42 @@ get_os_info() { local pname="" pname=$(sh -c '. /etc/os-release; echo $PRETTY_NAME') case "$pname" in - *buster*) OS_VERSION=10;; - *sid*) OS_VERSION="sid";; + *buster*) OS_VERSION=10 ;; + *sid*) OS_VERSION="sid" ;; esac fi elif [ -f /etc/centos-release ]; then local line="" read line < /etc/centos-release case "$line" in - CentOS\ *\ 6.*) OS_VERSION="6"; OS_NAME="centos";; + CentOS\ *\ 6.*) + OS_VERSION="6" + OS_NAME="centos" + ;; esac fi [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] || - { error "Unable to determine OS_NAME/OS_VERSION"; return 1; } + { + error "Unable to determine OS_NAME/OS_VERSION" + return 1 + } } yum_install() { local n=0 max=10 ret bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1" - while n=$((n+1)); do - error ":: running $bcmd $* [$n/$max]" - $bcmd "$@" - ret=$? - [ $ret -eq 0 ] && break - [ $n -ge $max ] && { error "gave up on $bcmd"; exit $ret; } - nap=$((n*5)) - error ":: failed [$ret] ($n/$max). sleeping $nap." - sleep $nap + while n=$((n + 1)); do + error ":: running $bcmd $* [$n/$max]" + $bcmd "$@" + ret=$? + [ $ret -eq 0 ] && break + [ $n -ge $max ] && { + error "gave up on $bcmd" + exit $ret + } + nap=$((n * 5)) + error ":: failed [$ret] ($n/$max). sleeping $nap." + sleep $nap done error ":: running yum install --cacheonly --assumeyes $*" yum install --cacheonly --assumeyes "$@" @@ -251,11 +275,13 @@ apt_install() { install_packages() { get_os_info || return case "$OS_NAME" in - centos|rocky*|fedora) yum_install "$@";; - opensuse*) zypper_install "$@";; - debian|ubuntu) apt_install "$@" -y;; - *) error "Do not know how to install packages on ${OS_NAME}"; - return 1;; + centos | rocky* | fedora) yum_install "$@" ;; + opensuse*) zypper_install "$@" ;; + debian | ubuntu) apt_install "$@" -y ;; + *) + error "Do not know how to install packages on ${OS_NAME}" + return 1 + ;; esac } @@ -270,7 +296,8 @@ prep() { local py3pkg="python3" case "$OS_NAME" in opensuse) - py3pkg="python3-base";; + py3pkg="python3-base" + ;; esac pairs="$pairs python3:$py3pkg" @@ -278,7 +305,7 @@ prep() { for pair in $pairs; do pkg=${pair#*:} cmd=${pair%%:*} - command -v "$cmd" >/dev/null 2>&1 || needed="${needed} $pkg" + command -v "$cmd" > /dev/null 2>&1 || needed="${needed} $pkg" done needed=${needed# } if [ -z "$needed" ]; then @@ -302,15 +329,16 @@ is_done_cloudinit() { is_done_systemd() { local s="" num="$1" - s=$(systemctl is-system-running 2>&1); + s=$(systemctl is-system-running 2>&1) _RET="$? $s" case "$s" in - initializing|starting) return 1;; + initializing | starting) return 1 ;; *[Ff]ailed*connect*bus*) # warn if not the first run. [ "$num" -lt 5 ] || - error "Failed to connect to systemd bus [${_RET%% *}]"; - return 1;; + error "Failed to connect to systemd bus [${_RET%% *}]" + return 1 + ;; esac return 0 } @@ -323,20 +351,20 @@ is_done_other() { wait_inside() { local name="$1" max="${2:-${WAIT_MAX}}" debug=${3:-0} - local i=0 check="is_done_other"; + local i=0 check="is_done_other" if [ -e /run/systemd ]; then check=is_done_systemd elif [ -x /usr/bin/cloud-init ]; then check=is_done_cloudinit fi [ "$debug" != "0" ] && debug 1 "check=$check" - while ! $check $i && i=$((i+1)); do + while ! $check $i && i=$((i + 1)); do [ "$i" -ge "$max" ] && exit 1 [ "$debug" = "0" ] || echo -n . sleep 1 done if [ "$debug" != "0" ]; then - read up _ /dev/null && system_up=true && break + inside "$name" true 2> /dev/null && system_up=true && break done - [ $system_up == true ] || { errorrc "exec command inside $name failed."; return; } + [ $system_up == true ] || { + errorrc "exec command inside $name failed." + return + } get_os_info_in "$name" [ "$OS_NAME" = "debian" ] && wtime=300 && debug 1 "on debian we wait for ${wtime}s" debug 1 "waiting for boot of $name" run_self_inside "$name" wait_inside "$name" "$wtime" "$VERBOSITY" || - { errorrc "wait inside $name failed."; return; } + { + errorrc "wait inside $name failed." + return + } if [ -n "${http_proxy-}" ]; then if [ "$OS_NAME" = "centos" -o "$OS_NAME" = "fedora" ]; then @@ -364,21 +398,21 @@ wait_for_boot() { inside "$name" sh -c "sed -i --regexp-extended '/^#baseurl=/s/#// ; /^(mirrorlist|metalink)=/s/^/#/' /etc/yum.repos.d/*.repo" inside "$name" sh -c "sed -i 's/download\.fedoraproject\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo" inside "$name" sh -c "sed -i 's/download\.example/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo" - if [ "$OS_NAME" = "centos" ]; then - CENTOS_REPO="/etc/yum.repos.d/centos.repo" - inside "$name" sh -c "grep -q baseurl $CENTOS_REPO" - if [ $? -eq 1 ]; then - # CentOS 9 does not provide baseurl definitions - inside "$name" sh -c "sed -i '/\[baseos\]/a baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os' ${CENTOS_REPO}" - inside "$name" sh -c "sed -i '/\[appstream\]/a baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os' ${CENTOS_REPO}" - inside "$name" sh -c "sed -i '/\[crb\]/a baseurl=https://mirror.stream.centos.org/9-stream/CRB/x86_64/os' ${CENTOS_REPO}" + if [ "$OS_NAME" = "centos" ]; then + CENTOS_REPO="/etc/yum.repos.d/centos.repo" + inside "$name" sh -c "grep -q baseurl $CENTOS_REPO" + if [ $? -eq 1 ]; then + # CentOS 9 does not provide baseurl definitions + inside "$name" sh -c "sed -i '/\[baseos\]/a baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os' ${CENTOS_REPO}" + inside "$name" sh -c "sed -i '/\[appstream\]/a baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os' ${CENTOS_REPO}" + inside "$name" sh -c "sed -i '/\[crb\]/a baseurl=https://mirror.stream.centos.org/9-stream/CRB/x86_64/os' ${CENTOS_REPO}" CENTOS_EXTRAS_REPO="/etc/yum.repos.d/centos-addons.repo" - inside "$name" sh -c "sed -i '/\[extras-common\]/a baseurl=https://mirror.stream.centos.org/SIGs/9-stream/extras/x86_64/extras-common' ${CENTOS_EXTRAS_REPO}" - inside "$name" sh -c "dnf install -y 'dnf-command(config-manager)'" - inside "$name" sh -c "dnf config-manager --set-enabled crb" - inside "$name" sh -c "dnf config-manager --set-disabled epel-cisco-openh264" || true + inside "$name" sh -c "sed -i '/\[extras-common\]/a baseurl=https://mirror.stream.centos.org/SIGs/9-stream/extras/x86_64/extras-common' ${CENTOS_EXTRAS_REPO}" + inside "$name" sh -c "dnf install -y 'dnf-command(config-manager)'" + inside "$name" sh -c "dnf config-manager --set-enabled crb" + inside "$name" sh -c "dnf config-manager --set-disabled epel-cisco-openh264" || true fi - fi + fi else debug 1 "do not know how to configure proxy on $OS_NAME" fi @@ -391,7 +425,7 @@ start_instance() { launch_flags=() [ "$use_vm" == true ] && launch_flags+=(--vm) $LXC launch "$src" "$name" "${launch_flags[@]}" || { - errorrc "Failed to start container '$name' from '$src'"; + errorrc "Failed to start container '$name' from '$src'" return } CONTAINER=$name @@ -407,13 +441,13 @@ run_self_inside() { # run_self_inside(container, args) local name="$1" shift - inside "$name" bash -s "$@" <"$0" + inside "$name" bash -s "$@" < "$0" } run_self_inside_as_cd() { local name="$1" user="$2" dir="$3" shift 3 - inside_as_cd "$name" "$user" "$dir" bash -s "$@" <"$0" + inside_as_cd "$name" "$user" "$dir" bash -s "$@" < "$0" } main() { @@ -423,7 +457,10 @@ main() { getopt_out=$(getopt --name "${0##*/}" \ --options "${short_opts}" --long "${long_opts}" -- "$@") && eval set -- "${getopt_out}" || - { bad_Usage; return; } + { + bad_Usage + return + } local cur="" next="" local package=false srcpackage=false unittest="" name="" @@ -431,31 +468,50 @@ main() { local use_vm=false while [ $# -ne 0 ]; do - cur="${1:-}"; next="${2:-}"; + cur="${1:-}" + next="${2:-}" case "$cur" in - -a|--artifacts) artifact_d="$next";; - --dirty) dirty=true;; - -h|--help) Usage ; exit 0;; - -k|--keep) KEEP=true;; - -n|--name) name="$next"; shift;; - -p|--package) package=true;; - -s|--source-package) srcpackage=true;; - -u|--unittest) unittest=1;; - -v|--verbose) VERBOSITY=$((VERBOSITY+1));; - --vm) use_vm=true;; - --wait-max) WAIT_MAX="$next"; shift;; - --commitish) COMMITISH="$next"; shift;; - --) shift; break;; + -a | --artifacts) artifact_d="$next" ;; + --dirty) dirty=true ;; + -h | --help) + Usage + exit 0 + ;; + -k | --keep) KEEP=true ;; + -n | --name) + name="$next" + shift + ;; + -p | --package) package=true ;; + -s | --source-package) srcpackage=true ;; + -u | --unittest) unittest=1 ;; + -v | --verbose) VERBOSITY=$((VERBOSITY + 1)) ;; + --vm) use_vm=true ;; + --wait-max) + WAIT_MAX="$next" + shift + ;; + --commitish) + COMMITISH="$next" + shift + ;; + --) + shift + break + ;; esac - shift; + shift done COMMITISH=${COMMITISH:-HEAD} - [ $# -eq 1 ] || { bad_Usage "Expected 1 arg, got $# ($*)"; return; } + [ $# -eq 1 ] || { + bad_Usage "Expected 1 arg, got $# ($*)" + return + } local img_ref_in="$1" case "${img_ref_in}" in - *:*) img_ref="${img_ref_in}";; - *) img_ref="images:${img_ref_in}";; + *:*) img_ref="${img_ref_in}" ;; + *) img_ref="images:${img_ref_in}" ;; esac # program starts here @@ -476,18 +532,30 @@ main() { trap cleanup EXIT start_instance "$img_ref" "$name" "$use_vm" || - { errorrc "Failed to start container for $img_ref"; return; } + { + errorrc "Failed to start container for $img_ref" + return + } get_os_info_in "$name" || - { errorrc "failed to get os_info in $name"; return; } + { + errorrc "failed to get os_info in $name" + return + } # prep the container (install very basic dependencies) run_self_inside "$name" prep || - { errorrc "Failed to prep container $name"; return; } + { + errorrc "Failed to prep container $name" + return + } # add the user inside "$name" useradd "$user" --create-home "--home-dir=$home" || - { errorrc "Failed to add user '$user' in '$name'"; return 1; } + { + errorrc "Failed to add user '$user' in '$name'" + return 1 + } debug 1 "inserting cloud-init" inject_cloud_init "$name" "$user" "$dirty" || { @@ -501,36 +569,38 @@ main() { return } - local errors=( ) + local errors=() inside_as_cd "$name" "$user" "$cdir" git status || { errorrc "git checkout failed." - errors[${#errors[@]}]="git checkout"; + errors[${#errors[@]}]="git checkout" } if [ -n "$unittest" ]; then debug 1 "running unit tests." run_self_inside_as_cd "$name" "$user" "$cdir" pytest \ tests/unittests cloudinit/ || { - errorrc "pytest failed."; - errors[${#errors[@]}]="pytest" - } + errorrc "pytest failed." + errors[${#errors[@]}]="pytest" + } fi local build_pkg="" build_srcpkg="" pkg_ext="" distflag="" case "$OS_NAME" in - centos|rocky*|fedora) distflag="--distro=redhat";; - opensuse*) distflag="--distro=suse";; + centos | rocky* | fedora) distflag="--distro=redhat" ;; + opensuse*) distflag="--distro=suse" ;; esac case "$OS_NAME" in - debian|ubuntu) - build_pkg="./packages/bddeb -d" + debian | ubuntu) + build_pkg="./packages/bddeb -d" build_srcpkg="./packages/bddeb -S -d" - pkg_ext=".deb";; - centos|opensuse*|rocky*|fedora) + pkg_ext=".deb" + ;; + centos | opensuse* | rocky* | fedora) build_pkg="./packages/brpm $distflag" build_srcpkg="./packages/brpm $distflag --srpm" - pkg_ext=".rpm";; + pkg_ext=".rpm" + ;; esac if [ "$srcpackage" = "true" ]; then [ -n "$build_srcpkg" ] || { @@ -540,7 +610,7 @@ main() { debug 1 "building source package with $build_srcpkg." # shellcheck disable=SC2086 inside_as_cd "$name" "$user" "$cdir" python3 $build_srcpkg || { - errorrc "failed: $build_srcpkg"; + errorrc "failed: $build_srcpkg" errors[${#errors[@]}]="source package" } fi @@ -553,7 +623,7 @@ main() { debug 1 "building binary package with $build_pkg." # shellcheck disable=SC2086 inside_as_cd "$name" "$user" "$cdir" python3 $build_pkg || { - errorrc "failed: $build_pkg"; + errorrc "failed: $build_pkg" errors[${#errors[@]}]="binary package" } fi @@ -588,6 +658,10 @@ main() { } case "${1:-}" in - prep|os_info|wait_inside|pytest) _n=$1; shift; "$_n" "$@";; - *) main "$@";; + prep | os_info | wait_inside | pytest) + _n=$1 + shift + "$_n" "$@" + ;; + *) main "$@" ;; esac diff --git a/tools/run-lint b/tools/run-lint deleted file mode 100755 index 2bd0ab17ac9..00000000000 --- a/tools/run-lint +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# This file runs flake8 for compatibility's sake. As soon as we move off python 3.6, this should be changed to use ruff. - -CR=" -" -pycheck_dirs=( "cloudinit/" "tests/" "tools/" "setup.py" ) - -set -f -if [ $# -eq 0 ]; then - files=( "${pycheck_dirs[@]}" ) -else - files=( "$@" ) -fi - -if [ -z "$PYTHON" ]; then - PYTHON="python3" -fi -cmd=( "$PYTHON" -m "flake8" "${files[@]}" ) - -echo "Running: " "${cmd[@]}" 1>&2 -exec "${cmd[@]}" diff --git a/tools/tox-venv b/tools/tox-venv deleted file mode 100755 index c22f6faca4d..00000000000 --- a/tools/tox-venv +++ /dev/null @@ -1,183 +0,0 @@ -#!/bin/sh -# https://gist.github.com/smoser/2d4100a6a5d230ca937f - -CR=' -' -error() { echo "$@" 1>&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } -get_env_dirs() { - # read 'tox --showconfig'. return list of - # envname:dir - local key="" equal="" val="" curenv="" out="" - while read key equal val; do - case "$key" in - "[testenv:"*) - curenv=${key#*:}; - curenv=${curenv%%"]"*}; - continue;; - esac - if [ "${key#*=}" != "$key" ]; then - # older tox shows key=value or key= value - # newer tox shows: key = value - key=${key%%=*} - val=${equal} - fi - [ "$key" = "envdir" ] || [ "$key" = "env_dir" ] || continue - out="${out:+${out}${CR}}${curenv}:$val" - done - echo "$out" -} - -load_config() { - local tox_ini="$1" out="" envs="" - if [ "$tox_ini" = "${CACHED_ENVS_INI}" ]; then - _RET="$CACHED_ENVS" - return - fi - out=$(tox -c "$tox_ini" --showconfig) || return 1 - envs=$(echo "$out" | get_env_dirs) || return 1 - CACHED_ENVS="$envs" - CACHED_ENVS_INI="$tox_ini" - _RET="$envs" -} - -list_environments() { - local tox_ini="$1" prefix=" " out="" envs="" oifs="$IFS" - load_config "$tox_ini" || return 1 - envs="${_RET}" - IFS="$CR" - for d in ${envs}; do - env=${d%%:*} - dir=${d#*:} - [ -f "$dir/bin/activate" ] && s="*" || s="" - echo "${prefix}$env$s"; - done - IFS="$oifs" -} - -get_command() { - local tox_ini="$1" env="$2" out="" - shift 2 - out=$( - sed -e ':x; /\\$/ { N; s/\\\n[ ]*//; tx };' "${tox_ini}" | - gawk ' - $1 ~ /^\[testenv.*\]/ { - name=$1; - sub("\\[", "", name); sub(".*:", "", name); - sub("].*", "", name); - curenv=name; }; - $1 == "basepython" && (name == "testenv" || name == n) { python=$3 } - $1 == "commands" && (name == "testenv" || name == n) { - sub("commands = ", ""); cmd = $0; }; - END { - sub("{envpython}", python, cmd); - sub("{toxinidir}", toxinidir, cmd); - if (inargs == "") replacement = "\\1" - else replacement = inargs - cmd = gensub(/{posargs:?([^}]*)}/, replacement, "global", cmd) - print(cmd); - }' n="$env" toxinidir="$(dirname $tox_ini)" inargs="$*") - if [ -z "$out" ]; then - error "Failed to find command for $env in $tox_ini" - return 1 - fi - echo "$out" -} - -get_env_dir() { - local tox_ini="$1" env="$2" oifs="$IFS" t="" d="" envs="" - if [ "${TOX_VENV_SHORTCUT:-1}" != "0" ]; then - local stox_d="${tox_ini%/*}/.tox/${env}" - if [ -e "${stox_d}/bin/activate" ]; then - _RET="${stox_d}" - return - fi - fi - load_config "$tox_ini" && envs="$_RET" || return 1 - IFS="$CR" - for t in $envs; do - [ "$env" = "${t%%:*}" ] && d="${t#*:}" && break - done - IFS=${oifs} - [ -n "$d" ] || return 1 - _RET="$d" -} - -Usage() { - local tox_ini="$1" - cat <&2; exit 1; } -[ "$1" = "-h" -o "$1" = "--help" ] && { Usage "$tox_ini"; exit 0; } - -[ -f "$tox_ini" ] || fail "$tox_ini: did not find tox.ini" - -if [ "$1" = "-l" -o "$1" = "--list" ]; then - list_environments "$tox_ini" - exit -fi - -nocreate="false" -if [ "$1" = "--no-create" ]; then - nocreate="true" - shift -fi - -env="$1" -shift -[ "$1" = "--" ] && shift -get_env_dir "$tox_ini" "$env" && activate="$_RET/bin/activate" || activate="" - -if [ -z "$activate" -o ! -f "$activate" ]; then - if $nocreate; then - fail "tox env '$env' did not exist, and no-create specified" - elif [ -n "$activate" ]; then - error "attempting to create $env:" - error " tox -c $tox_ini --recreate --notest -e $env" - tox -c "$tox_ini" --recreate --notest -e "$env" || - fail "failed creation of env $env" - else - error "$env: not a valid tox environment?" - error "found tox_ini=$tox_ini" - error "try one of:" - list_environments "$tox_ini" 1>&2 - fail - fi -fi -. "$activate" - -[ $# -eq 0 ] && set -- cmd -if [ "$1" = "cmd" -o "$1" = "-" ]; then - shift - out=$(get_command "$tox_ini" "$env" "$@") || exit - eval set -- "$out" -fi -echo "inside tox:$env running: $*" 1>&2 -debian_chroot="tox:$env" exec "$@" diff --git a/tools/uncloud-init b/tools/uncloud-init index 7900ab73229..68dfa3a6b14 100755 --- a/tools/uncloud-init +++ b/tools/uncloud-init @@ -14,122 +14,156 @@ MARK=/var/lib/cloud/sem/uncloud-init.once ROOT_RW="" doexec() { - if [ -n "$ROOT_RW" ]; then - mkdir -p "${MARK%/*}"; - date > "${MARK}"; - fi - cleanup; - log "invoking /sbin/init $*" - exec /sbin/init "$@"; + if [ -n "$ROOT_RW" ]; then + mkdir -p "${MARK%/*}" + date > "${MARK}" + fi + cleanup + log "invoking /sbin/init $*" + exec /sbin/init "$@" } log() { echo "::${0##*/}:" "$@"; } cleanup() { - [ -z "${UMOUNT}" ] || { umount "${UMOUNT}" && unset UMOUNT; } - [ -z "${RMDIR}" ] || { rm -Rf "${RMDIR}" && unset RMDIR; } - [ -z "${ROOT_RW}" ] || { mount -o remount,ro / ; unset ROOT_RW; } + [ -z "${UMOUNT}" ] || { umount "${UMOUNT}" && unset UMOUNT; } + [ -z "${RMDIR}" ] || { rm -Rf "${RMDIR}" && unset RMDIR; } + [ -z "${ROOT_RW}" ] || { + mount -o remount,ro / + unset ROOT_RW + } } updateFrom() { - local dev=$1 fmt=$2 - local mp=""; - - [ "${fmt}" = "tar" -o "${fmt}" = "mnt" ] || - { log FAIL "unknown format ${fmt}"; return 1; } - - log INFO "updating from ${dev} format ${fmt}" - [ ! -e "${dev}" -a -e "/dev/${dev}" ] && dev="/dev/${dev}" - [ -e "${dev}" ] || { echo "no file $dev"; return 2; } - - mp=$(mktemp -d "${TEMPDIR:-/tmp}/update.XXXXXX") && - RMDIR="${mp}" || - { log FAIL "failed to mktemp"; return 1; } - - if [ "$fmt" = "tar" ]; then - dd "if=${dev}" | ( tar -C "${mp}" -xf - ) || - { log FAIL "failed to extract ${dev}"; return 1; } - elif [ "$fmt" = "mnt" ]; then - mount -o ro "${dev}" "${mp}" && UMOUNT=${mp} || - { log FAIL "failed mount ${mp}"; return 1; } - else - log FAIL "unknown format ${fmt}"; return 1; - fi - - if [ -d "${mp}/updates" ]; then - rsync -av "${mp}/updates/" "/" || - { log FAIL "failed rsync updates/ /"; return 1; } - fi - if [ -f "${mp}/updates.tar" ]; then - tar -C / -xvf "${mp}/updates.tar" || - { log FAIL "failed tar -C / -xvf ${mp}/updates.tar"; return 1; } - fi - script="${mp}/updates.script" - if [ -f "${script}" -a -x "${script}" ]; then - MP_DIR=${mp} "${mp}/updates.script" || - { log FAIL "failed to run updates.script"; return 1; } - fi + local dev=$1 fmt=$2 + local mp="" + + [ "${fmt}" = "tar" -o "${fmt}" = "mnt" ] || + { + log FAIL "unknown format ${fmt}" + return 1 + } + + log INFO "updating from ${dev} format ${fmt}" + [ ! -e "${dev}" -a -e "/dev/${dev}" ] && dev="/dev/${dev}" + [ -e "${dev}" ] || { + echo "no file $dev" + return 2 + } + + mp=$(mktemp -d "${TEMPDIR:-/tmp}/update.XXXXXX") && + RMDIR="${mp}" || + { + log FAIL "failed to mktemp" + return 1 + } + + if [ "$fmt" = "tar" ]; then + dd "if=${dev}" | (tar -C "${mp}" -xf -) || + { + log FAIL "failed to extract ${dev}" + return 1 + } + elif [ "$fmt" = "mnt" ]; then + mount -o ro "${dev}" "${mp}" && UMOUNT=${mp} || + { + log FAIL "failed mount ${mp}" + return 1 + } + else + log FAIL "unknown format ${fmt}" + return 1 + fi + + if [ -d "${mp}/updates" ]; then + rsync -av "${mp}/updates/" "/" || + { + log FAIL "failed rsync updates/ /" + return 1 + } + fi + if [ -f "${mp}/updates.tar" ]; then + tar -C / -xvf "${mp}/updates.tar" || + { + log FAIL "failed tar -C / -xvf ${mp}/updates.tar" + return 1 + } + fi + script="${mp}/updates.script" + if [ -f "${script}" -a -x "${script}" ]; then + MP_DIR=${mp} "${mp}/updates.script" || + { + log FAIL "failed to run updates.script" + return 1 + } + fi } -fail() { { [ $# -eq 0 ] && log "FAILING" ; } || log "$@"; exit 1; } +fail() { + { [ $# -eq 0 ] && log "FAILING"; } || log "$@" + exit 1 +} -[ -s "$MARK" ] && { log "already updated" ; doexec "$@"; } +[ -s "$MARK" ] && { + log "already updated" + doexec "$@" +} mount -o remount,rw / || fail "failed to mount rw" ROOT_RW=1 if [ ! -e /proc/cmdline ]; then - mount -t proc /proc /proc - read cmdline < /proc/cmdline - umount /proc + mount -t proc /proc /proc + read cmdline < /proc/cmdline + umount /proc else - read cmdline < /proc/cmdline + read cmdline < /proc/cmdline fi ubuntu_pass="" for x in ${cmdline}; do - case "$x" in - ${KEY}=*) - val=${x#${KEY}=} - dev=${val%:*} - [ "${dev}" = "${val}" ] && fmt="" || fmt=${val#${dev}:} - log "update from ${dev},${fmt}" - updateFrom "${dev}" "${fmt}" || fail "update failed" - log "end update ${dev},${fmt}" - ;; - ubuntu-pass=*|ubuntu_pass=*) ubuntu_pass=${x#*=};; - helpmount) helpmount=1;; - root=*) rootspec=${x#root=};; - esac + case "$x" in + ${KEY}=*) + val=${x#${KEY}=} + dev=${val%:*} + [ "${dev}" = "${val}" ] && fmt="" || fmt=${val#${dev}:} + log "update from ${dev},${fmt}" + updateFrom "${dev}" "${fmt}" || fail "update failed" + log "end update ${dev},${fmt}" + ;; + ubuntu-pass=* | ubuntu_pass=*) ubuntu_pass=${x#*=} ;; + helpmount) helpmount=1 ;; + root=*) rootspec=${x#root=} ;; + esac done if [ "${ubuntu_pass}" = "R" -o "${ubuntu_pass}" = "random" ]; then - ubuntu_pass=$(python -c 'import string, random; + ubuntu_pass=$(python -c 'import string, random; random.seed(); print "".join(random.sample(string.letters+string.digits, 8))') - log "setting ubuntu pass = ${ubuntu_pass}" - printf "\n===\nubuntu_pass = %s\n===\n" "${ubuntu_pass}" >/dev/ttyS0 + log "setting ubuntu pass = ${ubuntu_pass}" + printf "\n===\nubuntu_pass = %s\n===\n" "${ubuntu_pass}" > /dev/ttyS0 fi [ -z "${ubuntu_pass}" ] || - printf "ubuntu:%s\n" "${ubuntu_pass}" > /root/ubuntu-user-pass + printf "ubuntu:%s\n" "${ubuntu_pass}" > /root/ubuntu-user-pass if [ -e /root/ubuntu-user-pass ]; then - log "changing ubuntu user's password!" - chpasswd < /root/ubuntu-user-pass || - log "FAIL: failed changing pass" + log "changing ubuntu user's password!" + chpasswd < /root/ubuntu-user-pass || + log "FAIL: failed changing pass" fi cp /etc/init/tty2.conf /etc/init/ttyS0.conf && - sed -i s,tty2,ttyS0,g /etc/init/ttyS0.conf 2>/dev/null && - log "enabled console on ttyS0" + sed -i s,tty2,ttyS0,g /etc/init/ttyS0.conf 2> /dev/null && + log "enabled console on ttyS0" pa=PasswordAuthentication -sed -i "s,${pa} no,${pa} yes," /etc/ssh/sshd_config 2>/dev/null && - log "enabled passwd auth in ssh" || - log "failed to enable passwd ssh" +sed -i "s,${pa} no,${pa} yes," /etc/ssh/sshd_config 2> /dev/null && + log "enabled passwd auth in ssh" || + log "failed to enable passwd ssh" grep -q vga16fb /etc/modprobe.d/blacklist.conf || { - echo "blacklist vga16fb" >> /etc/modprobe.d/blacklist.conf && - log "blacklisted vga16fb" + echo "blacklist vga16fb" >> /etc/modprobe.d/blacklist.conf && + log "blacklisted vga16fb" } #lstr="${rootspec}" diff --git a/tools/write-ssh-key-fingerprints b/tools/write-ssh-key-fingerprints index 9409257dba0..55711d1da24 100755 --- a/tools/write-ssh-key-fingerprints +++ b/tools/write-ssh-key-fingerprints @@ -1,7 +1,6 @@ #!/bin/sh # This file is part of cloud-init. See LICENSE file for license information. - do_syslog() { log_message=$1 @@ -17,7 +16,6 @@ do_syslog() { logger $logger_opts "$log_message" } - # Redirect stderr to stdout exec 2>&1 diff --git a/tools/xkvm b/tools/xkvm deleted file mode 100755 index b030dc43f05..00000000000 --- a/tools/xkvm +++ /dev/null @@ -1,707 +0,0 @@ -#!/bin/bash -# This file is part of cloud-init. -# See LICENSE file for copyright and license info. - -set -f - -VERBOSITY=0 -KVM_PID="" -DRY_RUN=false -TEMP_D="" -DEF_BRIDGE="virbr0" -TAPDEVS=( ) -# OVS_CLEANUP gets populated with bridge:devname pairs used with ovs -OVS_CLEANUP=( ) -MAC_PREFIX="52:54:00:12:34" -# allow this to be set externally. -_QEMU_SUPPORTS_FILE_LOCKING="${_QEMU_SUPPORTS_FILE_LOCKING}" -KVM="kvm" -declare -A KVM_DEVOPTS - -error() { echo "$@" 1>&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } - -bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; } -randmac() { - # return random mac addr within final 3 tokens - local random="" - random=$(printf "%02x:%02x:%02x" \ - "$((${RANDOM}%256))" "$((${RANDOM}%256))" "$((${RANDOM}%256))") - padmac "$random" -} - -cleanup() { - [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" - [ -z "${KVM_PID}" ] || kill "$KVM_PID" - if [ ${#TAPDEVS[@]} -ne 0 ]; then - local name item - for item in "${TAPDEVS[@]}"; do - [ "${item}" = "skip" ] && continue - debug 1 "removing" "$item" - name="${item%:*}" - if $DRY_RUN; then - error ip tuntap del mode tap "$name" - else - ip tuntap del mode tap "$name" - fi - [ $? -eq 0 ] || error "failed removal of $name" - done - if [ ${#OVS_CLEANUP[@]} -ne 0 ]; then - # with linux bridges, there seems to be no harm in just deleting - # the device (not detaching from the bridge). However, with - # ovs, you have to remove them from the bridge, or later it - # will refuse to add the same name. - error "cleaning up ovs ports: ${OVS_CLEANUP[@]}" - if ${DRY_RUN}; then - error sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}" - else - sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}" - fi - fi - fi -} - -debug() { - local level=${1}; shift; - [ "${level}" -gt "${VERBOSITY}" ] && return - error "${@}" -} - -Usage() { - cat <&1) && - out=$(echo "$out" | sed -e "s,[^.]*[.],," -e 's,=.*,,') && - KVM_DEVOPTS[$model]="$out" || - { error "bad device model $model?"; exit 1; } - fi - opts=( ${KVM_DEVOPTS[$model]} ) - for opt in "${opts[@]}"; do - [ "$input" = "$opt" ] && return 0 - done - return 1 -} - -qemu_supports_file_locking() { - # hackily check if qemu has file.locking in -drive params (LP: #1716028) - if [ -z "$_QEMU_SUPPORTS_FILE_LOCKING" ]; then - # The only way we could find to check presence of file.locking is - # qmp (query-qmp-schema). Simply checking if the virtio-blk driver - # supports 'share-rw' is expected to be equivalent and simpler. - isdevopt virtio-blk share-rw && - _QEMU_SUPPORTS_FILE_LOCKING=true || - _QEMU_SUPPORTS_FILE_LOCKING=false - debug 1 "qemu supports file locking = ${_QEMU_SUPPORTS_FILE_LOCKING}" - fi - [ "$_QEMU_SUPPORTS_FILE_LOCKING" = "true" ] - return -} - -padmac() { - # return a full mac, given a subset. - # assume whatever is input is the last portion to be - # returned, and fill it out with entries from MAC_PREFIX - local mac="$1" num="$2" prefix="${3:-$MAC_PREFIX}" itoks="" ptoks="" - # if input is empty set to :$num - [ -n "$mac" ] || mac=$(printf "%02x" "$num") || return - itoks=( ${mac//:/ } ) - ptoks=( ${prefix//:/ } ) - rtoks=( ) - for r in ${ptoks[@]:0:6-${#itoks[@]}} ${itoks[@]}; do - rtoks[${#rtoks[@]}]="0x$r" - done - _RET=$(printf "%02x:%02x:%02x:%02x:%02x:%02x" "${rtoks[@]}") -} - -make_nics_Usage() { - cat <: for each tap created - # type is one of "ovs" or "brctl" - local short_opts="v" - local long_opts="--verbose" - local getopt_out="" - getopt_out=$(getopt --name "${0##*/} make-nics" \ - --options "${short_opts}" --long "${long_opts}" -- "$@") && - eval set -- "${getopt_out}" || { make_nics_Usage 1>&2; return 1; } - - local cur="" next="" - while [ $# -ne 0 ]; do - cur=${1}; next=${2}; - case "$cur" in - -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; - --) shift; break;; - esac - shift; - done - - [ $# -ne 0 ] || { - make_nics_Usage 1>&2; error "must give bridge"; - return 1; - } - - local owner="" ovsbrs="" tap="" tapnum="0" brtype="" bridge="" - [ "$(id -u)" = "0" ] || { error "must be root for make-nics"; return 1; } - owner="${SUDO_USER:-root}" - ovsbrs="" - if command -v ovs-vsctl >/dev/null 2>&1; then - out=$(ovs-vsctl list-br) - out=$(echo "$out" | sed "s/\n/,/") - ovsbrs=",$out," - fi - for bridge in "$@"; do - [ "$bridge" = "user" ] && echo skip && continue - [ "${ovsbrs#*,${bridge},}" != "$ovsbrs" ] && - btype="ovs" || btype="brctl" - tapnum=0; - while [ -e /sys/class/net/tapvm$tapnum ]; do tapnum=$(($tapnum+1)); done - tap="tapvm$tapnum" - debug 1 "creating $tap:$btype on $bridge" 1>&2 - ip tuntap add mode tap user "$owner" "$tap" || - { error "failed to create tap '$tap' for '$owner'"; return 1; } - ip link set "$tap" up 1>&2 || { - error "failed to bring up $tap"; - ip tuntap del mode tap "$tap"; - return 1; - } - if [ "$btype" = "ovs" ]; then - ovs-vsctl add-port "$bridge" "$tap" 1>&2 || { - error "failed: ovs-vsctl add-port $bridge $tap"; - ovs-vsctl del-port "$bridge" "$tap" - return 1; - } - else - ip link set "$tap" master "$bridge" 1>&2 || { - error "failed to add tap '$tap' to '$bridge'" - ip tuntap del mode tap "$tap"; - return 1 - } - fi - echo "$tap:$btype" - done -} - -ovs_cleanup() { - [ "$(id -u)" = "0" ] || - { error "must be root for ovs-cleanup"; return 1; } - local item="" errors=0 - # TODO: if get owner (SUDO_USERNAME) and if that isn't - # the owner, then do not delete. - for item in "$@"; do - name=${item#*:} - bridge=${item%:*} - ovs-vsctl del-port "$bridge" "$name" || errors=$((errors+1)) - done - return $errors -} - -quote_cmd() { - local quote='"' x="" vline="" - for x in "$@"; do - if [ "${x#* }" != "${x}" ]; then - if [ "${x#*$quote}" = "${x}" ]; then - x="\"$x\"" - else - x="'$x'" - fi - fi - vline="${vline} $x" - done - echo "$vline" -} - -get_bios_opts() { - # get_bios_opts(bios, uefi, nvram) - # bios is a explicit bios to boot. - # uefi is boolean indicating uefi - # nvram is optional and indicates that ovmf vars should be copied - # to that file if it does not exist. if it exists, use it. - local bios="$1" uefi="${2:-false}" nvram="$3" - local ovmf_dir="/usr/share/OVMF" - local bios_opts="" pflash_common="if=pflash,format=raw" - unset _RET - _RET=( ) - if [ -n "$bios" ]; then - _RET=( -drive "${pflash_common},file=$bios" ) - return 0 - elif ! $uefi; then - return 0 - fi - - # ovmf in older releases (14.04) shipped only a single file - # /usr/share/ovmf/OVMF.fd - # newer ovmf ships split files - # /usr/share/OVMF/OVMF_CODE.fd - # /usr/share/OVMF/OVMF_VARS.fd - # with single file, pass only one file and read-write - # with split, pass code as readonly and vars as read-write - local joined="/usr/share/ovmf/OVMF.fd" - local code="/usr/share/OVMF/OVMF_CODE.fd" - local vars="/usr/share/OVMF/OVMF_VARS.fd" - local split="" nvram_src="" - if [ -e "$code" -o -e "$vars" ]; then - split=true - nvram_src="$vars" - elif [ -e "$joined" ]; then - split=false - nvram_src="$joined" - elif [ -n "$nvram" -a -e "$nvram" ]; then - error "WARN: nvram given, but did not find expected ovmf files." - error " assuming this is code and vars (OVMF.fd)" - split=false - else - error "uefi support requires ovmf bios: apt-get install -qy ovmf" - return 1 - fi - - if [ -n "$nvram" ]; then - if [ ! -f "$nvram" ]; then - cp "$nvram_src" "$nvram" || - { error "failed copy $nvram_src to $nvram"; return 1; } - debug 1 "copied $nvram_src to $nvram" - fi - else - debug 1 "uefi without --uefi-nvram storage." \ - "nvram settings likely will not persist." - nvram="${nvram_src}" - fi - - if [ ! -w "$nvram" ]; then - debug 1 "nvram file ${nvram} is readonly" - nvram_ro="readonly" - fi - - if $split; then - # to ensure bootability firmware must be first, then variables - _RET=( -drive "${pflash_common},file=$code,readonly" ) - fi - _RET=( "${_RET[@]}" - -drive "${pflash_common},file=$nvram${nvram_ro:+,${nvram_ro}}" ) -} - -main() { - local short_opts="hd:n:v" - local long_opts="bios:,help,dowait,disk:,dry-run,kvm:,no-dowait,netdev:,uefi,uefi-nvram:,verbose" - local getopt_out="" - getopt_out=$(getopt --name "${0##*/}" \ - --options "${short_opts}" --long "${long_opts}" -- "$@") && - eval set -- "${getopt_out}" || { bad_Usage; return 1; } - - local bridge="$DEF_BRIDGE" oifs="$IFS" - local netdevs="" need_tap="" ret="" p="" i="" pt="" cur="" conn="" - local kvm="" kvmcmd="" archopts="" - local def_disk_driver=${DEF_DISK_DRIVER:-"virtio-blk"} - local def_netmodel=${DEF_NETMODEL:-"virtio-net-pci"} - local bios="" uefi=false uefi_nvram="" - - archopts=( ) - kvmcmd=( ) - netdevs=( ) - addargs=( ) - diskdevs=( ) - diskargs=( ) - - # dowait: run qemu-system with a '&' and then 'wait' on the pid. - # the reason to do this or not do this has to do with interactivity - # if detached with &, then user input will not go to xkvm. - # if *not* detached, then signal handling is blocked until - # the foreground subprocess returns. which means we can't handle - # a sigterm and kill the qemu-system process. - # We default to dowait=false if input and output are a terminal - local dowait="" - [ -t 0 -a -t 1 ] && dowait=false || dowait=true - while [ $# -ne 0 ]; do - cur=${1}; next=${2}; - case "$cur" in - -h|--help) Usage; exit 0;; - -d|--disk) - diskdevs[${#diskdevs[@]}]="$next"; shift;; - --dry-run) DRY_RUN=true;; - --kvm) kvm="$next"; shift;; - -n|--netdev) - netdevs[${#netdevs[@]}]=$next; shift;; - -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; - --dowait) dowait=true;; - --no-dowait) dowait=false;; - --bios) bios="$next"; shift;; - --uefi) uefi=true;; - --uefi-nvram) uefi=true; uefi_nvram="$next"; shift;; - --) shift; break;; - esac - shift; - done - - [ ${#netdevs[@]} -eq 0 ] && netdevs=( "${DEF_BRIDGE}" ) - pt=( "$@" ) - - local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci" virtio_rng_device="virtio-rng-pci" - [ -n "$kvm" ] && kvm_pkg="none" - case $(uname -m) in - i?86) - [ -n "$kvm" ] || - { kvm="qemu-system-i386"; kvm_pkg="qemu-system-x86"; } - ;; - x86_64) - [ -n "$kvm" ] || - { kvm="qemu-system-x86_64"; kvm_pkg="qemu-system-x86"; } - ;; - s390x) - [ -n "$kvm" ] || - { kvm="qemu-system-s390x"; kvm_pkg="qemu-system-misc"; } - def_netmodel=${DEF_NETMODEL:-"virtio-net-ccw"} - # disable virtio-scsi-bus - virtio_scsi_bus="virtio-scsi-ccw" - virtio_blk_bus="virtio-blk-ccw" - virtio_rng_device="virtio-rng-ccw" - ;; - ppc64*) - [ -n "$kvm" ] || - { kvm="qemu-system-ppc64"; kvm_pkg="qemu-system-ppc"; } - def_netmodel="virtio-net-pci" - # virtio seems functional on in 14.10, but might want scsi here - #def_diskif="scsi" - archopts=( "${archopts[@]}" -machine pseries,usb=off ) - archopts=( "${archopts[@]}" -device spapr-vscsi ) - ;; - *) kvm=qemu-system-$(uname -m);; - esac - KVM="$kvm" - kvmcmd=( $kvm -enable-kvm ) - - local bios_opts="" - if [ -n "$bios" ] && $uefi; then - error "--uefi (or --uefi-nvram) is incompatible with --bios" - return 1 - fi - get_bios_opts "$bios" "$uefi" "$uefi_nvram" || - { error "failed to get bios opts"; return 1; } - bios_opts=( "${_RET[@]}" ) - - local out="" fmt="" bus="" unit="" index="" serial="" driver="" devopts="" - local busorindex="" driveopts="" cur="" val="" file="" wwn="" - for((i=0;i<${#diskdevs[@]};i++)); do - cur=${diskdevs[$i]} - IFS=","; set -- $cur; IFS="$oifs" - driver="" - id=$(printf "disk%02d" "$i") - file="" - fmt="" - bus="" - unit="" - index="" - serial="" - wwn="" - for tok in "$@"; do - [ "${tok#*=}" = "${tok}" -a -f "${tok}" -a -z "$file" ] && file="$tok" - val=${tok#*=} - case "$tok" in - driver=*) driver=$val;; - if=virtio) driver=virtio-blk;; - if=scsi) driver=scsi-hd;; - if=pflash) driver=;; - if=sd|if=mtd|floppy) fail "do not know what to do with $tok on $cur";; - id=*) id=$val;; - file=*) file=$val;; - fmt=*|format=*) fmt=$val;; - serial=*) serial=$val;; - wwn=*) wwn=$val;; - bus=*) bus=$val;; - unit=*) unit=$val;; - index=*) index=$val;; - esac - done - [ -z "$file" ] && fail "did not read a file from $cur" - if [ -f "$file" -a -z "$fmt" ]; then - out=$(LANG=C qemu-img info "$file") && - fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') || - { error "failed to determine format of $file"; return 1; } - elif [ -z "$fmt" ]; then - fmt=raw - fi - if [ -z "$driver" ]; then - driver="$def_disk_driver" - fi - if [ -z "$serial" ]; then - # use filename as serial if not provided a wwn - if [ -n "$wwn" ]; then - serial="$wwn" - else - serial="${file##*/}" - fi - fi - - # make sure we add either bus= or index= - if [ -n "$bus" -o "$unit" ] && [ -n "$index" ]; then - fail "bus and index cant be specified together: $cur" - elif [ -z "$bus" -a -z "$unit" -a -z "$index" ]; then - index=$i - elif [ -n "$bus" -a -z "$unit" ]; then - unit=$i - fi - - busorindex="${bus:+bus=$bus,unit=$unit}${index:+index=${index}}" - diskopts="file=${file},id=$id,if=none,format=$fmt,$busorindex" - devopts="$driver,drive=$id${serial:+,serial=${serial}}" - for tok in "$@"; do - case "$tok" in - id=*|if=*|driver=*|$file|file=*) continue;; - fmt=*|format=*) continue;; - serial=*|bus=*|unit=*|index=*) continue;; - file.locking=*) - qemu_supports_file_locking || { - debug 2 "qemu has no file locking." \ - "Dropping '$tok' from: $cur" - continue - };; - esac - isdevopt "$driver" "$tok" && devopts="${devopts},$tok" || - diskopts="${diskopts},${tok}" - done - case $driver in - virtio-blk-ccw) - # disable scsi when using virtio-blk-ccw - devopts="${devopts},scsi=off";; - esac - diskargs=( "${diskargs[@]}" -drive "$diskopts" -device "$devopts" ) - done - - local mnics_vflag="" - for((i=0;i<${VERBOSITY}-1;i++)); do mnics_vflag="${mnics_vflag}v"; done - [ -n "$mnics_vflag" ] && mnics_vflag="-${mnics_vflag}" - - # now go through and split out options - # -device virtio-net-pci,netdev=virtnet0,mac=52:54:31:15:63:02 - # -netdev type=tap,id=virtnet0,vhost=on,script=/etc/kvm/kvm-ifup.br0,downscript=no - local netopts="" devopts="" id="" need_taps=0 model="" - local device_args netdev_args - device_args=( ) - netdev_args=( ) - connections=( ) - for((i=0;i<${#netdevs[@]};i++)); do - id=$(printf "net%02d" "$i") - netopts=""; - devopts="" - # mac=auto is 'unspecified' (let qemu assign one) - mac="auto" - #vhost="off" - - IFS=","; set -- ${netdevs[$i]}; IFS="$oifs" - bridge=$1; shift; - if [ "$bridge" = "user" ]; then - netopts="type=user" - ntype="user" - connections[$i]="user" - else - need_taps=1 - ntype="tap" - netopts="type=tap" - connections[$i]="$bridge" - fi - netopts="${netopts},id=$id" - [ "$ntype" = "tap" ] && netopts="${netopts},script=no,downscript=no" - - model="${def_netmodel}" - for tok in "$@"; do - [ "${tok#model=}" = "${tok}" ] && continue - case "${tok#model=}" in - virtio) model=virtio-net-pci;; - *) model=${tok#model=};; - esac - done - - for tok in "$@"; do - case "$tok" in - mac=*) mac="${tok#mac=}"; continue;; - macaddr=*) mac=${tok#macaddr=}; continue;; - model=*) continue;; - esac - - isdevopt "$model" "$tok" && devopts="${devopts},$tok" || - netopts="${netopts},${tok}" - done - devopts=${devopts#,} - netopts=${netopts#,} - - if [ "$mac" != "auto" ]; then - [ "$mac" = "random" ] && randmac && mac="$_RET" - padmac "$mac" "$i" - devopts="${devopts:+${devopts},}mac=$_RET" - fi - devopts="$model,netdev=$id${devopts:+,${devopts}}" - #netopts="${netopts},vhost=${vhost}" - - device_args[$i]="$devopts" - netdev_args[$i]="$netopts" - done - - trap cleanup EXIT - - reqs=( "$kvm" ) - pkgs=( "$kvm_pkg" ) - for((i=0;i<${#reqs[@]};i++)); do - req=${reqs[$i]} - pkg=${pkgs[$i]} - [ "$pkg" = "none" ] && continue - command -v "$req" >/dev/null || { - missing="${missing:+${missing} }${req}" - missing_pkgs="${missing_pkgs:+${missing_pkgs} }$pkg" - } - done - if [ -n "$missing" ]; then - local reply cmd="" - cmd=( sudo apt-get --quiet install ${missing_pkgs} ) - error "missing prereqs: $missing"; - error "install them now with the following?: ${cmd[*]}" - read reply && [ "$reply" = "y" -o "$reply" = "Y" ] || - { error "run: apt-get install ${missing_pkgs}"; return 1; } - "${cmd[@]}" || { error "failed to install packages"; return 1; } - fi - - if [ $need_taps -ne 0 ]; then - local missing="" missing_pkgs="" reqs="" req="" pkgs="" pkg="" - for i in "${connections[@]}"; do - [ "$i" = "user" -o -e "/sys/class/net/$i" ] || - missing="${missing} $i" - done - [ -z "$missing" ] || { - error "cannot create connection on: ${missing# }." - error "bridges do not exist."; - return 1; - } - error "creating tap devices: ${connections[*]}" - if $DRY_RUN; then - error "sudo $0 tap-control make-nics" \ - $mnics_vflag "${connections[@]}" - taps="" - for((i=0;i<${#connections[@]};i++)); do - if [ "${connections[$i]}" = "user" ]; then - taps="${taps} skip" - else - taps="${taps} dryruntap$i:brctl" - fi - done - else - taps=$(sudo "$0" tap-control make-nics \ - ${mnics_vflag} "${connections[@]}") || - { error "$failed to make-nics ${connections[*]}"; return 1; } - fi - TAPDEVS=( ${taps} ) - for((i=0;i<${#TAPDEVS[@]};i++)); do - cur=${TAPDEVS[$i]} - [ "${cur#*:}" = "ovs" ] || continue - conn=${connections[$i]} - OVS_CLEANUP[${#OVS_CLEANUP[@]}]="${conn}:${cur%:*}" - done - - debug 2 "tapdevs='${TAPDEVS[@]}'" - [ ${#OVS_CLEANUP[@]} -eq 0 ] || error "OVS_CLEANUP='${OVS_CLEANUP[*]}'" - - for((i=0;i<${#TAPDEVS[@]};i++)); do - cur=${TAPDEVS[$i]} - [ "$cur" = "skip" ] && continue - netdev_args[$i]="${netdev_args[$i]},ifname=${cur%:*}"; - done - fi - - netargs=() - for((i=0;i<${#device_args[@]};i++)); do - netargs=( "${netargs[@]}" -device "${device_args[$i]}" - -netdev "${netdev_args[$i]}") - done - - local bus_devices - if [ -n "${virtio_scsi_bus}" ]; then - bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" ) - fi - local rng_devices - rng_devices=( -object "rng-random,filename=/dev/urandom,id=objrng0" - -device "$virtio_rng_device,rng=objrng0,id=rng0" ) - cmd=( "${kvmcmd[@]}" "${archopts[@]}" - "${bios_opts[@]}" - "${bus_devices[@]}" - "${rng_devices[@]}" - "${netargs[@]}" - "${diskargs[@]}" "${pt[@]}" ) - local pcmd=$(quote_cmd "${cmd[@]}") - error "$pcmd" - ${DRY_RUN} && return 0 - - if $dowait; then - "${cmd[@]}" & - KVM_PID=$! - debug 1 "kvm pid=$KVM_PID. my pid=$$" - wait - ret=$? - KVM_PID="" - else - "${cmd[@]}" - ret=$? - fi - return $ret -} - - -if [ "$1" = "tap-control" ]; then - shift - mode=$1 - shift || fail "must give mode to tap-control" - case "$mode" in - make-nics) make_nics "$@";; - ovs-cleanup) ovs_cleanup "$@";; - *) fail "tap mode must be either make-nics or ovs-cleanup";; - esac -else - main "$@" -fi diff --git a/tox.ini b/tox.ini index d4c750463bb..faf449bf114 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,7 @@ deps = hypothesis_jsonschema==0.23.1 isort==6.0.1 mypy==1.19.1 - pylint==3.3.8 + pylint==4.0.6 ruff==0.12.9 [latest_versions] @@ -94,27 +94,40 @@ deps = -r{toxinidir}/integration-requirements.txt {[testenv]deps} {[pinned_versions]deps} +allowlist_externals = + shfmt + sh commands = {envpython} -m ruff check {posargs:.} {envpython} -m pylint {posargs:cloudinit/ tests/ tools/} {envpython} -m black --check {posargs:.} {envpython} -m isort --check-only --diff {posargs:.} {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} + sh -c "{envpython} -m json.tool --indent 2 {[files]schema} | diff -q - {[files]schema} > /dev/null" + sh -c "{envpython} -m json.tool --indent 2 {[files]version} | diff -q - {[files]version} > /dev/null" + sh -c "{envpython} -m json.tool --indent 2 {[files]network_v1} | diff -q - {[files]network_v1} > /dev/null" + sh -c "{envpython} -m json.tool --indent 2 {[files]network_v2} | diff -q - {[files]network_v2} > /dev/null" + shfmt --list --case-indent --indent=4 --space-redirects --diff {posargs:./tools/} [testenv:check_format_tip] deps = -r{toxinidir}/integration-requirements.txt {[testenv]deps} {[latest_versions]deps} +allowlist_externals = + shfmt commands = {envpython} -m ruff check {posargs:.} {envpython} -m pylint {posargs:.} {envpython} -m black --check {posargs:.} {envpython} -m isort --check-only --diff {posargs:.} {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} + shfmt --list --case-indent --indent=4 --space-redirects --diff {posargs:./tools/} [testenv:do_format] deps = {[pinned_versions]deps} +allowlist_externals = + shfmt commands = {envpython} -m isort . {envpython} -m black . @@ -122,9 +135,12 @@ commands = {envpython} -m json.tool --indent 2 {[files]version} {[files]version} {envpython} -m json.tool --indent 2 {[files]network_v1} {[files]network_v1} {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} + shfmt --list --case-indent --indent=4 --space-redirects -w {posargs:./tools/} [testenv:do_format_tip] deps = {[latest_versions]deps} +allowlist_externals = + sh commands = {envpython} -m isort . {envpython} -m black . @@ -132,6 +148,7 @@ commands = {envpython} -m json.tool --indent 2 {[files]version} {[files]version} {envpython} -m json.tool --indent 2 {[files]network_v1} {[files]network_v1} {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} + shfmt --list --case-indent --indent=4 --space-redirects -w {posargs:./tools/} [testenv:py3] commands = {envpython} -m pytest -vv --cov=cloudinit --cov-branch {posargs:tests/unittests} @@ -186,6 +203,7 @@ deps = pytest-mock==3.6.1 responses==0.18.0 passlib==1.7.4 + pyfakefs==4.5.4 commands = {envpython} -m pytest --cov=cloud-init --cov-branch {posargs:tests/unittests} [testenv:doc] @@ -287,9 +305,8 @@ setenv = ON_JENKINS="1" [pytest] -# TODO: s/--strict/--strict-markers/ once pytest version is high enough testpaths = tools tests/unittests -addopts = --strict +addopts = --strict --strict-markers log_format = %(asctime)s %(levelname)-9s %(name)s:%(filename)s:%(lineno)d %(message)s log_date_format = %Y-%m-%d %H:%M:%S markers =