diff --git a/.zuul.d/centos.yaml b/.zuul.d/centos.yaml index 11a821d392..fa8956a711 100644 --- a/.zuul.d/centos.yaml +++ b/.zuul.d/centos.yaml @@ -1,13 +1,4 @@ --- -- project: - check: - jobs: - - kolla-build-centos9s - - kolla-ansible-centos9s - experimental: - jobs: - - kolla-build-no-infra-wheels-centos9s - - job: name: kolla-build-centos9s parent: kolla-base diff --git a/.zuul.d/debian.yaml b/.zuul.d/debian.yaml index 10d59bdaad..93b08629bf 100644 --- a/.zuul.d/debian.yaml +++ b/.zuul.d/debian.yaml @@ -1,24 +1,4 @@ --- -- project: - check: - jobs: - - kolla-build-debian - - kolla-build-debian-podman - - kolla-ansible-debian - check-arm64: - jobs: - - kolla-build-debian-aarch64 - - kolla-ansible-debian-aarch64 - gate: - jobs: - - kolla-build-debian - - kolla-build-debian-podman - - kolla-ansible-debian - experimental: - jobs: - - kolla-ansible-debian-ironic: - files: ^docker\/(base|dnsmasq|ironic|ironic-inspector|iscsid|openstack-base)\/.* - - job: name: kolla-build-debian parent: kolla-base diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index 4433e1cf39..b3c2d74b03 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -2,7 +2,6 @@ - project: templates: - openstack-python3-jobs-kolla - - openstack-python3-jobs-arm64 - openstack-cover-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 @@ -10,9 +9,6 @@ jobs: - kolla-tox-genconfig - openstack-tox-py312 - check-arm64: - jobs: - - openstack-tox-py312-arm64 gate: jobs: - kolla-tox-genconfig diff --git a/.zuul.d/rocky.yaml b/.zuul.d/rocky.yaml index 6b4ac43c00..067aac16c7 100644 --- a/.zuul.d/rocky.yaml +++ b/.zuul.d/rocky.yaml @@ -1,39 +1,4 @@ --- -- project: - check: - jobs: - - kolla-build-rocky9 - - kolla-build-rocky9-podman - - kolla-ansible-rocky9 - # Test rabbitmq and mariadb in multinode ceph jobs. - - kolla-ansible-rocky9-cephadm: - files: ^docker\/(base|cinder|glance|mariadb|openstack-base|rabbitmq)\/.* - - kolla-ansible-rocky9-ironic: - files: ^docker\/(base|dnsmasq|ironic|ironic-inspector|iscsid|openstack-base)\/.* - - kolla-ansible-rocky9-swift: - files: ^docker/(base|openstack-base|glance|swift)/ - - kolla-ansible-rocky9-mariadb: - files: ^docker/(base|mariadb)/ - - kolla-ansible-rocky9-masakari: - files: ^docker/(base|masakari|openstack-base)/ - - kolla-ansible-rocky9-octavia: - files: ^docker/(base|neutron|octavia|openstack-base|openvswitch|ovn)/ - - kolla-ansible-rocky9-ovn: - files: ^docker/(base|neutron|openstack-base|openvswitch|ovn)/ - - kolla-ansible-rocky9-prometheus-opensearch: - files: ^docker/(base|opensearch|fluentd|grafana|prometheus)/ - - kolla-ansible-rocky9-kvm: - files: ^docker/nova/ - - kolla-ansible-rocky9-cells: - files: ^docker/proxysql/ - - kolla-ansible-rocky9-bifrost: - files: ^docker/bifrost/ - gate: - jobs: - - kolla-build-rocky9 - - kolla-build-rocky9-podman - - kolla-ansible-rocky9 - - job: name: kolla-build-rocky9 parent: kolla-base diff --git a/kolla/image/kolla_worker.py b/kolla/image/kolla_worker.py index 7097899b78..0327ad0cb1 100644 --- a/kolla/image/kolla_worker.py +++ b/kolla/image/kolla_worker.py @@ -21,6 +21,7 @@ import time import jinja2 +import jinja2.sandbox from kolla.common import config as common_config from kolla.common import utils from kolla.engine_adapter import engine @@ -334,7 +335,7 @@ def create_dockerfiles(self): 'rpm_setup': self.rpm_setup, 'build_date': build_date, 'clean_package_cache': self.clean_package_cache} - env = jinja2.Environment( # nosec: not used to render HTML + env = jinja2.sandbox.SandboxedEnvironment( loader=jinja2.FileSystemLoader(self.working_dir)) env.filters.update(self._get_filters()) env.globals.update(self._get_methods()) @@ -347,7 +348,7 @@ def create_dockerfiles(self): tpl_dict = self._merge_overrides(self.conf.template_override) template_name = os.path.basename(list(tpl_dict.keys())[0]) values['parent_template'] = template - env = jinja2.Environment( # nosec: not used to render HTML + env = jinja2.sandbox.SandboxedEnvironment( loader=jinja2.DictLoader(tpl_dict)) env.filters.update(self._get_filters()) env.globals.update(self._get_methods()) diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index 78462d1317..6dddcdc454 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -11,6 +11,7 @@ # limitations under the License. import fixtures +import jinja2 import os import requests import sys @@ -579,6 +580,72 @@ def test_set_time(self): kolla.setup_working_dir() kolla.set_time() + @mock.patch.dict(os.environ, {'http_proxy': 'http://test-proxy:8080'}, + clear=False) + @mock.patch(engine_client) + def test_create_dockerfiles_env_http_proxy(self, mock_client): + tmpdir = self.useFixture(fixtures.TempDir()).path + base_dir = os.path.join(tmpdir, 'base') + os.makedirs(base_dir) + with open(os.path.join(base_dir, 'Dockerfile.j2'), 'w') as f: + f.write( + '{% if env.http_proxy %}' + 'ARG http_proxy={{ env.http_proxy }}' + '{% endif %}\n' + ) + with mock.patch.object(build.KollaWorker, '_get_images_dir', + return_value=tmpdir): + kolla = build.KollaWorker(self.conf) + kolla.setup_working_dir() + kolla.find_dockerfiles() + kolla.create_dockerfiles() + dockerfile = os.path.join(kolla.working_dir, 'base', 'Dockerfile') + with open(dockerfile) as f: + content = f.read() + self.assertIn('http://test-proxy:8080', content) + + @mock.patch(engine_client) + def test_create_dockerfiles_ssti_blocked(self, mock_client): + tmpdir = self.useFixture(fixtures.TempDir()).path + base_dir = os.path.join(tmpdir, 'base') + os.makedirs(base_dir) + with open(os.path.join(base_dir, 'Dockerfile.j2'), 'w') as f: + f.write( + "{{ self.__init__.__globals__['__builtins__']" + "['__import__']('os').popen('id').read() }}\n" + ) + with mock.patch.object(build.KollaWorker, '_get_images_dir', + return_value=tmpdir): + kolla = build.KollaWorker(self.conf) + kolla.setup_working_dir() + kolla.find_dockerfiles() + self.assertRaises(jinja2.exceptions.SecurityError, + kolla.create_dockerfiles) + + @mock.patch(engine_client) + def test_create_dockerfiles_template_override_ssti_blocked( + self, mock_client): + tmpdir = self.useFixture(fixtures.TempDir()).path + base_dir = os.path.join(tmpdir, 'base') + os.makedirs(base_dir) + with open(os.path.join(base_dir, 'Dockerfile.j2'), 'w') as f: + f.write('FROM {{ base_distro }}:{{ base_distro_tag }}\n') + override_file = os.path.join( + self.useFixture(fixtures.TempDir()).path, 'template_override.j2') + with open(override_file, 'w') as f: + f.write( + "{{ self.__init__.__globals__['__builtins__']" + "['__import__']('os').popen('id').read() }}\n" + ) + self.conf.set_override('template_override', [override_file]) + with mock.patch.object(build.KollaWorker, '_get_images_dir', + return_value=tmpdir): + kolla = build.KollaWorker(self.conf) + kolla.setup_working_dir() + kolla.find_dockerfiles() + self.assertRaises(jinja2.exceptions.SecurityError, + kolla.create_dockerfiles) + def _get_matched_images(self, images): return [image for image in images if image.status == utils.Status.MATCHED] diff --git a/kolla/tests/test_validate_all_file.py b/kolla/tests/test_validate_all_file.py new file mode 100644 index 0000000000..04dab530ee --- /dev/null +++ b/kolla/tests/test_validate_all_file.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import os +import tempfile +from unittest import mock + +from kolla.tests import base + +_VALIDATE_SCRIPT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', 'tools', + 'validate-all-file.py')) +_spec = importlib.util.spec_from_file_location('validate_all_file', + _VALIDATE_SCRIPT) +validate_all_file = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(validate_all_file) + +_SSTI_PAYLOAD = ( + "{{ self.__init__.__globals__['__builtins__']" + "['__import__']('os').popen('id').read() }}" +) + + +class ValidateAllFileTest(base.TestCase): + + def test_check_json_j2_ssti_blocked(self): + with tempfile.TemporaryDirectory() as tmpdir: + malicious = os.path.join(tmpdir, 'test.json.j2') + with open(malicious, 'w') as f: + f.write(_SSTI_PAYLOAD + '\n') + with mock.patch.object(validate_all_file, 'PROJECT_ROOT', tmpdir): + result = validate_all_file.check_json_j2() + self.assertEqual(1, result) diff --git a/tools/validate-all-file.py b/tools/validate-all-file.py index 67f03218ad..12a50a70c9 100755 --- a/tools/validate-all-file.py +++ b/tools/validate-all-file.py @@ -21,6 +21,7 @@ import sys import jinja2 +import jinja2.sandbox PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -75,7 +76,7 @@ def hostvars(): return collections.defaultdict(hostvars) def validate_json_j2(root, filename): - env = jinja2.Environment( # nosec: not used to render HTML + env = jinja2.sandbox.SandboxedEnvironment( loader=jinja2.FileSystemLoader(root)) env.filters['bool'] = bool_filter template = env.get_template(filename)