From 405ecfa5be13e137e3a86242d33f1aa092a0fbf7 Mon Sep 17 00:00:00 2001 From: Shivam Mehta Date: Fri, 30 Jan 2026 15:12:33 -0700 Subject: [PATCH 1/6] Allow for user provied repo config Signed-off-by: Shivam Mehta --- README.md | 24 +++++-- src/installer.py | 175 ++++++++++++----------------------------------- src/layer.py | 20 ++---- 3 files changed, 69 insertions(+), 150 deletions(-) diff --git a/README.md b/README.md index cf42755..df5a347 100644 --- a/README.md +++ b/README.md @@ -122,13 +122,25 @@ options: # Package repositories to add. This example uses YUM/DNF repositories. repos: - alias: 'rocky-baseos' - url: 'http://dl.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os' - - alias: 'rock_appstream' - url: 'http://dl.rockylinux.org/pub/rocky/8/AppStream/x86_64/os' - - alias: 'rock_powertools' - url: 'http://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os' + content: | + name=rocky-baseos + baseurl='http://dl.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os' + enabled=1 + - alias: 'rocky-appstream' + content: | + name=rocky-appstream + baseurl='http://dl.rockylinux.org/pub/rocky/8/AppStream/x86_64/os' + enabled=1 + - alias: 'rocky-powertools' + content: | + name=rocky-appstream + baseurl='http://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os' + enabled=1 - alias: 'epel' - url: 'http://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/' + content: | + name=epel + baseurl='http://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/' + enabled=1 # Package groups to install, in this example YUM/DNF package groups. package_groups: diff --git a/src/installer.py b/src/installer.py index e287945..2383b28 100644 --- a/src/installer.py +++ b/src/installer.py @@ -1,17 +1,16 @@ -import subprocess import logging import os import pathmod import tempfile +from pathlib import Path # Written Modules from utils import cmd class Installer: - def __init__(self, pkg_man, cname, mname, gpgcheck=True): + def __init__(self, pkg_man, cname, mname): self.pkg_man = pkg_man self.cname = cname self.mname = mname - self.gpgcheck = gpgcheck # Create temporary directory for logs, cache, etc. for package manager os.makedirs(os.path.join(mname, "tmp"), exist_ok=True) @@ -22,78 +21,32 @@ def __init__(self, pkg_man, cname, mname, gpgcheck=True): # DNF complains if the log directory is not present os.makedirs(os.path.join(self.tdir, "dnf/log")) - def install_scratch_repos(self, repos, repo_dest, proxy): + def install_scratch_repos(self, repos, repo_dest): # check if there are repos passed for install if len(repos) == 0: logging.info("REPOS: no repos passed to install\n") return - logging.info(f"REPOS: Installing these repos to {self.cname}") for r in repos: - args = [] - logging.info(r['alias'] + ': ' + r['url']) - if self.pkg_man == "zypper": - args.append("-D") - args.append(os.path.join(self.mname, pathmod.sep_strip(repo_dest))) - args.append("addrepo") - args.append("-f") - args.append("-p") - if 'priority' in r: - args.append(r['priority']) - else: - args.append('99') - args.append(r['url']) - args.append(r['alias']) - elif self.pkg_man == "dnf": - args.append("--setopt=reposdir="+os.path.join(self.mname, pathmod.sep_strip(repo_dest))) - args.append("--setopt=logdir="+os.path.join(self.tdir, self.pkg_man, "log")) - args.append("--setopt=cachedir="+os.path.join(self.tdir, self.pkg_man, "cache")) - if proxy != "": - args.append("--setopt=proxy="+proxy) - args.append("config-manager") - args.append("--save") - args.append("--add-repo") - args.append(r['url']) - - rc = cmd([self.pkg_man] + args) - if rc != 0: - raise Exception("Failed to install repo", r['alias'], r['url']) - - if proxy != "": - if r['url'].endswith('.repo'): - repo_name = r['url'].split('/')[-1].split('.repo')[0] + "*" - elif r['url'].startswith('https'): - repo_name = r['url'].split('https://')[1].replace('/','_') - elif r['url'].startswith('http'): - repo_name = r['url'].split('http://')[1].replace('/','_') - args = [] - args.append('config-manager') - args.append('--save') - args.append("--setopt=reposdir="+os.path.join(self.mname, pathmod.sep_strip(repo_dest))) - args.append("--setopt=logdir="+os.path.join(self.tdir, self.pkg_man, "log")) - args.append("--setopt=cachedir="+os.path.join(self.tdir, self.pkg_man, "cache")) - args.append('--setopt=*.proxy='+proxy) - args.append(repo_name) + logging.info(r['alias']) - rc = cmd([self.pkg_man] + args) - if rc != 0: - raise Exception("Failed to set proxy for repo", r['alias'], r['url'], proxy) + alias = r['alias'] + config = r['config'].lstrip() - if "gpg" in r: - # Using rpm apparently works for both Yum- and Zypper-based distros. - args = [] - if proxy != "": - arg_env = os.environ.copy() - arg_env['https_proxy'] = proxy - args.append("--root="+self.mname) - args.append("--import") - args.append(r["gpg"]) + # Makes sure configs start with an alias + if not config.startswith(f'[{alias}]'): + config = f'[{alias}]\n{config}' - rc = cmd(["rpm"] + args) - if rc != 0: - raise Exception("Failed to install gpg key for", r['alias'], "at URL", r['gpg']) + repo_path = Path(*[self.mname, pathmod.sep_strip(repo_dest), f'{alias}.repo']) + try: + repo_path.parent.mkdir(parents=True, exist_ok=True) + logging.info(f"REPOS: Writing {alias} to {repo_dest}") + with open(repo_path, "w") as f: + f.write(config) + except Exception as e: + raise Exception(f"Failed to generate repo file for r['alias'] got:\n{e}") - def install_scratch_packages(self, packages, registry_loc, proxy): + def install_scratch_packages(self, packages, registry_loc): # check if there are packages to install if len(packages) == 0: logging.warn("PACKAGES: no packages passed to install\n") @@ -109,21 +62,14 @@ def install_scratch_packages(self, packages, registry_loc, proxy): args.append(os.path.join(self.mname, pathmod.sep_strip(registry_loc))) args.append("-C") args.append(self.tdir) - args.append("--no-gpg-checks") args.append("--installroot") args.append(self.mname) args.append("install") args.append("-l") args.extend(packages) elif self.pkg_man == "dnf": - args.append("--setopt=reposdir="+os.path.join(self.mname, pathmod.sep_strip(registry_loc))) - args.append("--setopt=logdir="+os.path.join(self.tdir, self.pkg_man, "log")) - args.append("--setopt=cachedir="+os.path.join(self.tdir, self.pkg_man, "cache")) - if proxy != "": - args.append("--setopt=proxy="+proxy) args.append("install") args.append("-y") - args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(packages) @@ -135,7 +81,7 @@ def install_scratch_packages(self, packages, registry_loc, proxy): if rc == 107: logging.warn("one or more RPM postscripts failed to run") - def install_scratch_package_groups(self, package_groups, registry_loc, proxy): + def install_scratch_package_groups(self, package_groups): # check if there are packages groups to install if len(package_groups) == 0: logging.warn("PACKAGE GROUPS: no package groups passed to install\n") @@ -148,14 +94,8 @@ def install_scratch_package_groups(self, package_groups, registry_loc, proxy): if self.pkg_man == "zypper": logging.warn("zypper does not support package groups") elif self.pkg_man == "dnf": - args.append("--setopt=reposdir="+os.path.join(self.mname, pathmod.sep_strip(registry_loc))) - args.append("--setopt=logdir="+os.path.join(self.tdir, self.pkg_man, "log")) - args.append("--setopt=cachedir="+os.path.join(self.tdir, self.pkg_man, "cache")) - if proxy != "": - args.append("--setopt=proxy="+proxy) args.append("groupinstall") args.append("-y") - args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(package_groups) @@ -164,7 +104,7 @@ def install_scratch_package_groups(self, package_groups, registry_loc, proxy): if rc == 104: raise Exception("Installing base packages failed") - def install_scratch_modules(self, modules, registry_loc, proxy): + def install_scratch_modules(self, modules): # check if there are modules groups to install if len(modules) == 0: logging.warn("PACKAGE MODULES: no modules passed to install\n") @@ -178,15 +118,9 @@ def install_scratch_modules(self, modules, registry_loc, proxy): logging.warn("zypper does not support package groups") return elif self.pkg_man == "dnf": - args.append("--setopt=reposdir="+os.path.join(self.mname, pathmod.sep_strip(registry_loc))) - args.append("--setopt=logdir="+os.path.join(self.tdir, self.pkg_man, "log")) - args.append("--setopt=cachedir="+os.path.join(self.tdir, self.pkg_man, "cache")) - if proxy != "": - args.append("--setopt=proxy="+proxy) args.append("module") args.append(mod_cmd) args.append("-y") - args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(mod_list) @@ -194,7 +128,7 @@ def install_scratch_modules(self, modules, registry_loc, proxy): if rc != 0: raise Exception("Failed to run module cmd", mod_cmd, ' '.join(mod_list)) - def install_repos(self, repos, proxy): + def install_repos(self, repos, repo_dest): # check if there are repos passed for install if len(repos) == 0: logging.info("REPOS: no repos passed to install\n") @@ -202,44 +136,32 @@ def install_repos(self, repos, proxy): logging.info(f"REPOS: Installing these repos to {self.cname}") for r in repos: - logging.info(r['alias'] + ': ' + r['url']) - if self.pkg_man == "zypper": - if 'priority' in r: - priority = r['priority'] - else: - priority = 99 - rargs = ' addrepo -f -p ' + priority + ' ' + r['url'] + ' ' + r['alias'] - elif self.pkg_man == "dnf": - rargs = ' config-manager --save --add-repo ' + r['url'] - - args = [self.cname, '--', 'bash', '-c', self.pkg_man + rargs] - rc = cmd(["buildah","run"] + args) - if rc != 0: - raise Exception("Failed to install repo", r['alias'], r['url']) - # Set Proxy if using DNF - if proxy != "": - if r['url'].endswith('.repo'): - repo_name = r['url'].split('/')[-1].split('.repo')[0] + "*" - elif r['url'].startswith('https'): - repo_name = r['url'].split('https://')[1].replace('/','_') - elif r['url'].startswith('http'): - repo_name = r['url'].split('http://')[1].replace('/','_') - pargs = ' config-manager --save --setopt=*.proxy= ' + proxy + ' ' + repo_name - - args = [self.cname, '--', 'bash', '-c', self.pkg_man + pargs] - rc = cmd(["buildah","run"] + args) + alias = r['alias'] + config = r['config'].lstrip() + + # Makes sure configs start with an alias + if not config.startswith(f'[{alias}]'): + config = f'[{alias}]\n{config}' + + with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp: + tmp.write(config) + tmp_path = tmp.name + + repo_path = Path(*[repo_dest, f'{alias}.repo']) + copy_config = ['buildah', 'copy' , self.cname, tmp_path, repo_path] + mkparent_dir = ['buildah', 'run', self.cname, '--', 'mkdir', '-p', repo_dest] + try: + rc = cmd(mkparent_dir) if rc != 0: - raise Exception("Failed to set proxy for repo", r['alias'], r['url'], proxy) - - if "gpg" in r: - # Using rpm apparently works for both Yum- and Zypper-based distros. - gargs = [self.cname, '--', 'bash', '-c', 'rpm --import ' + r['gpg']] - if proxy != "": - arg_env = os.environ.copy() - arg_env['https_proxy'] = proxy - rc = cmd(["buildah","run"] + gargs) + raise Exception(f"Failed to create parent dir {repo_dest}") + rc = cmd(copy_config) if rc != 0: - raise Exception("Failed to install gpg key for", r['alias'], "at URL", r['gpg']) + raise Exception(f"Failed to generate repo config at {repo_path}") + logging.info(f"REPOS: Writing {alias} to {repo_path}") + except Exception as e: + raise Exception(f"Failed to generate repo file for r['alias'] got:\n{e}") + finally: + os.remove(tmp_path) def install_packages(self, packages): if len(packages) == 0: @@ -249,11 +171,6 @@ def install_packages(self, packages): logging.info("\n".join(packages)) args = [self.cname, '--', 'bash', '-c'] pkg_cmd = [self.pkg_man] - if self.gpgcheck is not True: - if self.pkg_man == 'dnf': - pkg_cmd.append('--nogpgcheck') - elif self.pkg_man == 'zypper': - pkg_cmd.append('--no-gpg-checks') args.append(" ".join(pkg_cmd + [ 'install', '-y'] + packages)) cmd(["buildah","run"] + args) @@ -267,8 +184,6 @@ def install_package_groups(self, package_groups): pkg_cmd = [self.pkg_man, 'groupinstall', '-y'] if self.pkg_man == "zypper": logging.warn("zypper does not support package groups") - if self.gpgcheck is not True: - pkg_cmd.append('--nogpgcheck') args.append(" ".join(pkg_cmd + [f'"{pg}"' for pg in package_groups])) cmd(["buildah","run"] + args) diff --git a/src/layer.py b/src/layer.py index df02c14..fb4b021 100644 --- a/src/layer.py +++ b/src/layer.py @@ -24,14 +24,6 @@ def _build_base(self, repos, modules, packages, package_groups, remove_packages, container = self.args['name'] registry_opts_pull = self.args['registry_opts_pull'] package_manager = self.args['pkg_man'] - if 'gpgcheck' in self.args: - gpgcheck = self.args['gpgcheck'] - else: - gpgcheck = True - if 'proxy' in self.args: - proxy = self.args['proxy'] - else: - proxy = "" # container and mount name def buildah_handler(line): @@ -91,7 +83,7 @@ def buildah_handler(line): inst = None try: - inst = installer.Installer(package_manager, cname, mname, gpgcheck) + inst = installer.Installer(package_manager, cname, mname) except Exception as e: self.logger.error(f"Error preparing installer: {e}") cmd(["buildah","rm"] + [cname]) @@ -104,9 +96,9 @@ def buildah_handler(line): # Install Repos try: if parent == "scratch": - inst.install_scratch_repos(repos, repo_dest, proxy) + inst.install_scratch_repos(repos, repo_dest) else: - inst.install_repos(repos, proxy) + inst.install_repos(repos, repo_dest) except Exception as e: self.logger.error(f"Error installing repos: {e}") cmd(["buildah","rm"] + [cname]) @@ -120,11 +112,11 @@ def buildah_handler(line): try: if parent == "scratch": # Enable modules - inst.install_scratch_modules(modules, repo_dest, self.args['proxy']) + inst.install_scratch_modules(modules) # Base Package Groups - inst.install_scratch_package_groups(package_groups, repo_dest, proxy) + inst.install_scratch_package_groups(package_groups) # Packages - inst.install_scratch_packages(packages, repo_dest, proxy) + inst.install_scratch_packages(packages, repo_dest) else: inst.install_package_groups(package_groups) inst.install_packages(packages) From 1bf1cd2c4260f80766cec2b2874792f583f9a4cf Mon Sep 17 00:00:00 2001 From: Shivam Mehta Date: Fri, 30 Jan 2026 16:25:48 -0700 Subject: [PATCH 2/6] zypper changes Signed-off-by: Shivam Mehta --- dockerfiles/zypper/Dockerfile | 8 +++----- src/installer.py | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dockerfiles/zypper/Dockerfile b/dockerfiles/zypper/Dockerfile index 9e64b4f..e4e9372 100644 --- a/dockerfiles/zypper/Dockerfile +++ b/dockerfiles/zypper/Dockerfile @@ -42,8 +42,6 @@ RUN zypper -n --no-gpg-checks install \ tar \ squashfs \ libcap-progs \ - cni \ - cni-plugins \ rsync \ unzip @@ -59,14 +57,14 @@ COPY requirements.txt /tmp/requirements.txt # install ansible-bender #RUN pip3.11 --proxy=http://192.12.184.19:8080 install ansible==4.10.0 ansible-base==2.10.17 ansible-bender boto3 dnspython requests jinja2_ansible_filters -RUN pip3.11 --proxy=http://192.12.184.19:8080 install -r /tmp/requirements.txt && \ +RUN pip3.11 install -r /tmp/requirements.txt && \ rm -f /tmp/requirements.txt RUN echo "%_netsharedpath /dev" >> /etc/rpm/macros # allow fuse mount -COPY containers/storage.conf /etc/containers/storage.conf -RUN chmod 0644 /etc/containers/storage.conf +#COPY containers/storage.conf /etc/containers/storage.conf +#RUN chmod 0644 /etc/containers/storage.conf # Allow non-root to run buildah commands RUN setcap cap_setuid=ep "$(command -v newuidmap)" && \ diff --git a/src/installer.py b/src/installer.py index 2383b28..0888504 100644 --- a/src/installer.py +++ b/src/installer.py @@ -28,13 +28,11 @@ def install_scratch_repos(self, repos, repo_dest): return for r in repos: - logging.info(r['alias']) - alias = r['alias'] config = r['config'].lstrip() # Makes sure configs start with an alias - if not config.startswith(f'[{alias}]'): + if not config.startswith(f'['): config = f'[{alias}]\n{config}' repo_path = Path(*[self.mname, pathmod.sep_strip(repo_dest), f'{alias}.repo']) @@ -57,11 +55,13 @@ def install_scratch_packages(self, packages, registry_loc): args = [] if self.pkg_man == "zypper": + args.append("--non-interactive") args.append("-n") args.append("-D") args.append(os.path.join(self.mname, pathmod.sep_strip(registry_loc))) args.append("-C") args.append(self.tdir) + args.append('--no-gpg-checks') args.append("--installroot") args.append(self.mname) args.append("install") @@ -140,7 +140,7 @@ def install_repos(self, repos, repo_dest): config = r['config'].lstrip() # Makes sure configs start with an alias - if not config.startswith(f'[{alias}]'): + if not config.startswith(f'['): config = f'[{alias}]\n{config}' with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp: From 6d0a76644fbbda80ffcd236f2207636d9a3dba8b Mon Sep 17 00:00:00 2001 From: Travis Cotton Date: Tue, 3 Feb 2026 11:06:17 -0700 Subject: [PATCH 3/6] add --gpg-auto-import-keys to zypper package installs. moving RHEL repos to a more generic place Signed-off-by: Travis Cotton --- src/installer.py | 2 ++ src/layer.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/installer.py b/src/installer.py index 0888504..5db99a1 100644 --- a/src/installer.py +++ b/src/installer.py @@ -171,6 +171,8 @@ def install_packages(self, packages): logging.info("\n".join(packages)) args = [self.cname, '--', 'bash', '-c'] pkg_cmd = [self.pkg_man] + if self.pkg_man == zypper: + pkg_cmd.append('--gpg-auto-import-keys') args.append(" ".join(pkg_cmd + [ 'install', '-y'] + packages)) cmd(["buildah","run"] + args) diff --git a/src/layer.py b/src/layer.py index fb4b021..35ae941 100644 --- a/src/layer.py +++ b/src/layer.py @@ -46,7 +46,7 @@ def buildah_handler(line): if package_manager == "zypper": repo_dest = "/etc/zypp/repos.d" elif package_manager == "dnf": - repo_dest = os.path.expanduser("~/.pkg_repos/yum.repos.d") + repo_dest = os.path.expanduser("/etc/imgbuild/yum.repos.d") # Create repo dest, if needed os.makedirs(os.path.join(mname, pathmod.sep_strip(repo_dest)), exist_ok=True) From ef2b887bc72af6dfba6b21bbb893b36692dd9b53 Mon Sep 17 00:00:00 2001 From: Shivam Mehta Date: Mon, 9 Feb 2026 09:44:18 -0700 Subject: [PATCH 4/6] restore zypper dockerfile Signed-off-by: Shivam Mehta --- dockerfiles/zypper/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dockerfiles/zypper/Dockerfile b/dockerfiles/zypper/Dockerfile index e4e9372..9e64b4f 100644 --- a/dockerfiles/zypper/Dockerfile +++ b/dockerfiles/zypper/Dockerfile @@ -42,6 +42,8 @@ RUN zypper -n --no-gpg-checks install \ tar \ squashfs \ libcap-progs \ + cni \ + cni-plugins \ rsync \ unzip @@ -57,14 +59,14 @@ COPY requirements.txt /tmp/requirements.txt # install ansible-bender #RUN pip3.11 --proxy=http://192.12.184.19:8080 install ansible==4.10.0 ansible-base==2.10.17 ansible-bender boto3 dnspython requests jinja2_ansible_filters -RUN pip3.11 install -r /tmp/requirements.txt && \ +RUN pip3.11 --proxy=http://192.12.184.19:8080 install -r /tmp/requirements.txt && \ rm -f /tmp/requirements.txt RUN echo "%_netsharedpath /dev" >> /etc/rpm/macros # allow fuse mount -#COPY containers/storage.conf /etc/containers/storage.conf -#RUN chmod 0644 /etc/containers/storage.conf +COPY containers/storage.conf /etc/containers/storage.conf +RUN chmod 0644 /etc/containers/storage.conf # Allow non-root to run buildah commands RUN setcap cap_setuid=ep "$(command -v newuidmap)" && \ From aed190b00e5820cea42d66e4c5801607b9e25a54 Mon Sep 17 00:00:00 2001 From: Shivam Mehta Date: Tue, 17 Feb 2026 11:13:02 -0700 Subject: [PATCH 5/6] Re add 'nogpgcheck' to group and module installs Signed-off-by: Shivam Mehta --- src/installer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/installer.py b/src/installer.py index 5db99a1..1302da6 100644 --- a/src/installer.py +++ b/src/installer.py @@ -61,7 +61,6 @@ def install_scratch_packages(self, packages, registry_loc): args.append(os.path.join(self.mname, pathmod.sep_strip(registry_loc))) args.append("-C") args.append(self.tdir) - args.append('--no-gpg-checks') args.append("--installroot") args.append(self.mname) args.append("install") @@ -96,6 +95,7 @@ def install_scratch_package_groups(self, package_groups): elif self.pkg_man == "dnf": args.append("groupinstall") args.append("-y") + args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(package_groups) @@ -121,6 +121,7 @@ def install_scratch_modules(self, modules): args.append("module") args.append(mod_cmd) args.append("-y") + args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(mod_list) @@ -171,7 +172,7 @@ def install_packages(self, packages): logging.info("\n".join(packages)) args = [self.cname, '--', 'bash', '-c'] pkg_cmd = [self.pkg_man] - if self.pkg_man == zypper: + if self.pkg_man == "zypper": pkg_cmd.append('--gpg-auto-import-keys') args.append(" ".join(pkg_cmd + [ 'install', '-y'] + packages)) cmd(["buildah","run"] + args) From 093621c5223ae1a63d7f9e9e2b4c258cb7b24483 Mon Sep 17 00:00:00 2001 From: Shivam Mehta Date: Fri, 27 Feb 2026 11:48:22 -0700 Subject: [PATCH 6/6] Add gpg key import feature Signed-off-by: Shivam Mehta --- examples/README.md | 34 +++- examples/containerized/base.yaml | 37 ++-- examples/containerized/compute.yaml | 16 +- examples/mini-bootcamp/rocky-8-base.yaml | 37 ++-- examples/mini-bootcamp/rocky-8-compute.yaml | 69 ++++--- .../mini-bootcamp/rocky-8-kubernetes.yaml | 52 ++++-- examples/mini-bootcamp/rocky-9-base.yaml | 23 ++- src/installer.py | 170 ++++++++++++++---- src/layer.py | 113 ++++++++---- tests/dnf/rocky9_scratch.yaml | 14 +- 10 files changed, 420 insertions(+), 145 deletions(-) diff --git a/examples/README.md b/examples/README.md index 6f76c27..947973f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -28,14 +28,23 @@ options: repos: - alias: 'Rock_BaseOS' - url: 'https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' - alias: 'Rock_AppStream' - url: 'https://download.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' - alias: 'Epel' - url: 'https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/' - gpg: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9' + config: | + baseurl=https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9' package_groups: @@ -61,6 +70,19 @@ cmds: - cmd: 'echo DRACUT LOG:; cat /tmp/dracut.log' ``` +### Repository Configuration + +Each repository requires: +- **`alias`**: Short name for the repo (used for the `.repo` filename) +- **`config`**: Repository configuration content (will be written to `/etc/imgbuild/yum.repos.d/.repo`) + - Do not include the `[alias]` header - it's added automatically by the code + - Must include `baseurl`, typically includes `enabled=1` and `gpgcheck=1` +- **`gpg_key`** (optional): URL to the GPG key for this repository + - If all repos have `gpg_key` defined, keys will be downloaded and imported automatically + - Packages will be installed without `--nogpgcheck` + - If any repo is missing `gpg_key` or download fails, `--nogpgcheck` will be used with a warning + - Works for both scratch and non-scratch installs + We have an example that activates our wireguard tunnel for cloud-init in [rocky-9-base.yaml](/examples/mini-bootcamp/rocky-9-base.yaml). This is part of the larger tutorial for running an OpenCHAMI system. You can use the contents of the file above or the included example file in the command below. ## 2. Use the published container to build the image diff --git a/examples/containerized/base.yaml b/examples/containerized/base.yaml index 411ddd8..cbf6dfc 100644 --- a/examples/containerized/base.yaml +++ b/examples/containerized/base.yaml @@ -8,20 +8,35 @@ options: repos: - alias: 'Rock_BaseOS' - url: 'https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_AppStream' - url: 'https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_PowerTools' - url: 'https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Epel' - url: 'https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/' - gpg: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' + config: | + baseurl=https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' - alias: 'elrepo' - url: 'http://elrepo.org/linux/elrepo/el8/x86_64/' - gpg: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' + config: | + baseurl=http://elrepo.org/linux/elrepo/el8/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' package_groups: - 'Minimal Install' @@ -45,4 +60,4 @@ packages: cmds: - cmd: 'dracut --add "dmsquash-live livenet network-manager" --kver $(basename /lib/modules/*) -N -f --logfile /tmp/dracut.log 2>/dev/null' - - cmd: 'echo DRACUT LOG:; cat /tmp/dracut.log' \ No newline at end of file + - cmd: 'echo DRACUT LOG:; cat /tmp/dracut.log' diff --git a/examples/containerized/compute.yaml b/examples/containerized/compute.yaml index 536313c..5a86114 100644 --- a/examples/containerized/compute.yaml +++ b/examples/containerized/compute.yaml @@ -16,11 +16,17 @@ options: repos: - alias: 'OpenHPC' - url: 'https://repos.openhpc.community/OpenHPC/2/EL_8/' - gpg: 'https://repos.openhpc.community/OpenHPC/2/EL_8/repodata/repomd.xml.key' + config: | + baseurl=https://repos.openhpc.community/OpenHPC/2/EL_8/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://repos.openhpc.community/OpenHPC/2/EL_8/repodata/repomd.xml.key' - alias: 'OpenHPC-updates' - url: 'https://repos.openhpc.community/OpenHPC/2/update.2.9/EL_8/' - gpg: 'https://repos.openhpc.community/OpenHPC/2/update.2.9/EL_8/repodata/repomd.xml.key' + config: | + baseurl=https://repos.openhpc.community/OpenHPC/2/update.2.9/EL_8/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://repos.openhpc.community/OpenHPC/2/update.2.9/EL_8/repodata/repomd.xml.key' packages: - python3 @@ -39,4 +45,4 @@ packages: - tcpdump - traceroute - nss_db - - lua-posix \ No newline at end of file + - lua-posix diff --git a/examples/mini-bootcamp/rocky-8-base.yaml b/examples/mini-bootcamp/rocky-8-base.yaml index 37457a6..f4d8c32 100644 --- a/examples/mini-bootcamp/rocky-8-base.yaml +++ b/examples/mini-bootcamp/rocky-8-base.yaml @@ -10,20 +10,35 @@ options: repos: - alias: 'Rock_BaseOS' - url: 'https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_AppStream' - url: 'https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_PowerTools' - url: 'https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Epel' - url: 'https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/' - gpg: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' + config: | + baseurl=https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' - alias: 'elrepo' - url: 'http://elrepo.org/linux/elrepo/el8/x86_64/' - gpg: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' + config: | + baseurl=http://elrepo.org/linux/elrepo/el8/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' package_groups: - 'Minimal Install' @@ -52,4 +67,4 @@ copyfiles: cmds: - cmd: 'dracut --add "dmsquash-live livenet network-manager" --kver $(basename /lib/modules/*) -N -f --logfile /tmp/dracut.log 2>/dev/null' - - cmd: 'echo DRACUT LOG:; cat /tmp/dracut.log' \ No newline at end of file + - cmd: 'echo DRACUT LOG:; cat /tmp/dracut.log' diff --git a/examples/mini-bootcamp/rocky-8-compute.yaml b/examples/mini-bootcamp/rocky-8-compute.yaml index 4c921b1..37ed019 100644 --- a/examples/mini-bootcamp/rocky-8-compute.yaml +++ b/examples/mini-bootcamp/rocky-8-compute.yaml @@ -11,38 +11,65 @@ options: - '--tls-verify=false' repos: - alias: 'Rock_BaseOS' - url: 'https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_AppStream' - url: 'https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_PowerTools' - url: 'https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Epel' - url: 'https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/' - gpg: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' + config: | + baseurl=https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' - alias: 'elrepo' - url: 'http://elrepo.org/linux/elrepo/el8/x86_64/' - gpg: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' + config: | + baseurl=http://elrepo.org/linux/elrepo/el8/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' - alias: 'OpenHPC' - url: 'http://dist.si.usrc/repo/el8/openhpc' - gpg: 'http://dist.si.usrc/repo/el8/openhpc/repodata/repomd.xml.key' + config: | + baseurl=http://dist.si.usrc/repo/el8/openhpc + enabled=1 + gpgcheck=1 + gpg_key: 'http://dist.si.usrc/repo/el8/openhpc/repodata/repomd.xml.key' - alias: 'OpenHPC-updates' - url: 'http://dist.si.usrc/repo/el8/openhpc/updates' - gpg: 'http://dist.si.usrc/repo/el8/openhpc/updates/repodata/repomd.xml.key' + config: | + baseurl=http://dist.si.usrc/repo/el8/openhpc/updates + enabled=1 + gpgcheck=1 + gpg_key: 'http://dist.si.usrc/repo/el8/openhpc/updates/repodata/repomd.xml.key' package_groups: - 'Development Tools' packages: + - hwloc + - valgrind + - prun-ohpc + - lmod-ohpc + - openssh-clients + - vim + - nfs-utils + - chrony + - perf + - nss-pam-ldapd + - nss_db + - lua-posix - slurm-ohpc - - slurm-slurmd-ohpc - - slurm-example-configs-ohpc cmds: - - cmd: 'chown -R munge:munge /var/lib/munge' - - cmd: 'chown -R munge:munge /var/log/munge' - - cmd: 'chown -R munge:munge /etc/munge' - - cmd: 'systemctl enable slurmd' - - cmd: 'systemctl start slurmd' \ No newline at end of file + - cmd: 'systemctl enable chronyd' diff --git a/examples/mini-bootcamp/rocky-8-kubernetes.yaml b/examples/mini-bootcamp/rocky-8-kubernetes.yaml index 5a1011d..60ebbe1 100644 --- a/examples/mini-bootcamp/rocky-8-kubernetes.yaml +++ b/examples/mini-bootcamp/rocky-8-kubernetes.yaml @@ -11,30 +11,48 @@ options: - '--tls-verify=false' repos: - alias: 'Rock_BaseOS' - url: 'https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/BaseOS/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_AppStream' - url: 'https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Rock_PowerTools' - url: 'https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' + config: | + baseurl=https://dl.rockylinux.org/pub/rocky/8/PowerTools/x86_64/os + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-8' - alias: 'Epel' - url: 'https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/' - gpg: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' + config: | + baseurl=https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8' - alias: 'elrepo' - url: 'http://elrepo.org/linux/elrepo/el8/x86_64/' - gpg: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' + config: | + baseurl=http://elrepo.org/linux/elrepo/el8/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://www.elrepo.org/RPM-GPG-KEY-elrepo.org' - alias: 'kubernetes' - url: 'https://pkgs.k8s.io/core:/stable:/v1.32/rpm/' - gpg: 'https://pkgs.k8s.io/core:/stable:/v1.32/rpm/repodata/repomd.xml.key' + config: | + baseurl=https://pkgs.k8s.io/core:/stable:/v1.32/rpm/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://pkgs.k8s.io/core:/stable:/v1.32/rpm/repodata/repomd.xml.key' packages: - kubelet - kubeadm - kubectl - - e2fsprogs - - iproute - - iptables - - systemd - - openssl \ No newline at end of file + - containerd.io + +cmds: + - cmd: 'systemctl enable kubelet' + - cmd: 'systemctl enable containerd' diff --git a/examples/mini-bootcamp/rocky-9-base.yaml b/examples/mini-bootcamp/rocky-9-base.yaml index 46000cd..f6053df 100644 --- a/examples/mini-bootcamp/rocky-9-base.yaml +++ b/examples/mini-bootcamp/rocky-9-base.yaml @@ -10,14 +10,23 @@ options: repos: - alias: 'Rock_BaseOS' - url: 'https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' - alias: 'Rock_AppStream' - url: 'https://download.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' + config: | + baseurl=https://download.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' - alias: 'Epel' - url: 'https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/' - gpg: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9' + config: | + baseurl=https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9' package_groups: @@ -46,4 +55,4 @@ copyfiles: cmds: - cmd: 'dracut --add "dmsquash-live livenet network-manager" --kver $(basename /lib/modules/*) -N -f --logfile /tmp/dracut.log 2>/dev/null' - - cmd: 'echo DRACUT LOG:; cat /tmp/dracut.log' \ No newline at end of file + - cmd: 'echo DRACUT LOG:; cat /tmp/dracut.log' diff --git a/src/installer.py b/src/installer.py index 3e962e6..3b65e7d 100644 --- a/src/installer.py +++ b/src/installer.py @@ -3,9 +3,13 @@ import pathmod import tempfile from pathlib import Path +import urllib.request +import urllib.error + # Written Modules from utils import cmd + class Installer: def __init__(self, pkg_man, cname, mname): self.pkg_man = pkg_man @@ -21,12 +25,84 @@ def __init__(self, pkg_man, cname, mname): # DNF complains if the log directory is not present os.makedirs(os.path.join(self.tdir, "dnf/log")) + def download_and_import_gpg_keys_from_repos(self, repos): + """ + Download GPG keys from repos and import them into the RPM database. + + Args: + repos: List of repo dicts, each may contain 'gpg_key' field + + Returns: + True if all repos have gpg_key and all keys were successfully downloaded and imported + False if any repo lacks gpg_key or any key failed to download or import + """ + # Extract gpg_key URLs and validate all repos have them + gpg_key_urls = [] + for repo in repos: + alias = repo.get("alias", "unknown") + if "gpg_key" not in repo: + logging.info(f"Repo '{alias}' missing gpg_key field") + return False + gpg_key_urls.append(repo["gpg_key"]) + + if not gpg_key_urls: + logging.info("No GPG key URLs found in repos") + return False + + # Deduplicate URLs + unique_urls = list(set(gpg_key_urls)) + logging.info( + f"Found {len(unique_urls)} unique GPG key(s) to download from {len(gpg_key_urls)} repos" + ) + + # Download and import each unique key + success = True + for idx, url in enumerate(unique_urls): + key_filename = f"gpg-key-{idx + 1}.asc" + key_path = os.path.join(self.tdir, key_filename) + + try: + logging.info(f"Downloading GPG key from {url}") + with urllib.request.urlopen(url, timeout=30) as response: + key_data = response.read() + + with open(key_path, "wb") as key_file: + key_file.write(key_data) + + logging.info(f"Importing GPG key: {key_filename}") + import_cmd = ["rpm", "--root", self.mname, "--import", key_path] + rc = cmd(import_cmd) + + if rc != 0: + logging.warning(f"Failed to import GPG key from {url}") + success = False + else: + logging.info(f"Successfully imported GPG key from {url}") + + except urllib.error.URLError as e: + logging.warning(f"Failed to download GPG key from {url}: {e}") + success = False + except Exception as e: + logging.warning(f"Error processing GPG key from {url}: {e}") + success = False + + return success + def install_scratch_repos(self, repos, repo_dest): # check if there are repos passed for install if len(repos) == 0: logging.info("REPOS: no repos passed to install\n") + self.gpg_keys_imported = False return + # Try to download and import GPG keys from repos + self.gpg_keys_imported = self.download_and_import_gpg_keys_from_repos(repos) + + if not self.gpg_keys_imported: + logging.warning( + "GPG keys not available - will use --nogpgcheck for package operations" + ) + for r in repos: alias = r['alias'] config = r['config'].lstrip() @@ -47,12 +123,17 @@ def install_scratch_repos(self, repos, repo_dest): def install_scratch_packages(self, packages, registry_loc): # check if there are packages to install if len(packages) == 0: - logging.warn("PACKAGES: no packages passed to install\n") + logging.warning("PACKAGES: no packages passed to install\n") return logging.info(f"PACKAGES: Installing these packages to {self.cname}") logging.info("\n".join(packages)) + # Check if GPG keys were successfully imported during repo setup + use_nogpgcheck = not getattr(self, "gpg_keys_imported", False) + if use_nogpgcheck: + logging.warning("WARNING: Using --nogpgcheck for package installation. GPG keys were not available or failed to download.") + args = [] if self.pkg_man == "zypper": args.append("--non-interactive") @@ -65,10 +146,14 @@ def install_scratch_packages(self, packages, registry_loc): args.append(self.mname) args.append("install") args.append("-l") + if use_nogpgcheck: + args.append("--no-gpg-checks") args.extend(packages) elif self.pkg_man == "dnf": args.append("install") args.append("-y") + if use_nogpgcheck: + args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(packages) @@ -78,24 +163,31 @@ def install_scratch_packages(self, packages, registry_loc): raise Exception("Installing base packages failed") if rc == 107: - logging.warn("one or more RPM postscripts failed to run") + logging.warning("one or more RPM postscripts failed to run") def install_scratch_package_groups(self, package_groups): # check if there are packages groups to install if len(package_groups) == 0: - logging.warn("PACKAGE GROUPS: no package groups passed to install\n") + logging.warning("PACKAGE GROUPS: no package groups passed to install\n") return logging.info(f"PACKAGE GROUPS: Installing these package groups to {self.cname}") logging.info("\n".join(package_groups)) + + # Check if GPG keys were successfully imported during repo setup + use_nogpgcheck = not getattr(self, "gpg_keys_imported", False) + if use_nogpgcheck: + logging.warning("Using --nogpgcheck for package group installation. GPG keys were not available or failed to download.") + args = [] if self.pkg_man == "zypper": - logging.warn("zypper does not support package groups") + logging.warning("zypper does not support package groups") elif self.pkg_man == "dnf": args.append("groupinstall") args.append("-y") - args.append("--nogpgcheck") + if use_nogpgcheck: + args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(package_groups) @@ -107,34 +199,50 @@ def install_scratch_package_groups(self, package_groups): def install_scratch_modules(self, modules): # check if there are modules groups to install if len(modules) == 0: - logging.warn("PACKAGE MODULES: no modules passed to install\n") + logging.warning("PACKAGE MODULES: no modules passed to install\n") return logging.info(f"MODULES: Running these module commands for {self.cname}") for mod_cmd, mod_list in modules.items(): logging.info(mod_cmd + ": " + " ".join(mod_list)) + + # Check if GPG keys were successfully imported during repo setup + use_nogpgcheck = not getattr(self, "gpg_keys_imported", False) + if use_nogpgcheck: + logging.warning("Using --nogpgcheck for module operations. GPG keys were not available or failed to download.") + for mod_cmd, mod_list in modules.items(): args = [] if self.pkg_man == "zypper": - logging.warn("zypper does not support package groups") + logging.warning("zypper does not support package groups") return elif self.pkg_man == "dnf": args.append("module") args.append(mod_cmd) args.append("-y") - args.append("--nogpgcheck") + if use_nogpgcheck: + args.append("--nogpgcheck") args.append("--installroot") args.append(self.mname) args.extend(mod_list) rc = cmd([self.pkg_man] + args) if rc != 0: raise Exception("Failed to run module cmd", mod_cmd, ' '.join(mod_list)) - + def install_repos(self, repos, repo_dest): # check if there are repos passed for install if len(repos) == 0: logging.info("REPOS: no repos passed to install\n") + self.gpg_keys_imported = False return + # Try to download and import GPG keys from repos + self.gpg_keys_imported = self.download_and_import_gpg_keys_from_repos(repos) + + if not self.gpg_keys_imported: + logging.warning( + "GPG keys not available - will use --nogpgcheck for package operations" + ) + logging.info(f"REPOS: Installing these repos to {self.cname}") for r in repos: alias = r['alias'] @@ -149,7 +257,7 @@ def install_repos(self, repos, repo_dest): tmp_path = tmp.name repo_path = Path(*[repo_dest, f'{alias}.repo']) - copy_config = ['buildah', 'copy' , self.cname, tmp_path, repo_path] + copy_config = ['buildah', 'copy', self.cname, tmp_path, repo_path] mkparent_dir = ['buildah', 'run', self.cname, '--', 'mkdir', '-p', repo_dest] try: rc = cmd(mkparent_dir) @@ -166,12 +274,12 @@ def install_repos(self, repos, repo_dest): def install_packages(self, packages): if len(packages) == 0: - logging.warn("PACKAGE GROUPS: no package groups passed to install\n") + logging.warning("PACKAGE GROUPS: no package groups passed to install\n") return logging.info(f"PACKAGES: Installing these packages to {self.cname}") logging.info("\n".join(packages)) args = [self.cname, '--', 'bash', '-c'] - pkg_cmd = [self.pkg_man] + pkg_cmd = [self.pkg_man] if self.gpgcheck is not True: if self.pkg_man == 'dnf': @@ -180,26 +288,26 @@ def install_packages(self, packages): pkg_cmd.append('--no-gpg-checks') pkg_cmd.append('--gpg-auto-import-keys') - args.append(" ".join(pkg_cmd + [ 'install', '-y'] + packages)) - cmd(["buildah","run"] + args) + args.append(" ".join(pkg_cmd + ['install', '-y'] + packages)) + cmd(["buildah", "run"] + args) def install_package_groups(self, package_groups): if len(package_groups) == 0: - logging.warn("PACKAGE GROUPS: no package groups passed to install\n") + logging.warning("PACKAGE GROUPS: no package groups passed to install\n") return logging.info(f"PACKAGES: Installing these package groups to {self.cname}") logging.info("\n".join(package_groups)) args = [self.cname, '--', 'bash', '-c'] pkg_cmd = [self.pkg_man, 'groupinstall', '-y'] if self.pkg_man == "zypper": - logging.warn("zypper does not support package groups") + logging.warning("zypper does not support package groups") args.append(" ".join(pkg_cmd + [f'"{pg}"' for pg in package_groups])) - cmd(["buildah","run"] + args) + cmd(["buildah", "run"] + args) def install_modules(self, modules): # check if there are modules groups to install if len(modules) == 0: - logging.warn("PACKAGE MODULES: no modules passed to install\n") + logging.warning("PACKAGE MODULES: no modules passed to install\n") return logging.info(f"MODULES: Running these module commands for {self.cname}") args = [self.cname, '--', 'bash', '-c'] @@ -208,7 +316,7 @@ def install_modules(self, modules): logging.info(mod_cmd + ": " + " ".join(mod_list)) for mod_cmd, mod_list in modules.items(): if self.pkg_man == "zypper": - logging.warn("zypper does not support package groups") + logging.warning("zypper does not support package groups") return elif self.pkg_man == "dnf": pkg_cmd.append("module") @@ -217,46 +325,46 @@ def install_modules(self, modules): pkg_cmd.append("--nogpgcheck") args.append(" ".join(pkg_cmd + mod_list)) cmd(["buildah", "run"] + args) - + def remove_packages(self, remove_packages): # check if there are packages to remove if len(remove_packages) == 0: - logging.warn("REMOVE PACKAGES: no package passed to remove\n") + logging.warning("REMOVE PACKAGES: no package passed to remove\n") return logging.info(f"REMOVE PACKAGES: removing these packages from container {self.cname}") logging.info("\n".join(remove_packages)) for p in remove_packages: args = [self.cname, '--', 'rpm', '-e', '--nodeps', p] - cmd(["buildah","run"] + args) + cmd(["buildah", "run"] + args) def install_commands(self, commands): # check if there are commands to install if len(commands) == 0: - logging.warn("COMMANDS: no commands passed to run\n") + logging.warning("COMMANDS: no commands passed to run\n") return logging.info(f"COMMANDS: running these commands in {self.cname}") for c in commands: logging.info(c['cmd']) - build_cmd = ["buildah","run"] - if 'buildah_extra_args' in c: - build_cmd.extend(c['buildah_extra_args']) + build_cmd = ['buildah', 'run'] + if "buildah_extra_args" in c: + build_cmd.extend(c["buildah_extra_args"]) args = [self.cname, '--', 'bash', '-c', c['cmd']] if 'loglevel' in c: if c['loglevel'].upper() == "INFO": loglevel = logging.info elif c['loglevel'].upper() == "WARN": - loglevel = logging.warn + loglevel = logging.warning else: loglevel = logging.error else: loglevel = logging.error - cmd(["buildah","run"] + args, stderr_handler=loglevel) + cmd(['buildah', 'run'] + args, stderr_handler=loglevel) def install_copyfiles(self, copyfiles): if len(copyfiles) == 0: - logging.warn("COPYFILES: no files to copy\n") + logging.warning('COPYFILES: no files to copy\n') return logging.info(f"COPYFILES: copying these files to {self.cname}") for f in copyfiles: @@ -265,5 +373,5 @@ def install_copyfiles(self, copyfiles): for o in f['opts']: args.extend(o.split()) logging.info(f['src'] + ' -> ' + f['dest']) - args += [ self.cname, f['src'], f['dest'] ] - cmd(["buildah","copy"] + args) + args += [self.cname, f['src'], f['dest']] + cmd(["buildah", "copy"] + args) diff --git a/src/layer.py b/src/layer.py index f3d6433..4e3002f 100644 --- a/src/layer.py +++ b/src/layer.py @@ -2,6 +2,7 @@ import pathmod import sys import os + # written modules from image_config import ImageConfig from utils import cmd, run_playbook @@ -17,7 +18,17 @@ def __init__(self, args, image_config): self.image_config = image_config self.logger = logging.getLogger(__name__) - def _build_base(self, repos, modules, packages, package_groups, remove_packages, commands, copyfiles, oscap_options): + def _build_base( + self, + repos, + modules, + packages, + package_groups, + remove_packages, + commands, + copyfiles, + oscap_options, + ): # Set local variables dt_string = datetime.now().strftime("%Y%m%d%H%M%S") parent = self.args['parent'] @@ -31,7 +42,12 @@ def buildah_handler(line): # Create a new container from parent out = [] - cmd(["buildah", "from"] + registry_opts_pull + ["--name", container + dt_string, parent], stdout_handler = buildah_handler) + cmd( + ["buildah", "from"] + + registry_opts_pull + + ["--name", container + dt_string, parent], + stdout_handler=buildah_handler, + ) cname = out[0] # Only mount when doing a scratch install @@ -48,7 +64,9 @@ def buildah_handler(line): elif package_manager == "dnf": repo_dest = os.path.expanduser("/etc/imgbuild/yum.repos.d") # Create repo dest, if needed - os.makedirs(os.path.join(mname, pathmod.sep_strip(repo_dest)), exist_ok=True) + os.makedirs( + os.path.join(mname, pathmod.sep_strip(repo_dest)), exist_ok=True + ) # Create dnf.conf file, if needed os.makedirs(os.path.join(mname, "etc/dnf"), exist_ok=True) @@ -86,11 +104,11 @@ def buildah_handler(line): inst = installer.Installer(package_manager, cname, mname) except Exception as e: self.logger.error(f"Error preparing installer: {e}") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") except KeyboardInterrupt: self.logger.error(f"Keyboard Interrupt") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") # Install Repos @@ -101,11 +119,11 @@ def buildah_handler(line): inst.install_repos(repos, repo_dest) except Exception as e: self.logger.error(f"Error installing repos: {e}") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") except KeyboardInterrupt: self.logger.error(f"Keyboard Interrupt") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") # Install Packages @@ -125,11 +143,11 @@ def buildah_handler(line): inst.remove_packages(remove_packages) except Exception as e: self.logger.error(f"Error installing packages: {e}") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") except KeyboardInterrupt: self.logger.error(f"Keyboard Interrupt") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") # Copy Files @@ -137,30 +155,30 @@ def buildah_handler(line): inst.install_copyfiles(copyfiles) except Exception as e: self.logger.error(f"Error running commands: {e}") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now") except KeyboardInterrupt: self.logger.error(f"Keyboard Interrupt") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") # Run Commands try: inst.install_commands(commands) - if os.path.islink(mname + '/etc/resolv.conf'): + if os.path.islink(mname + "/etc/resolv.conf"): self.logger.info("removing resolv.conf link (this link breaks running a container)") - os.unlink(mname + '/etc/resolv.conf') + os.unlink(mname + "/etc/resolv.conf") except Exception as e: self.logger.error(f"Error running commands: {e}") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now") except KeyboardInterrupt: self.logger.error(f"Keyboard Interrupt") - cmd(["buildah","rm"] + [cname]) + cmd(["buildah", "rm"] + [cname]) sys.exit("Exiting now ...") - - # OpenSCAP - if self.args['install_scap'] or self.args['scap_benchmark'] or self.args['oval_eval']: + + # OpenSCAP + if self.args["install_scap"] or self.args["scap_benchmark"] or self.args["oval_eval"]: oscap = Oscap(oscap_options, self.args, inst) if self.args['install_scap']: oscap.install_scap() @@ -172,35 +190,49 @@ def buildah_handler(line): return cname - def _build_ansible(self, target, parent, ansible_groups, ansible_pb, ansible_inv, ansible_vars, ansible_verbosity): + def _build_ansible( + self, + target, + parent, + ansible_groups, + ansible_pb, + ansible_inv, + ansible_vars, + ansible_verbosity, + ): cnames = {} + def buildah_handler(line): out.append(line) out = [] - cmd(["buildah","from"] + self.args['registry_opts_pull'] + ["--name", target, parent], stdout_handler = buildah_handler) + cmd( + ["buildah", "from"] + + self.args["registry_opts_pull"] + + ["--name", target, parent], + stdout_handler=buildah_handler, + ) container_name = out[0] - cnames[container_name] = { - 'ansible_groups': ansible_groups, - 'ansible_pb': ansible_pb, - 'ansible_vars': ansible_vars - } + cnames[container_name] = { + 'ansible_groups': ansible_groups, + 'ansible_pb': ansible_pb, + 'ansible_vars': ansible_vars, + } try: pb_res = run_playbook(cnames, ansible_inv, ansible_verbosity) except Exception as e: self.logger.error(e) - cmd(["buildah","rm"] + [target]) + cmd(["buildah", "rm"] + [target]) self.logger.error("Exiting Now...") sys.exit(1) return container_name def build_layer(self): - print("BUILD LAYER".center(50, '-')) + print("BUILD LAYER".center(50, "-")) - if self.args['layer_type'] == "base": - + if self.args["layer_type"] == "base": repos = self.image_config.get_repos() modules = self.image_config.get_modules() packages = self.image_config.get_packages() @@ -210,7 +242,16 @@ def build_layer(self): copyfiles = self.image_config.get_copy_files() oscap_options = self.image_config.get_oscap_options() - cname = self._build_base(repos, modules, packages, package_groups, remove_packages, commands, copyfiles, oscap_options) + cname = self._build_base( + repos, + modules, + packages, + package_groups, + remove_packages, + commands, + copyfiles, + oscap_options, + ) elif self.args['layer_type'] == "ansible": layer_name = self.args['name'] print("Layer_Name =", layer_name) @@ -221,11 +262,19 @@ def build_layer(self): ansible_vars = self.args['ansible_vars'] ansible_verbosity = self.args['ansible_verbosity'] - cname = self._build_ansible(layer_name, parent, ansible_groups, ansible_pb, ansible_inv, ansible_vars, ansible_verbosity) + cname = self._build_ansible( + layer_name, + parent, + ansible_groups, + ansible_pb, + ansible_inv, + ansible_vars, + ansible_verbosity, + ) else: self.logger.error("Unrecognized layer type") sys.exit("Exiting now ...") - + # Publish the layer self.logger.info("Publishing Layer") publish(cname, self.args) diff --git a/tests/dnf/rocky9_scratch.yaml b/tests/dnf/rocky9_scratch.yaml index b7a3961..d3f00b2 100644 --- a/tests/dnf/rocky9_scratch.yaml +++ b/tests/dnf/rocky9_scratch.yaml @@ -10,11 +10,17 @@ options: repos: - alias: 'Rocky_9_BaseOS' - url: 'https://dl.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' + config: | + baseurl=https://dl.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' - alias: 'Rocky_9_AppStream' - url: 'https://dl.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/' - gpg: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' + config: | + baseurl=https://dl.rockylinux.org/pub/rocky/9/AppStream/x86_64/os/ + enabled=1 + gpgcheck=1 + gpg_key: 'https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9' package_groups: - 'Minimal Install'