cbsbuild run local

.gitignore

@@ -10,3 +10,7 @@ node_modules
 package.json
 yarn.lock
 .venv/
+!tests/resources/cbscore/cbs-build.config.yaml
+.coverage
+reports/
+coverage.xml

cbscore/src/cbscore/__main__.py

@@ -35,6 +35,12 @@
 
 
 @click.group()
+@click.option(
+    "--dev",
+    help="Run in development mode. Only used for testing.",
+    is_flag=True,
+    envvar="CBS_DEV",
+)
 @click.option(
     "-d", "--debug", help="Enable debug output", is_flag=True, envvar="CBS_DEBUG"
 )
@@ -54,12 +60,23 @@
     required=True,
     default="cbs-build.config.yaml",
 )
+@click.option(
+    "-l",
+    "--local",
+    "local",
+    is_flag=True,
+    default=False,
+    required=False,
+    help="Run without access to s3",
+)
 @pass_ctx
-def cmd_main(ctx: Ctx, debug: bool, config_path: Path) -> None:
+def cmd_main(ctx: Ctx, dev: bool, debug: bool, config_path: Path, local: bool) -> None:
     if debug:
         set_log_level(logging.DEBUG)
 
     ctx.config_path = config_path
+    ctx.local = local
+    ctx.dev = dev
 
 
 cmd_main.add_command(builds.cmd_build)

cbscore/src/cbscore/_tools/cbscore-entrypoint.sh

@@ -53,7 +53,15 @@ uv --directory "${CBSCORE_PATH}" \
 
 dbg=
 [[ -n ${CBS_DEBUG} ]] && [[ ${CBS_DEBUG} == "1" ]] && dbg="--debug"
+
+local_run=
+[[ -n ${CBS_LOCAL} ]] && [[ ${CBS_LOCAL} == "1" ]] && local_run="--local"
+
+dev=
+[[ -n ${CBS_DEV} ]] && [[ ${CBS_DEV} == "1" ]] && dev="--dev"
+
+
 # shellcheck disable=2048,SC2086
-cbsbuild --config "${RUNNER_PATH}/cbs-build.config.yaml" ${dbg} \
+cbsbuild --config "${RUNNER_PATH}/cbs-build.config.yaml" ${dbg} ${local_run} ${dev} \
   runner build \
   $* || exit 1

cbscore/src/cbscore/builder/builder.py

@@ -11,6 +11,8 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 
+import asyncio
+import shutil
 from pathlib import Path
 
 from cbscore.builder import BuilderError
@@ -30,8 +32,10 @@
 from cbscore.images.skopeo import skopeo_image_exists
 from cbscore.releases import ReleaseError
 from cbscore.releases.desc import (
+    S3_BASE_URL,
     ArchType,
     BuildType,
+    LocalReleaseRPMArtifacts,
     ReleaseBuildEntry,
     ReleaseComponent,
     ReleaseComponentVersion,
@@ -45,6 +49,7 @@
     release_upload_components,
 )
 from cbscore.releases.utils import get_component_release_rpm
+from cbscore.utils import CommandError, async_run_cmd
 from cbscore.utils.containers import get_container_canonical_uri
 from cbscore.utils.secrets import SecretsMgrError
 from cbscore.utils.secrets.mgr import SecretsMgr
@@ -65,6 +70,9 @@ class Builder:
     skip_build: bool
     force: bool
     tls_verify: bool
+    local: bool
+    base_url: str
+    dev: bool
 
     def __init__(
         self,
@@ -73,7 +81,9 @@ def __init__(
         *,
         skip_build: bool = False,
         force: bool = False,
-        tls_verify: bool = True
+        tls_verify: bool = True,
+        local: bool = False,
+        dev: bool = False,
     ) -> None:
         self.desc = desc
         self.config = config
@@ -85,6 +95,9 @@ def __init__(
         self.skip_build = skip_build
         self.force = force
         self.tls_verify = tls_verify
+        self.local = local
+        self.base_url = "file://" if self.local else S3_BASE_URL
+        self.dev = dev
 
         try:
             vault_config = self.config.get_vault_config()
@@ -112,7 +125,9 @@ async def run(self) -> None:
             raise BuilderError(msg=msg) from e
 
         container_img_uri = get_container_canonical_uri(self.desc)
-        if skopeo_image_exists(container_img_uri, self.secrets, tls_verify=self.tls_verify):
+        if skopeo_image_exists(
+            container_img_uri, self.secrets, tls_verify=self.tls_verify
+        ):
             logger.info(f"image '{container_img_uri}' already exists -- do not build!")
             return
         else:
@@ -166,7 +181,9 @@ async def run(self) -> None:
             return
 
         try:
-            ctr_builder = ContainerBuilder(self.desc, release_desc, self.components)
+            ctr_builder = ContainerBuilder(
+                self.desc, release_desc, self.components, tls_verify=self.tls_verify
+            )
             await ctr_builder.build()
             await ctr_builder.finish(
                 self.secrets,
@@ -204,6 +221,7 @@ async def _build_release(self) -> ReleaseDesc | None:
                 self.components,
                 self.desc.components,
                 self.desc.version,
+                dev=self.dev,
             ) as components:
                 return await self._do_build_release(components)
         except BuilderError as e:
@@ -227,6 +245,10 @@ async def _do_build_release(
 
         Will return `None` if `self.upload` is `False`.
         """
+        if not self.local and (not self.storage_config or not self.storage_config.s3):
+            logger.warning("not uploading per config, stop release build")
+            return None
+
         # Check if any of the components have been previously built, and, if so,
         # reuse them instead of building them.
         #
@@ -279,10 +301,6 @@ async def _do_build_release(
                 logger.error(msg)
                 raise BuilderError(msg) from e
 
-        if not self.storage_config or not self.storage_config.s3:
-            logger.warning("not uploading per config, stop release build")
-            return None
-
         comp_versions = existing.copy()
         comp_versions.update(built)
 
@@ -302,7 +320,13 @@ async def _do_build_release(
         )
 
         release: ReleaseDesc | None = None
-        if self.storage_config and self.storage_config.s3:
+        if self.local:
+            release = ReleaseDesc(
+                version=self.desc.version,
+                builds={release_build.arch: release_build},
+                base_url=self.base_url,
+            )
+        elif self.storage_config and self.storage_config.s3:
             try:
                 release = await release_desc_upload(
                     self.secrets,
@@ -342,6 +366,23 @@ async def _build(
             logger.error(msg)
             raise BuilderError(msg) from e
 
+        await self._create_repos(comp_builds)
+
+        if self.local:
+            return {
+                comp_name: ReleaseComponentVersion(
+                    name=components[comp_name].name,
+                    version=components[comp_name].long_version,
+                    sha1=components[comp_name].sha1,
+                    arch=ArchType.x86_64,
+                    build_type=BuildType.rpm,
+                    os_version=f"el{self.desc.el_version}",
+                    repo_url=components[comp_name].repo_url,
+                    artifacts=LocalReleaseRPMArtifacts(loc=comp_build.rpms_path),
+                )
+                for comp_name, comp_build in comp_builds.items()
+            }
+
         if not self.storage_config or not self.storage_config.s3:
             return {}
 
@@ -370,7 +411,7 @@ async def _build_rpms(
 
         rpms_path = self.scratch_path.joinpath("rpms")
         rpms_path.mkdir(exist_ok=True)
-
+        is_sign_rpms = (self.signing_config and self.signing_config.gpg) is not None
         try:
             comp_builds = await build_rpms(
                 rpms_path,
@@ -379,13 +420,15 @@ async def _build_rpms(
                 components,
                 ccache_path=self.ccache_path,
                 skip_build=self.skip_build,
+                base_url=self.base_url,
+                is_sign_rpms=is_sign_rpms,
             )
         except (BuilderError, Exception) as e:
             msg = f"error building components ({components.keys()}): {e}"
             logger.error(msg)
             raise BuilderError(msg) from e
 
-        if not self.signing_config or not self.signing_config.gpg:
+        if not is_sign_rpms:
             logger.warning("no signing method provided, skip signing RPMs")
         else:
             logger.info(f"signing RPMs with gpg key '{self.signing_config.gpg}'")
@@ -402,6 +445,40 @@ async def _build_rpms(
 
         return comp_builds
 
+    async def _create_repos(self, comp_builds: dict[str, ComponentBuild]) -> None:
+        logger.info(f"creating rpm repos for {comp_builds.keys()}")
+        async with asyncio.TaskGroup() as tg:
+            for comp_name, comp_build in comp_builds.items():
+                logger.info(f"creating rpm repo for {comp_name}")
+                _ = tg.create_task(self._create_repo(comp_build))
+
+    async def _create_repo(self, comp_build: ComponentBuild) -> None:
+        async def _do_create_repo(p: Path) -> None:
+            repodata_path = p.joinpath("repodata")
+            if repodata_path.exists():
+                shutil.rmtree(repodata_path)
+
+            try:
+                _ = await async_run_cmd(["createrepo", p.resolve().as_posix()])
+            except CommandError as e:
+                msg = f"error creating repodata at '{repodata_path}': {e}"
+                logger.exception(msg)
+                raise BuilderError(msg) from e
+            except Exception as e:
+                msg = f"unknown error creating repodata at '{repodata_path}': {e}"
+                logger.exception(msg)
+                raise BuilderError(msg) from e
+
+            if not repodata_path.exists() or not repodata_path.is_dir():
+                msg = f"unexpected missing repodata dir at '{repodata_path}'"
+                logger.error(msg)
+                raise BuilderError(msg)
+
+        rpm_folders = {f.parent for f in comp_build.rpms_path.rglob("*.rpm")}
+
+        for p in rpm_folders:
+            await _do_create_repo(p)
+
     async def _upload(
         self,
         comp_infos: dict[str, BuildComponentInfo],

cbscore/src/cbscore/builder/prepare.py

@@ -115,7 +115,7 @@ async def _cb(s: str) -> None:
         )
         logger.debug(stdout)
         if rc == 2 and re.match(".*already installed.*", stderr):
-            msg = f'skip install cosign. allready installed'
+            msg = "skip install cosign. already installed"
             logger.debug(msg)
         elif rc != 0:
             msg = f"error installing cosign package: {stderr}"
@@ -182,6 +182,8 @@ async def prepare_components(
     components_loc: dict[str, CoreComponentLoc],
     components: list[VersionComponent],
     version: str,
+    *,
+    dev: bool,
 ) -> AsyncGenerator[dict[str, BuildComponentInfo]]:
     """
     Prepare all components by cloning them and applying required patches.
@@ -217,7 +219,7 @@ async def _clone_repo(comp: VersionComponent) -> Path:
         )
         start = dt.now(tz=datetime.UTC)
         try:
-            with secrets.git_url_for(comp.repo) as comp_url:
+            with secrets.git_url_for(comp.repo, dev=dev) as comp_url:
                 cloned_path = await git.git_clone(
                     comp_url,
                     git_repos_path,

cbscore/src/cbscore/builder/rpmbuild.py

@@ -60,6 +60,8 @@ async def _build_component(
     *,
     ccache_path: Path | None = None,
     skip_build: bool = False,
+    base_url: str = "",
+    is_sign_rpms: bool = True,
 ) -> tuple[int, Path]:
     """
     Build a given component.
@@ -88,8 +90,9 @@ async def _outcb(s: str) -> None:
         comp_rpms_path.resolve().as_posix(),
     ]
 
-    if version:
-        cmd.append(version)
+    cmd.append(version if version else '""')
+    cmd.append(base_url if base_url else '""')
+    cmd.append("1" if is_sign_rpms else "0")
 
     extra_env: dict[str, str] | None = None
     if ccache_path is not None:
@@ -177,6 +180,8 @@ async def build_rpms(
     *,
     ccache_path: Path | None = None,
     skip_build: bool = False,
+    base_url: str = "",
+    is_sign_rpms: bool = True,
 ) -> dict[str, ComponentBuild]:
     """
     Build RPMs for the various components provided in `components`.
@@ -245,6 +250,8 @@ def __init__(self, build_script: Path, version: str) -> None:
                         to_build[name].version,
                         ccache_path=ccache_path,
                         skip_build=skip_build,
+                        base_url=base_url,
+                        is_sign_rpms=is_sign_rpms,
                     )
                 )
                 for name in to_build