From 4898284a34e4f56b93a348541936138d40144e06 Mon Sep 17 00:00:00 2001 From: Abbas Jafari Date: Mon, 11 May 2026 21:07:18 +0200 Subject: [PATCH 1/2] feat(marketplace): support host-prefixed source in apm.yml Allow apm.yml package entries to specify a non-default git host inline via the 'host.tld/owner/repo' source form, removing the need to set GITHUB_HOST when building marketplaces against GitHub Enterprise or other self-hosted git servers. - yml_schema: extend SOURCE_RE to accept 3-segment host-prefixed form, add split_host_from_source() helper, add 'host' field on PackageEntry populated by the loader. - builder: thread host through ResolvedPackage; per-host RefResolver cache (pre-warmed before worker dispatch to avoid races); emit full https://host/owner/repo URLs in the generated marketplace.json so Claude Code consumers can clone from the correct host. Non-subdir packages on a non-default host emit a 'url' source instead of the github shorthand which is hard-coded to github.com. --- src/apm_cli/marketplace/builder.py | 57 ++++++++++++++++++++++++++- src/apm_cli/marketplace/yml_schema.py | 34 ++++++++++++++-- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/src/apm_cli/marketplace/builder.py b/src/apm_cli/marketplace/builder.py index 2d411ac3f..1dbbd838b 100644 --- a/src/apm_cli/marketplace/builder.py +++ b/src/apm_cli/marketplace/builder.py @@ -84,6 +84,7 @@ class ResolvedPackage: requested_version: str | None # original APM-only range (for diagnostics) tags: tuple[str, ...] is_prerelease: bool # True if the resolved ref was a prerelease semver + host: str | None = None # non-default git host parsed from apm.yml source @dataclass(frozen=True) @@ -227,6 +228,9 @@ def __init__( self._host: str = default_host() or "github.com" self._host_info: HostInfo | None = None self._auth_resolved: bool = False + # Per-host RefResolver cache for entries that override the host via + # the ``host.tld/owner/repo`` source form in apm.yml. + self._host_resolvers: dict[str, RefResolver] = {} @classmethod def from_config( @@ -280,6 +284,31 @@ def _get_resolver(self) -> RefResolver: ) return self._resolver + def _get_resolver_for_host(self, host: str | None) -> RefResolver: + """Return a RefResolver bound to *host* (default when ``None``). + + Per-host resolvers are cached for the lifetime of the build so each + unique host pays the auth-resolution cost only once. + """ + if host is None or host == self._host: + return self._get_resolver() + cached = self._host_resolvers.get(host) + if cached is not None: + return cached + self._ensure_auth() + # Reuse the resolved token only when the override host matches the + # default host class; otherwise leave token unset and rely on + # ambient git credentials (SSH key / git credential helper). + token = self._github_token if host == self._host else None + resolver = RefResolver( + timeout_seconds=self._options.timeout_seconds, + offline=self._options.offline, + host=host, + token=token, + ) + self._host_resolvers[host] = resolver + return resolver + def _ensure_auth(self) -> None: """Lazily resolve host classification and GitHub token. @@ -325,7 +354,7 @@ def _resolve_entry(self, entry: PackageEntry) -> ResolvedPackage: is_prerelease=False, ) yml = self._load_yml() - resolver = self._get_resolver() + resolver = self._get_resolver_for_host(entry.host) owner_repo = entry.source if entry.ref is not None: @@ -355,6 +384,7 @@ def _resolve_explicit_ref( requested_version=entry.version, tags=entry.tags, is_prerelease=sv.is_prerelease if sv else False, + host=entry.host, ) refs = resolver.list_remote_refs(owner_repo) @@ -375,6 +405,7 @@ def _resolve_explicit_ref( requested_version=entry.version, tags=entry.tags, is_prerelease=sv.is_prerelease if sv else False, + host=entry.host, ) # Try as full refname @@ -394,6 +425,7 @@ def _resolve_explicit_ref( requested_version=entry.version, tags=entry.tags, is_prerelease=sv.is_prerelease if sv else False, + host=entry.host, ) # Try as branch name @@ -410,6 +442,7 @@ def _resolve_explicit_ref( requested_version=entry.version, tags=entry.tags, is_prerelease=False, + host=entry.host, ) # HEAD special case @@ -479,6 +512,7 @@ def _resolve_version_range( requested_version=version_range, tags=entry.tags, is_prerelease=best_sv.is_prerelease, + host=entry.host, ) # -- concurrent resolution ---------------------------------------------- @@ -508,6 +542,12 @@ def resolve(self) -> ResolveResult: # spawning workers -- avoids a race on _ensure_auth() and # matches the pattern used in _prefetch_metadata(). self._get_resolver() + # Pre-warm any per-host resolvers needed by entries that override the + # default host via the ``host.tld/owner/repo`` source form. Done on + # the main thread so workers never race to create the same resolver. + for entry in entries: + if entry.host: + self._get_resolver_for_host(entry.host) with ThreadPoolExecutor(max_workers=min(self._options.concurrency, len(entries))) as pool: future_to_index = { @@ -858,11 +898,24 @@ def compose_marketplace_json(self, resolved: list[ResolvedPackage]) -> dict[str, # Subdirs use the ``git-subdir`` form; everything else uses # ``github`` shorthand. Field names: ``source``/``repo``/``sha`` # (NOT ``type``/``repository``/``commit``). + # + # When the package was authored with a host-prefixed source + # (``host.tld/owner/repo``), emit a real ``https://`` URL so + # Claude Code can clone from a non-default host (e.g. GHE). source_obj: dict[str, Any] = OrderedDict() if pkg.subdir: source_obj["source"] = "git-subdir" - source_obj["url"] = pkg.source_repo + if pkg.host: + source_obj["url"] = f"https://{pkg.host}/{pkg.source_repo}" + else: + source_obj["url"] = pkg.source_repo source_obj["path"] = pkg.subdir + elif pkg.host: + # Non-default host without subdir: ``github`` shorthand + # only resolves to github.com, so emit a ``url`` source + # with a full clone URL instead. + source_obj["source"] = "url" + source_obj["url"] = f"https://{pkg.host}/{pkg.source_repo}" else: source_obj["source"] = "github" source_obj["repo"] = pkg.source_repo diff --git a/src/apm_cli/marketplace/yml_schema.py b/src/apm_cli/marketplace/yml_schema.py index dd568386b..cc705cd25 100644 --- a/src/apm_cli/marketplace/yml_schema.py +++ b/src/apm_cli/marketplace/yml_schema.py @@ -65,11 +65,27 @@ r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$" ) -# Source field accepts either ``owner/repo`` (remote) or ``./...`` (local -# path within the same repo). Used by both yml_schema and yml_editor for -# source field validation. -SOURCE_RE = re.compile(r"^(?:[^/]+/[^/]+|\./.*)$") +# Source field accepts: +# - ``owner/repo`` (remote, default host) +# - ``host.tld/owner/repo`` (remote on a non-default host -- the first +# segment must look like a hostname, i.e. contain a dot) +# - ``./...`` (local path within the same repo) +# Used by both yml_schema and yml_editor for source field validation. +SOURCE_RE = re.compile(r"^(?:[^/\s]+\.[^/\s]+/[^/\s]+/[^/\s]+|[^/\s]+/[^/\s]+|\./.*)$") LOCAL_SOURCE_RE = re.compile(r"^\./") +# Matches ``host.tld/owner/repo`` (3 segments, first is FQDN-ish). +_HOST_PREFIXED_SOURCE_RE = re.compile(r"^([^/\s]+\.[^/\s]+)/([^/\s]+/[^/\s]+)$") + + +def split_host_from_source(source: str) -> tuple[str | None, str]: + """Split ``host.tld/owner/repo`` into ``(host, owner/repo)``. + + Returns ``(None, source)`` for ``owner/repo`` or local ``./...`` forms. + """ + m = _HOST_PREFIXED_SOURCE_RE.match(source) + if m: + return m.group(1), m.group(2) + return None, source # Placeholder tokens accepted in ``tag_pattern`` / ``build.tagPattern``. _TAG_PLACEHOLDERS = ("{version}", "{name}") @@ -224,6 +240,10 @@ class PackageEntry: repository: str | None = None # Derived (set by loader, not by user) is_local: bool = False + # Optional non-default git host parsed from ``source`` of the form + # ``host.tld/owner/repo``. ``None`` means use the default host + # (``GITHUB_HOST`` env or ``github.com``). + host: str | None = None @dataclass(frozen=True) @@ -385,6 +405,11 @@ def _parse_package_entry(raw: Any, index: int) -> PackageEntry: source = _require_str(raw, "source", context=f"packages[{index}]") _validate_source(source, index=index) is_local = bool(LOCAL_SOURCE_RE.match(source)) + # Detect host-prefixed source (e.g. ``host.tld/owner/repo``) and split + # the host off so downstream consumers continue to see ``owner/repo``. + host: str | None = None + if not is_local: + host, source = split_host_from_source(source) # APM-only: subdir (irrelevant for local packages but harmless) subdir: str | None = raw.get("subdir") @@ -527,6 +552,7 @@ def _parse_package_entry(raw: Any, index: int) -> PackageEntry: license=license_val, repository=repository, is_local=is_local, + host=host, ) From 81d5d63fc507581bc8cc95618f115024fc2e51f0 Mon Sep 17 00:00:00 2001 From: Abbas Jafari Date: Tue, 12 May 2026 08:56:48 +0200 Subject: [PATCH 2/2] test(marketplace): cover host-prefixed source forms; publisher accepts apm.yml - test_yml_schema: add TestHostPrefixedSource covering local / default-host / host-prefixed source forms, including subdir + host combinations and rejection of malformed 4-segment / dotless 3-segment paths. - test_builder: add emitter cases verifying github shorthand for default host, source:url for host-prefixed without subdir, git-subdir+url for host-prefixed with subdir, plus per-host RefResolver isolation. Adds _build_with_host_mock helper that stubs _get_resolver_for_host so non-default hosts resolve through the in-memory mock. - publisher: route through load_marketplace_config so 'apm publish' works with the apm.yml form, not only the legacy marketplace.yml. --- src/apm_cli/marketplace/publisher.py | 8 +- tests/unit/marketplace/test_builder.py | 162 ++++++++++++++++++++++ tests/unit/marketplace/test_yml_schema.py | 129 +++++++++++++++++ 3 files changed, 295 insertions(+), 4 deletions(-) diff --git a/src/apm_cli/marketplace/publisher.py b/src/apm_cli/marketplace/publisher.py index d4de9cb41..112c7bd2c 100644 --- a/src/apm_cli/marketplace/publisher.py +++ b/src/apm_cli/marketplace/publisher.py @@ -53,7 +53,8 @@ from .resolver import parse_marketplace_ref from .semver import parse_semver from .tag_pattern import render_tag -from .yml_schema import load_marketplace_yml +from .migration import load_marketplace_config +from .yml_schema import load_marketplace_yml # noqa: F401 — kept for back-compat logger = logging.getLogger(__name__) @@ -314,10 +315,9 @@ def __init__( self._yml = None def _load_yml(self): - """Lazy-load marketplace.yml.""" + """Lazy-load marketplace config (apm.yml or legacy marketplace.yml).""" if self._yml is None: - yml_path = self._root / "marketplace.yml" - self._yml = load_marketplace_yml(yml_path) + self._yml = load_marketplace_config(self._root) return self._yml # -- plan --------------------------------------------------------------- diff --git a/tests/unit/marketplace/test_builder.py b/tests/unit/marketplace/test_builder.py index cf23f2573..f98e7d3f5 100644 --- a/tests/unit/marketplace/test_builder.py +++ b/tests/unit/marketplace/test_builder.py @@ -133,6 +133,28 @@ def _build_with_mock( return builder.build() +def _build_with_host_mock( + tmp_path: Path, + yml_content: str, + refs_by_remote: dict[str, list[RemoteRef]], + options: BuildOptions | None = None, +) -> BuildReport: + """Build with a mock resolver that handles default *and* host-prefixed entries. + + The standard ``_build_with_mock`` only patches ``_resolver`` (the default + resolver). Host-prefixed entries trigger ``_get_resolver_for_host`` which + constructs a real ``RefResolver`` bound to the override host. For unit + tests we want every host to resolve through the same in-memory mock. + """ + yml_path = _write_yml(tmp_path, yml_content) + opts = options or BuildOptions(offline=True) + builder = MarketplaceBuilder(yml_path, opts) + mock = _MockRefResolver(refs_by_remote) + builder._resolver = mock # type: ignore[assignment] + builder._get_resolver_for_host = lambda _host: mock # type: ignore[assignment] + return builder.build() + + # --------------------------------------------------------------------------- # parse_semver # --------------------------------------------------------------------------- @@ -910,6 +932,146 @@ def test_no_subdir_no_path(self, tmp_path: Path) -> None: cr = data["plugins"][0] assert "path" not in cr["source"] + # -- default-host (``owner/repo``) ------------------------------------ + + def test_default_host_emits_github_shorthand(self, tmp_path: Path) -> None: + """A plain ``owner/repo`` source emits the ``github`` shorthand form.""" + refs = {"acme/code-reviewer": _make_refs("v2.0.0")} + yml = """\ +name: acme-tools +description: Test +version: 1.0.0 +owner: + name: Acme + email: t@acme.example.com + url: https://acme.example.com +packages: + - name: code-reviewer + source: acme/code-reviewer + version: "^2.0.0" +""" + report = _build_with_mock(tmp_path, yml, refs) + data = json.loads(report.output_path.read_text("utf-8")) + src = data["plugins"][0]["source"] + assert src["source"] == "github" + assert src["repo"] == "acme/code-reviewer" + assert "url" not in src + # The github shorthand never carries a ``path`` key. + assert "path" not in src + + # -- host-prefixed sources -------------------------------------------- + + def test_host_prefixed_without_subdir_emits_url_source(self, tmp_path: Path) -> None: + """``host.tld/owner/repo`` (no subdir) emits a full URL via ``source: url``.""" + refs = {"acme/agents": _make_refs("v0.3.0")} + yml = """\ +name: ghe-tools +description: Test +version: 1.0.0 +owner: + name: Acme + email: t@acme.example.com + url: https://acme.example.com +packages: + - name: ch-baseline + source: ghe.example.com/acme/agents + ref: v0.3.0 +""" + report = _build_with_host_mock(tmp_path, yml, refs) + data = json.loads(report.output_path.read_text("utf-8")) + src = data["plugins"][0]["source"] + assert src["source"] == "url" + assert src["url"] == "https://ghe.example.com/acme/agents" + assert "repo" not in src + assert "path" not in src + + def test_host_prefixed_with_subdir_emits_git_subdir_url(self, tmp_path: Path) -> None: + """``host.tld/owner/repo`` + subdir emits ``git-subdir`` with a full https URL.""" + refs = {"acme/agents": _make_refs("v0.3.0")} + yml = """\ +name: ghe-tools +description: Test +version: 1.0.0 +owner: + name: Acme + email: t@acme.example.com + url: https://acme.example.com +packages: + - name: ch-baseline + source: ghe.example.com/acme/agents + subdir: packages/ch-baseline + ref: v0.3.0 +""" + report = _build_with_host_mock(tmp_path, yml, refs) + data = json.loads(report.output_path.read_text("utf-8")) + src = data["plugins"][0]["source"] + assert src["source"] == "git-subdir" + assert src["url"] == "https://ghe.example.com/acme/agents" + assert src["path"] == "packages/ch-baseline" + + def test_default_host_with_subdir_emits_shorthand_url(self, tmp_path: Path) -> None: + """``owner/repo`` + subdir keeps the historical ``url: owner/repo`` shape.""" + refs = {"acme/test-generator": _make_refs("v1.0.0")} + yml = """\ +name: acme-tools +description: Test +version: 1.0.0 +owner: + name: Acme + email: t@acme.example.com + url: https://acme.example.com +packages: + - name: test-generator + source: acme/test-generator + version: "~1.0.0" + subdir: src/plugin +""" + report = _build_with_mock(tmp_path, yml, refs) + data = json.loads(report.output_path.read_text("utf-8")) + src = data["plugins"][0]["source"] + assert src["source"] == "git-subdir" + # Default host: keep the shorthand "owner/repo" (no scheme), preserving + # backwards compatibility with marketplaces emitted before host-prefix + # support landed. + assert src["url"] == "acme/test-generator" + assert src["path"] == "src/plugin" + + def test_per_host_resolvers_isolated(self, tmp_path: Path) -> None: + """Two entries on different hosts each get their own resolver instance.""" + refs = { + "acme/code-reviewer": _make_refs("v2.0.0"), + "team/repo": _make_refs("v1.0.0"), + } + yml = """\ +name: mixed-tools +description: Test +version: 1.0.0 +owner: + name: Mixed + email: t@mixed.example.com + url: https://mixed.example.com +packages: + - name: code-reviewer + source: acme/code-reviewer + version: "^2.0.0" + - name: gitlab-tool + source: gitlab.example.org/team/repo + ref: v1.0.0 +""" + yml_path = _write_yml(tmp_path, yml) + builder = MarketplaceBuilder(yml_path, BuildOptions(offline=True)) + mock = _MockRefResolver(refs) + builder._resolver = mock # type: ignore[assignment] + # Force per-host resolver lookups to reuse the same mock so the + # builder never tries to talk to a real remote. + builder._get_resolver_for_host = lambda _host: mock # type: ignore[assignment] + builder.build() + # Calls to _get_resolver_for_host return the mock for every host, + # but the cache (_host_resolvers) is only written by the original + # implementation -- assert the mock was used by checking refs were + # resolved successfully (build() succeeded without OfflineMissError). + assert mock is builder._get_resolver_for_host("gitlab.example.org") + # --------------------------------------------------------------------------- # Deterministic output (round-trip) diff --git a/tests/unit/marketplace/test_yml_schema.py b/tests/unit/marketplace/test_yml_schema.py index 8cbafff01..d021dc6a2 100644 --- a/tests/unit/marketplace/test_yml_schema.py +++ b/tests/unit/marketplace/test_yml_schema.py @@ -650,6 +650,135 @@ def test_build_default_tag_pattern(self, tmp_path: Path): assert result.build.tag_pattern == "v{version}" +# --------------------------------------------------------------------------- +# Host-prefixed source form (``host.tld/owner/repo``) +# --------------------------------------------------------------------------- + + +class TestHostPrefixedSource: + """``source: host.tld/owner/repo`` splits the host off into ``PackageEntry.host``.""" + + def test_default_owner_repo_has_no_host(self, tmp_path: Path): + """Plain ``owner/repo`` source leaves ``host`` as None (use default).""" + content = _minimal_yml( + packages=( + "packages:\n" + " - name: tool-a\n" + " source: acme/tool-a\n" + " ref: v1.0.0" + ) + ) + yml = _write_yml(tmp_path, content) + result = load_marketplace_yml(yml) + entry = result.packages[0] + assert entry.source == "acme/tool-a" + assert entry.host is None + assert entry.is_local is False + + def test_local_source_has_no_host(self, tmp_path: Path): + """``./path`` local sources never get a host.""" + content = _minimal_yml(packages=("packages:\n - name: tool-a\n source: ./acme")) + yml = _write_yml(tmp_path, content) + result = load_marketplace_yml(yml) + entry = result.packages[0] + assert entry.is_local is True + assert entry.host is None + + def test_ghe_host_prefixed_source_split(self, tmp_path: Path): + """``host.tld/owner/repo`` splits host out and leaves ``owner/repo``.""" + content = _minimal_yml( + packages=( + "packages:\n" + " - name: tool-a\n" + " source: ghe.example.com/acme/agents\n" + " ref: v0.3.0" + ) + ) + yml = _write_yml(tmp_path, content) + result = load_marketplace_yml(yml) + entry = result.packages[0] + assert entry.source == "acme/agents" + assert entry.host == "ghe.example.com" + + def test_github_com_host_prefix_accepted(self, tmp_path: Path): + """The default host can also be expressed explicitly as a host prefix.""" + content = _minimal_yml( + packages=( + "packages:\n" + " - name: tool-a\n" + " source: github.com/acme/tool-a\n" + " ref: v1.0.0" + ) + ) + yml = _write_yml(tmp_path, content) + result = load_marketplace_yml(yml) + entry = result.packages[0] + assert entry.source == "acme/tool-a" + assert entry.host == "github.com" + + def test_self_hosted_gitlab_host_accepted(self, tmp_path: Path): + """Any FQDN-shaped first segment is accepted (e.g. self-hosted GitLab).""" + content = _minimal_yml( + packages=( + "packages:\n" + " - name: tool-a\n" + " source: gitlab.example.org/team/repo\n" + " ref: main" + ) + ) + yml = _write_yml(tmp_path, content) + result = load_marketplace_yml(yml) + entry = result.packages[0] + assert entry.source == "team/repo" + assert entry.host == "gitlab.example.org" + + def test_four_segment_path_rejected(self, tmp_path: Path): + """Source with four slash-separated segments is not a valid form.""" + content = _minimal_yml( + packages=( + "packages:\n" + " - name: tool-a\n" + " source: host.tld/extra/owner/repo\n" + " ref: v1.0.0" + ) + ) + yml = _write_yml(tmp_path, content) + with pytest.raises(MarketplaceYmlError, match="source"): + load_marketplace_yml(yml) + + def test_three_segment_without_dot_rejected(self, tmp_path: Path): + """First segment must look like a hostname (contain a dot).""" + content = _minimal_yml( + packages=( + "packages:\n" + " - name: tool-a\n" + " source: not-a-host/owner/repo\n" + " ref: v1.0.0" + ) + ) + yml = _write_yml(tmp_path, content) + with pytest.raises(MarketplaceYmlError, match="source"): + load_marketplace_yml(yml) + + def test_subdir_preserved_with_host_prefix(self, tmp_path: Path): + """``subdir`` is independent of the host-prefix split.""" + content = _minimal_yml( + packages=( + "packages:\n" + " - name: ch-baseline\n" + " source: ghe.example.com/acme/agents\n" + " subdir: packages/ch-baseline\n" + " ref: v0.3.0" + ) + ) + yml = _write_yml(tmp_path, content) + result = load_marketplace_yml(yml) + entry = result.packages[0] + assert entry.host == "ghe.example.com" + assert entry.source == "acme/agents" + assert entry.subdir == "packages/ch-baseline" + + # --------------------------------------------------------------------------- # S1: Output path traversal guard # ---------------------------------------------------------------------------