From 0e0e6cbc1a30586807bb5e7c71bae5fd88d20d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarl=20Andr=C3=A9=20H=C3=BCbenthal?= Date: Wed, 22 Apr 2026 08:28:29 +0200 Subject: [PATCH 1/2] feat(docker): auto-resolve registry auth from ~/.docker/config.json Pulling images from private registries previously failed with HTTP 407 because the library never looked at the user's Docker credentials. Align with other Testcontainers implementations by reading `~/.docker/config.json` (honouring `DOCKER_CONFIG`) and passing an `X-Registry-Auth` header on image pulls when no explicit auth is configured on the container. Only the `auths` map is consulted. Credential helpers (`credsStore`, `credHelpers`) are intentionally out of scope for this change: when encountered they produce a debug log and resolution falls back to anonymous access so the pull can still proceed (or fail as before) rather than shelling out to a helper binary. The header value is URL-safe base64 (without padding) of a JSON payload with `username`, `password`, and `serveraddress`, per the Docker Engine API spec. Explicitly configured `Container.auth` continues to take precedence over auto-resolution, and missing/invalid config files are handled silently at debug level. --- lib/docker/auth.ex | 197 +++++++++++++++++++++++++++++++ lib/testcontainers.ex | 13 +- test/docker/auth_test.exs | 94 +++++++++++++++ test/fixtures/docker_config.json | 10 ++ 4 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 lib/docker/auth.ex create mode 100644 test/docker/auth_test.exs create mode 100644 test/fixtures/docker_config.json diff --git a/lib/docker/auth.ex b/lib/docker/auth.ex new file mode 100644 index 0000000..c7b14f4 --- /dev/null +++ b/lib/docker/auth.ex @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: MIT +defmodule Testcontainers.Docker.Auth do + @moduledoc """ + Resolves Docker registry credentials from the user's Docker config file + (typically `~/.docker/config.json`) and returns a ready-to-send + `X-Registry-Auth` header value. + + Scope: + + * Only the `auths` map in `config.json` is supported. + * Credential helpers (`credsStore`, `credHelpers`) are intentionally out + of scope — if encountered, a debug log is emitted and `nil` is returned + so the caller can fall back to anonymous access. + + The `DOCKER_CONFIG` environment variable is honoured: when set, the config + file is read from `$DOCKER_CONFIG/config.json`; otherwise the default path + `~/.docker/config.json` is used. + + The header value is a URL-safe base64 encoding (without padding) of a JSON + document describing the credentials, as specified by the Docker Engine API. + """ + + require Logger + + @docker_hub_key "https://index.docker.io/v1/" + + @doc """ + Resolves registry credentials for the given `image` and returns the + ready-to-send `X-Registry-Auth` header value, or `nil` if no matching + credentials can be found. + + `config_path` may be `nil`, in which case the default lookup logic is used + (respecting the `DOCKER_CONFIG` environment variable). + """ + @spec resolve(String.t(), String.t() | nil) :: String.t() | nil + def resolve(image, config_path \\ nil) when is_binary(image) do + case read_config(config_path) do + {:ok, config} -> + registry = registry_for_image(image) + resolve_from_config(config, registry) + + :error -> + nil + end + end + + @doc """ + Returns the registry key that should be used for looking up credentials + for the given `image` (Docker config convention). + + Unnamespaced or explicitly `docker.io`-hosted images resolve to + `https://index.docker.io/v1/`; everything else resolves to the registry + host component of the image reference. + """ + @spec registry_for_image(String.t()) :: String.t() + def registry_for_image(image) when is_binary(image) do + case String.split(image, "/", parts: 2) do + [_single] -> @docker_hub_key + [maybe_host, _rest] -> registry_from_host_component(maybe_host) + end + end + + defp registry_from_host_component(component) do + cond do + not host?(component) -> @docker_hub_key + component in ["docker.io", "index.docker.io"] -> @docker_hub_key + true -> component + end + end + + # A registry host contains a "." or ":" or is exactly "localhost". + defp host?(component) do + component == "localhost" or String.contains?(component, ".") or + String.contains?(component, ":") + end + + defp read_config(nil), do: read_config(default_config_path()) + + defp read_config(path) when is_binary(path) do + with {:ok, contents} <- File.read(path), + {:ok, decoded} <- Jason.decode(contents) do + {:ok, decoded} + else + {:error, reason} -> + Logger.debug( + "Testcontainers.Docker.Auth: could not read Docker config at #{path}: #{inspect(reason)}" + ) + + :error + end + end + + defp default_config_path do + case System.get_env("DOCKER_CONFIG") do + nil -> Path.join(System.user_home() || "", ".docker/config.json") + "" -> Path.join(System.user_home() || "", ".docker/config.json") + dir -> Path.join(dir, "config.json") + end + end + + defp resolve_from_config(config, registry) do + auths = Map.get(config, "auths", %{}) + + case find_auth_entry(auths, registry) do + {matched_key, %{"auth" => encoded}} when is_binary(encoded) and encoded != "" -> + build_header(encoded, matched_key) + + {_matched_key, _entry} -> + maybe_warn_cred_helper(config, registry) + nil + + :not_found -> + maybe_warn_cred_helper(config, registry) + nil + end + end + + defp find_auth_entry(auths, registry) when is_map(auths) do + candidates = candidate_keys(registry) + + Enum.find_value(candidates, :not_found, fn key -> + case Map.get(auths, key) do + nil -> nil + entry when is_map(entry) -> {key, entry} + _ -> nil + end + end) + end + + defp find_auth_entry(_auths, _registry), do: :not_found + + # Docker config.json keys are sometimes stored with scheme/path prefixes and + # sometimes as plain hostnames. We try the most specific form first and fall + # back to the bare host. + defp candidate_keys(@docker_hub_key) do + [ + @docker_hub_key, + "index.docker.io", + "docker.io", + "https://index.docker.io/v1", + "https://index.docker.io", + "https://docker.io" + ] + end + + defp candidate_keys(registry) do + [ + registry, + "https://" <> registry, + "http://" <> registry, + "https://" <> registry <> "/", + "http://" <> registry <> "/" + ] + end + + defp build_header(encoded, server_address) do + with {:ok, decoded} <- Base.decode64(encoded), + [username, password] <- String.split(decoded, ":", parts: 2) do + payload = %{ + "username" => username, + "password" => password, + "serveraddress" => server_address + } + + payload + |> Jason.encode!() + |> Base.url_encode64(padding: false) + else + _ -> + Logger.debug( + "Testcontainers.Docker.Auth: could not decode auth entry for #{server_address}" + ) + + nil + end + end + + defp maybe_warn_cred_helper(config, registry) do + cond do + Map.has_key?(config, "credsStore") -> + Logger.debug( + "Testcontainers.Docker.Auth: credsStore present in Docker config; " <> + "credential helpers are not supported, returning nil for #{registry}" + ) + + is_map(Map.get(config, "credHelpers")) and + Map.has_key?(config["credHelpers"], registry) -> + Logger.debug( + "Testcontainers.Docker.Auth: credHelpers entry for #{registry} present; " <> + "credential helpers are not supported, returning nil" + ) + + true -> + :ok + end + end +end diff --git a/lib/testcontainers.ex b/lib/testcontainers.ex index 31c2d57..5b57e2d 100644 --- a/lib/testcontainers.ex +++ b/lib/testcontainers.ex @@ -18,6 +18,7 @@ defmodule Testcontainers do alias Testcontainers.ContainerBuilder alias Testcontainers.CopyTo alias Testcontainers.Docker.Api + alias Testcontainers.Docker.Auth, as: DockerAuth alias Testcontainers.DockerCompose alias Testcontainers.PullPolicy alias Testcontainers.Util.PropertiesParser @@ -868,7 +869,7 @@ defmodule Testcontainers do end defp maybe_pull_image(%{pull_policy: %{always_pull: true}} = config, conn) do - case Api.pull_image(config.image, conn, auth: config.auth) do + case Api.pull_image(config.image, conn, auth: resolve_auth(config)) do {:ok, _nil} -> :ok error -> error end @@ -881,7 +882,7 @@ defmodule Testcontainers do :ok {:ok, false} -> - case Api.pull_image(config.image, conn, auth: config.auth) do + case Api.pull_image(config.image, conn, auth: resolve_auth(config)) do {:ok, _nil} -> :ok error -> error end @@ -894,7 +895,7 @@ defmodule Testcontainers do defp maybe_pull_image(%{pull_policy: %{pull_condition: expr}} = config, conn) when is_function(expr) do with {:eval, true} <- {:eval, expr.(config, conn)}, - {:ok, _nil} <- Api.pull_image(config.image, conn, auth: config.auth) do + {:ok, _nil} <- Api.pull_image(config.image, conn, auth: resolve_auth(config)) do :ok else {:eval, reason} -> @@ -913,6 +914,12 @@ defmodule Testcontainers do :ok end + # Use the explicitly configured auth if present; otherwise try to + # auto-resolve credentials from the user's Docker config. + defp resolve_auth(%{auth: auth}) when is_binary(auth) and auth != "", do: auth + defp resolve_auth(%{image: image}) when is_binary(image), do: DockerAuth.resolve(image, nil) + defp resolve_auth(_), do: nil + defp copy_to_container(id, config, conn) do Enum.reduce(config.copy_to, :ok, fn copy_to, :ok -> diff --git a/test/docker/auth_test.exs b/test/docker/auth_test.exs new file mode 100644 index 0000000..7d6034b --- /dev/null +++ b/test/docker/auth_test.exs @@ -0,0 +1,94 @@ +defmodule Testcontainers.Docker.AuthTest do + use ExUnit.Case, async: true + + alias Testcontainers.Docker.Auth + + @fixture Path.expand("../fixtures/docker_config.json", __DIR__) + + describe "registry_for_image/1" do + test "unnamespaced images resolve to Docker Hub" do + assert Auth.registry_for_image("redis") == "https://index.docker.io/v1/" + assert Auth.registry_for_image("redis:7") == "https://index.docker.io/v1/" + end + + test "library-style namespaced images resolve to Docker Hub" do + assert Auth.registry_for_image("library/redis:7") == "https://index.docker.io/v1/" + assert Auth.registry_for_image("myorg/myimage:tag") == "https://index.docker.io/v1/" + end + + test "explicit docker.io / index.docker.io resolve to Docker Hub" do + assert Auth.registry_for_image("docker.io/library/redis") == "https://index.docker.io/v1/" + assert Auth.registry_for_image("index.docker.io/foo/bar") == "https://index.docker.io/v1/" + end + + test "private registries return the registry host" do + assert Auth.registry_for_image("myreg.example.com/foo/bar:tag") == "myreg.example.com" + assert Auth.registry_for_image("registry.internal:5000/a/b") == "registry.internal:5000" + assert Auth.registry_for_image("localhost/foo:tag") == "localhost" + end + end + + describe "resolve/2" do + test "returns header for a Docker Hub image" do + header = Auth.resolve("library/redis:7", @fixture) + + assert is_binary(header) + decoded = decode_header(header) + + assert decoded == %{ + "username" => "alice", + "password" => "s3cret", + "serveraddress" => "https://index.docker.io/v1/" + } + end + + test "returns header for a private registry image" do + header = Auth.resolve("myreg.example.com/foo/bar:tag", @fixture) + + assert is_binary(header) + decoded = decode_header(header) + + assert decoded == %{ + "username" => "bob", + "password" => "hunter2", + "serveraddress" => "myreg.example.com" + } + end + + test "returns nil for an unknown registry" do + assert Auth.resolve("unknown.example.org/foo/bar:tag", @fixture) == nil + end + + test "returns nil when the config file is missing" do + missing = Path.join(System.tmp_dir!(), "testcontainers-missing-#{System.unique_integer()}.json") + refute File.exists?(missing) + + assert Auth.resolve("library/redis:7", missing) == nil + end + + test "returns nil when the config file is invalid JSON" do + path = Path.join(System.tmp_dir!(), "testcontainers-invalid-#{System.unique_integer()}.json") + File.write!(path, "this is not json") + + try do + assert Auth.resolve("library/redis:7", path) == nil + after + File.rm(path) + end + end + + test "header is URL-safe base64 without padding" do + header = Auth.resolve("library/redis:7", @fixture) + + refute String.contains?(header, "=") + refute String.contains?(header, "+") + refute String.contains?(header, "/") + end + end + + defp decode_header(header) do + header + |> Base.url_decode64!(padding: false) + |> Jason.decode!() + end +end diff --git a/test/fixtures/docker_config.json b/test/fixtures/docker_config.json new file mode 100644 index 0000000..dbbed6f --- /dev/null +++ b/test/fixtures/docker_config.json @@ -0,0 +1,10 @@ +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "YWxpY2U6czNjcmV0" + }, + "myreg.example.com": { + "auth": "Ym9iOmh1bnRlcjI=" + } + } +} From 4367df0eb6432728d9b0ffa09ec5cb56bffc2d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarl=20Andr=C3=A9=20H=C3=BCbenthal?= Date: Wed, 22 Apr 2026 08:51:05 +0200 Subject: [PATCH 2/2] fix(docker): normalize registry auth serveraddress and fall back on 4xx Podman rejects an X-Registry-Auth header whose `serveraddress` is a full URL (e.g. `https://index.docker.io/v1/`) with HTTP 400. The Docker Engine API spec calls for a bare domain/IP (optionally with port). Normalize the value in `Testcontainers.Docker.Auth.build_header/2` by stripping the scheme and any trailing path, and canonicalize `index.docker.io` to `docker.io` to match the Docker CLI. When auth is auto-resolved from the user's `~/.docker/config.json` (rather than set explicitly on the container config) and the pull fails with a 4xx, retry once without the header. This protects pulls against stale or otherwise invalid credentials that happen to sit in the config file on shared environments such as CI runners, where they would previously have broken the first user-visible pull. Fixes a regression seen on the rootless-podman CI job. --- lib/docker/auth.ex | 31 +++++++++++++++++++++++- lib/testcontainers.ex | 51 +++++++++++++++++++++++++++++++++------ test/docker/auth_test.exs | 33 +++++++++++++++++++++++-- 3 files changed, 104 insertions(+), 11 deletions(-) diff --git a/lib/docker/auth.ex b/lib/docker/auth.ex index c7b14f4..6424398 100644 --- a/lib/docker/auth.ex +++ b/lib/docker/auth.ex @@ -159,7 +159,7 @@ defmodule Testcontainers.Docker.Auth do payload = %{ "username" => username, "password" => password, - "serveraddress" => server_address + "serveraddress" => normalize_server_address(server_address) } payload @@ -175,6 +175,35 @@ defmodule Testcontainers.Docker.Auth do end end + # Docker config.json keys are often stored as URLs (e.g. + # "https://index.docker.io/v1/") but the Docker Engine API's + # `serveraddress` field expects a bare domain/IP (optionally with port) — + # podman in particular rejects a full URL with HTTP 400. + @doc false + @spec normalize_server_address(String.t()) :: String.t() + def normalize_server_address(address) when is_binary(address) do + address + |> strip_scheme() + |> strip_trailing_path() + |> canonicalize_docker_hub() + end + + defp strip_scheme(address) do + case String.split(address, "://", parts: 2) do + [_scheme, rest] -> rest + [single] -> single + end + end + + defp strip_trailing_path(address) do + address + |> String.split("/", parts: 2) + |> List.first() + end + + defp canonicalize_docker_hub("index.docker.io"), do: "docker.io" + defp canonicalize_docker_hub(host), do: host + defp maybe_warn_cred_helper(config, registry) do cond do Map.has_key?(config, "credsStore") -> diff --git a/lib/testcontainers.ex b/lib/testcontainers.ex index 5b57e2d..ccbe818 100644 --- a/lib/testcontainers.ex +++ b/lib/testcontainers.ex @@ -869,7 +869,7 @@ defmodule Testcontainers do end defp maybe_pull_image(%{pull_policy: %{always_pull: true}} = config, conn) do - case Api.pull_image(config.image, conn, auth: resolve_auth(config)) do + case pull_with_fallback(config, conn) do {:ok, _nil} -> :ok error -> error end @@ -882,7 +882,7 @@ defmodule Testcontainers do :ok {:ok, false} -> - case Api.pull_image(config.image, conn, auth: resolve_auth(config)) do + case pull_with_fallback(config, conn) do {:ok, _nil} -> :ok error -> error end @@ -895,7 +895,7 @@ defmodule Testcontainers do defp maybe_pull_image(%{pull_policy: %{pull_condition: expr}} = config, conn) when is_function(expr) do with {:eval, true} <- {:eval, expr.(config, conn)}, - {:ok, _nil} <- Api.pull_image(config.image, conn, auth: resolve_auth(config)) do + {:ok, _nil} <- pull_with_fallback(config, conn) do :ok else {:eval, reason} -> @@ -914,11 +914,46 @@ defmodule Testcontainers do :ok end - # Use the explicitly configured auth if present; otherwise try to - # auto-resolve credentials from the user's Docker config. - defp resolve_auth(%{auth: auth}) when is_binary(auth) and auth != "", do: auth - defp resolve_auth(%{image: image}) when is_binary(image), do: DockerAuth.resolve(image, nil) - defp resolve_auth(_), do: nil + # Pulls the image with the appropriate auth header. If auth was + # auto-resolved from ~/.docker/config.json (rather than set explicitly on + # the container config) and the pull fails with an HTTP 4xx, fall back to + # an anonymous pull — the daemon may reject stale or otherwise invalid + # auto-resolved credentials that we shouldn't have sent in the first place. + defp pull_with_fallback(config, conn) do + case resolve_auth(config) do + {:explicit, auth} -> + Api.pull_image(config.image, conn, auth: auth) + + {:auto, auth} -> + case Api.pull_image(config.image, conn, auth: auth) do + {:error, {:http_error, status}} when status >= 400 and status < 500 -> + Logger.debug( + "Auto-resolved registry auth rejected (HTTP #{status}) for #{config.image}; retrying without auth" + ) + + Api.pull_image(config.image, conn, auth: nil) + + result -> + result + end + + :none -> + Api.pull_image(config.image, conn, auth: nil) + end + end + + # Tag the auth source so pull_with_fallback knows whether it's safe to + # retry without credentials on a 4xx response. + defp resolve_auth(%{auth: auth}) when is_binary(auth) and auth != "", do: {:explicit, auth} + + defp resolve_auth(%{image: image}) when is_binary(image) do + case DockerAuth.resolve(image, nil) do + nil -> :none + auth -> {:auto, auth} + end + end + + defp resolve_auth(_), do: :none defp copy_to_container(id, config, conn) do Enum.reduce(config.copy_to, :ok, fn diff --git a/test/docker/auth_test.exs b/test/docker/auth_test.exs index 7d6034b..598773f 100644 --- a/test/docker/auth_test.exs +++ b/test/docker/auth_test.exs @@ -29,7 +29,7 @@ defmodule Testcontainers.Docker.AuthTest do end describe "resolve/2" do - test "returns header for a Docker Hub image" do + test "returns header for a Docker Hub image with canonical serveraddress" do header = Auth.resolve("library/redis:7", @fixture) assert is_binary(header) @@ -38,10 +38,21 @@ defmodule Testcontainers.Docker.AuthTest do assert decoded == %{ "username" => "alice", "password" => "s3cret", - "serveraddress" => "https://index.docker.io/v1/" + "serveraddress" => "docker.io" } end + test "serveraddress never contains a scheme or path" do + # Regression: podman (and the Docker Engine API spec) reject a + # serveraddress that is a full URL; it must be a bare host (optionally + # with port). + header = Auth.resolve("library/redis:7", @fixture) + %{"serveraddress" => addr} = decode_header(header) + + refute String.contains?(addr, "://") + refute String.contains?(addr, "/") + end + test "returns header for a private registry image" do header = Auth.resolve("myreg.example.com/foo/bar:tag", @fixture) @@ -86,6 +97,24 @@ defmodule Testcontainers.Docker.AuthTest do end end + describe "normalize_server_address/1" do + test "strips scheme and path from full URLs" do + assert Auth.normalize_server_address("https://index.docker.io/v1/") == "docker.io" + assert Auth.normalize_server_address("http://myreg.example.com/") == "myreg.example.com" + end + + test "passes through bare hosts" do + assert Auth.normalize_server_address("myreg.example.com") == "myreg.example.com" + assert Auth.normalize_server_address("registry.internal:5000") == "registry.internal:5000" + assert Auth.normalize_server_address("localhost") == "localhost" + end + + test "canonicalizes index.docker.io to docker.io" do + assert Auth.normalize_server_address("index.docker.io") == "docker.io" + assert Auth.normalize_server_address("https://index.docker.io/v1/") == "docker.io" + end + end + defp decode_header(header) do header |> Base.url_decode64!(padding: false)