EXPERIMENT: Switch container networking from host to named bridge networks

development/playbooks/deploy-dev/deploy-dev.yaml

@@ -17,6 +17,7 @@
     - role: certificates
     - role: postgresql
       vars:
+        postgresql_network: host
         postgresql_databases:
           - name: "{{ candlepin_database_name }}"
             owner: "{{ candlepin_database_user }}"
@@ -35,9 +36,21 @@
           - name: "{{ pulp_database_user }}"
             password: "{{ pulp_database_password }}"
     - role: redis
+      vars:
+        redis_network: host
     - role: candlepin
+      vars:
+        candlepin_networks: host
+        candlepin_database_host: localhost
     - role: httpd
     - role: pulp
+      vars:
+        pulp_networks: host
+        pulp_migration_networks: host
+        pulp_database_host: localhost
+        pulp_redis_url: "redis://localhost:6379/8"
+        pulp_api_ports: []
+        pulp_content_ports: []
     - role: foreman_development
   post_tasks:
     - name: Display development environment information

development/playbooks/remote-database/remote-database.yaml

@@ -15,6 +15,9 @@
     - role: pre_install
     - role: certificates
     - role: postgresql
+      vars:
+        postgresql_network: host
+        postgresql_ports: []
 
   tasks:
     - name: Fetch PostgreSQL SSL CA

development/roles/foreman_development/tasks/main.yaml

@@ -222,10 +222,10 @@
   ansible.builtin.uri:
     url: '{{ foreman_development_url }}/api/v2/ping'
     validate_certs: false
+  register: foreman_development_status
   until: foreman_development_status.status == 200
   retries: 30
   delay: 5
-  register: foreman_development_status
 
 - name: Configure Foreman Proxy for Pulp
   theforeman.foreman.smart_proxy:

development/roles/foreman_installer_certs/tasks/main.yml

@@ -14,3 +14,47 @@
 - name: Generate certs
   ansible.builtin.command: foreman-certs --apache true --foreman true --candlepin true
   changed_when: false
+
+# foreman-certs only generates certs for localhost and the server FQDN. Candlepin
+# runs in its own container and is reachable by other containers via the DNS name
+# "candlepin" on the bridge network. Foreman validates the TLS certificate hostname
+# when connecting to https://candlepin:23443, so a dedicated cert with SAN=candlepin
+# is required.
+- name: Create candlepin cert directory
+  ansible.builtin.file:
+    path: /root/ssl-build/candlepin
+    state: directory
+    mode: '0755'
+
+- name: Generate candlepin Tomcat private key
+  ansible.builtin.command: >
+    openssl genrsa
+      -out /root/ssl-build/candlepin/candlepin-tomcat.key
+      4096
+  args:
+    creates: /root/ssl-build/candlepin/candlepin-tomcat.key
+
+- name: Generate candlepin Tomcat certificate signing request
+  ansible.builtin.command: >
+    openssl req
+      -new
+      -key /root/ssl-build/candlepin/candlepin-tomcat.key
+      -subj "/CN=candlepin"
+      -addext "subjectAltName = DNS:candlepin"
+      -out /root/ssl-build/candlepin/candlepin-tomcat.csr
+  args:
+    creates: /root/ssl-build/candlepin/candlepin-tomcat.csr
+
+- name: Sign candlepin certificate with installer CA
+  ansible.builtin.command: >
+    openssl x509
+      -req
+      -in /root/ssl-build/candlepin/candlepin-tomcat.csr
+      -CA /root/ssl-build/katello-default-ca.crt
+      -CAkey /root/ssl-build/katello-default-ca.key
+      -CAcreateserial
+      -days 7300
+      -passin "file:/root/ssl-build/katello-default-ca.pwd"
+      -copy_extensions copy
+      -out /root/ssl-build/candlepin/candlepin-tomcat.crt
+  changed_when: true

docs/deployment.md

@@ -236,3 +236,95 @@ As `foremanctl` is Ansible-based, this means that the ["control node"](https://d
 To simplify the "install `foremanctl`" step, our test infrastructure uses different systems for the "control node" (the system the source code is cloned to) and the "target node" (the VM created by our development tooling).
 
 There is a desire to allow deployments where a single `foremanctl` control node manages multiple managed nodes, but no code exists yet for this.
+
+## Container Networking
+
+All containers are connected to one or more named Podman bridge networks instead of sharing the host network namespace, limiting lateral movement: a container can only reach the services it is explicitly connected to.
+
+### Networks
+
+#### `foreman-db`
+
+**Properties:** `internal: true`, `isolate: true`
+
+The database network. Only containers that need to read or write persistent data are attached.
+
+- `internal: true` removes the default gateway, so no container on this network can initiate outbound internet connections. Database servers have no reason to reach the internet, and clients that need internet access (e.g. for content sync) are multi-homed and use a different network for that.
+- `isolate: true` prevents containers on this network from forwarding packets to containers on other bridge networks, closing off lateral movement paths between network segments.
+
+| Container | Role |
+|-----------|------|
+| `postgresql` | Server — listens on port 5432 (internal DB only) |
+| `foreman` | Client |
+| `dynflow-sidekiq@*` | Client |
+| `foreman-recurring@*` | Client |
+| `candlepin` | Client (internal DB only) |
+| `pulp-api` | Client |
+| `pulp-content` | Client |
+| `pulp-worker@*` | Client |
+
+Ansible's `community.postgresql.*` modules reach the database during deployment via a Unix socket: `/var/run/postgresql` is bind-mounted from the host into the container so the socket is accessible on the host without publishing a TCP port.
+
+#### `foreman-cache`
+
+**Properties:** `internal: true`, `isolate: true`
+
+The cache network. Only containers that need to reach Redis are attached. The same rationale as `foreman-db` applies: cache servers have no business reaching the internet, and the `isolate` flag prevents bridge pivoting.
+
+| Container | Role |
+|-----------|------|
+| `redis` | Server — listens on port 6379 |
+| `foreman` | Client — app cache and Dynflow queue |
+| `dynflow-sidekiq@*` | Client — job queue |
+| `foreman-recurring@*` | Client — job queue |
+| `pulp-api` | Client |
+| `pulp-content` | Client |
+| `pulp-worker@*` | Client |
+
+Redis does not publish any port to the host: it is a purely internal service with no legitimate consumers outside the container network.
+
+#### `foreman-app`
+
+**Properties:** none (`internal: false`, `isolate: false`)
+
+The application network. Containers that need to communicate with each other at the application layer, or that need outbound internet access (e.g. for content synchronisation), are attached here.
+
+| Container | Role |
+|-----------|------|
+| `candlepin` | Server — Tomcat (23443) and Artemis STOMP broker (61613) |
+| `foreman` | Client to Candlepin; serves Foreman Proxy requests |
+| `dynflow-sidekiq@*` | Client |
+| `foreman-recurring@*` | Client |
+| `pulp-api` | Server — API (24817); needs internet for content sync |
+| `pulp-content` | Server — content (24816); needs internet for content sync |
+| `pulp-worker@*` | Worker — needs internet for content sync |
+
+Candlepin does not publish any ports to the host: `foreman` reaches it directly over the bridge using its DNS name. `foreman`, `pulp-api`, and `pulp-content` publish their respective ports to `127.0.0.1` so that the `httpd` reverse proxy running on the host can reach them.
+
+#### `foreman-proxy-net`
+
+**Properties:** none (`internal: false`, `isolate: false`)
+
+The proxy network, used exclusively for communication between Foreman and Foreman Proxy. Keeping this traffic on a dedicated network makes it straightforward to apply stricter controls in future without affecting the rest of the application.
+
+| Container | Role |
+|-----------|------|
+| `foreman-proxy` | Server — listens on 0.0.0.0:8443 (external) |
+| `foreman` | Client |
+
+`foreman-proxy` publishes port `0.0.0.0:8443` so that remote Foreman Proxies and clients can register and communicate with it from outside the host.
+
+### Network membership summary
+
+| Container | foreman-db | foreman-cache | foreman-app | foreman-proxy-net |
+|-----------|:----------:|:-------------:|:-----------:|:-----------------:|
+| `postgresql` | ✓ | | | |
+| `redis` | | ✓ | | |
+| `candlepin` | ✓ (internal DB) | | ✓ | |
+| `foreman` | ✓ | ✓ | ✓ | ✓ |
+| `dynflow-sidekiq@*` | ✓ | ✓ | ✓ | |
+| `foreman-recurring@*` | ✓ | ✓ | ✓ | |
+| `foreman-proxy` | | | | ✓ |
+| `pulp-api` | ✓ | ✓ | ✓ | |
+| `pulp-content` | ✓ | ✓ | ✓ | |
+| `pulp-worker@*` | ✓ | ✓ | ✓ | |

src/playbooks/deploy/deploy.yaml

@@ -22,6 +22,22 @@
         certificate_checks_certificate: "{{ server_certificate }}"
         certificate_checks_key: "{{ server_key }}"
         certificate_checks_ca: "{{ ca_certificate }}"
+    - role: deploy_network
+      vars:
+        deploy_network_name: foreman-db
+        deploy_network_internal: true
+        deploy_network_isolate: true
+    - role: deploy_network
+      vars:
+        deploy_network_name: foreman-cache
+        deploy_network_internal: true
+        deploy_network_isolate: true
+    - role: deploy_network
+      vars:
+        deploy_network_name: foreman-app
+    - role: deploy_network
+      vars:
+        deploy_network_name: foreman-proxy-net
     - role: postgresql
       when:
         - database_mode == 'internal'

src/roles/candlepin/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 candlepin_ssl_port: 23443
-candlepin_hostname: localhost
+candlepin_hostname: "0.0.0.0"
 candlepin_tls_versions:
   - "TLSv1.2"
 candlepin_ciphers:
@@ -15,8 +15,9 @@ candlepin_ciphers:
 candlepin_container_image: quay.io/foreman/candlepin
 candlepin_container_tag: "4.4.14"
 candlepin_registry_auth_file: /etc/foreman/registry-auth.json
+candlepin_networks: "{{ (['foreman-db'] if database_mode == 'internal' else []) + ['foreman-app'] }}"
 
-candlepin_database_host: localhost
+candlepin_database_host: postgresql
 candlepin_database_port: 5432
 candlepin_database_ssl: false
 candlepin_database_ssl_mode: disable

src/roles/candlepin/tasks/main.yml

@@ -71,7 +71,7 @@
     name: "candlepin"
     image: "{{ candlepin_container_image }}:{{ candlepin_container_tag }}"
     state: quadlet
-    network: host
+    network: "{{ candlepin_networks }}"
     hostname: "{{ ansible_facts['hostname'] }}.local"
     secrets:
       - 'candlepin-ca-cert,target=/etc/candlepin/certs/candlepin-ca.crt,mode=0440,type=mount'
@@ -100,7 +100,7 @@
         After=redis.service postgresql.service
         [Service]
         TimeoutStartSec=300
-    healthcheck: curl --fail --insecure https://localhost:23443/candlepin/status
+    healthcheck: curl --fail --cacert /etc/candlepin/certs/candlepin-ca.crt --resolve candlepin:23443:127.0.0.1 https://candlepin:23443/candlepin/status
     sdnotify: healthy
 
 - name: Run daemon reload to make Quadlet create the service files

src/roles/candlepin/templates/broker.xml.j2

@@ -10,7 +10,7 @@
 
         <acceptors>
             <acceptor name="in-vm">vm://0</acceptor>
-            <acceptor name="stomp">tcp://localhost:61613?protocols=STOMP;useEpoll=false;sslEnabled=true;trustStorePath=/etc/candlepin/certs/truststore;trustStorePassword={{ candlepin_keystore_password }};keyStorePath=/etc/candlepin/certs/keystore;keyStorePassword={{ candlepin_keystore_password }};needClientAuth=true</acceptor>
+            <acceptor name="stomp">tcp://{{ candlepin_hostname }}:61613?protocols=STOMP;useEpoll=false;sslEnabled=true;trustStorePath=/etc/candlepin/certs/truststore;trustStorePassword={{ candlepin_keystore_password }};keyStorePath=/etc/candlepin/certs/keystore;keyStorePassword={{ candlepin_keystore_password }};needClientAuth=true</acceptor>
         </acceptors>
 
         <security-enabled>true</security-enabled>

src/roles/deploy_network/defaults/main.yaml

@@ -0,0 +1,22 @@
+---
+deploy_network_name: foreman-network
+
+# Network driver
+deploy_network_driver: bridge
+
+# Subnet and gateway (optional - auto-assigned if omitted)
+# deploy_network_subnet: "10.89.0.0/24"
+# deploy_network_gateway: "10.89.0.1"
+
+# Prevent containers on this network from reaching external hosts or the host itself
+deploy_network_internal: false
+
+# Prevent containers on this network from reaching containers on other networks
+deploy_network_isolate: false
+
+# Enable IPv6
+deploy_network_ipv6: false
+
+# DNS servers (optional - uses system defaults if omitted)
+# deploy_network_dns:
+#   - 8.8.8.8

src/roles/deploy_network/tasks/main.yaml

@@ -0,0 +1,3 @@
+---
+- name: Deploy network
+  ansible.builtin.include_tasks: podman.yaml

src/roles/deploy_network/tasks/podman.yaml

@@ -0,0 +1,12 @@
+---
+- name: Create network {{ deploy_network_name }}
+  containers.podman.podman_network:
+    name: "{{ deploy_network_name }}"
+    driver: "{{ deploy_network_driver }}"
+    internal: "{{ deploy_network_internal }}"
+    ipv6: "{{ deploy_network_ipv6 }}"
+    subnet: "{{ deploy_network_subnet | default(omit) }}"
+    gateway: "{{ deploy_network_gateway | default(omit) }}"
+    dns: "{{ deploy_network_dns | default(omit) }}"
+    opt: "{{ {'isolate': deploy_network_isolate} if deploy_network_isolate else omit }}"
+    state: present

src/roles/foreman/defaults/main.yaml

@@ -5,7 +5,7 @@ foreman_registry_auth_file: /etc/foreman/registry-auth.json
 
 foreman_database_name: foreman
 foreman_database_user: foreman
-foreman_database_host: localhost
+foreman_database_host: postgresql
 foreman_database_port: 5432
 foreman_database_pool: 9
 foreman_database_ssl_mode: disable

src/roles/foreman/tasks/main.yaml

@@ -101,7 +101,13 @@
     image: "{{ foreman_container_image }}:{{ foreman_container_tag }}"
     state: quadlet
     sdnotify: true
-    network: host
+    network:
+      - foreman-db
+      - foreman-cache
+      - foreman-app
+      - foreman-proxy-net
+    ports:
+      - "127.0.0.1:3000:3000"
     hostname: "{{ ansible_facts['hostname'] }}.local"
     volume:
       - 'foreman-data-run:/var/run/foreman:z'
@@ -137,7 +143,10 @@
     image: "{{ foreman_container_image }}:{{ foreman_container_tag }}"
     state: quadlet
     sdnotify: true
-    network: host
+    network:
+      - foreman-db
+      - foreman-cache
+      - foreman-app
     hostname: "{{ ansible_facts['hostname'] }}.local"
     volume:
       - 'foreman-data-run:/var/run/foreman:z'
@@ -151,7 +160,7 @@
       - 'foreman-dynflow-worker-hosts-queue-yaml,type=mount,target=/etc/foreman/dynflow/worker-hosts-queue.yml'
       - 'foreman-db-ca,type=mount,target={{ foreman_database_ssl_ca_path }}'
     env:
-      DYNFLOW_REDIS_URL: "redis://localhost:6379/6"
+      DYNFLOW_REDIS_URL: "redis://redis:6379/6"
       REDIS_PROVIDER: "DYNFLOW_REDIS_URL"
       FOREMAN_ENABLED_PLUGINS: "{{ foreman_plugins | join(' ') }}"
     command: "/usr/libexec/foreman/sidekiq-selinux -e production -r ./extras/dynflow-sidekiq.rb -C /etc/foreman/dynflow/%i.yml"
@@ -189,7 +198,10 @@
     state: quadlet
     image: "{{ foreman_container_image }}:{{ foreman_container_tag }}"
     sdnotify: false
-    network: host
+    network:
+      - foreman-db
+      - foreman-cache
+      - foreman-app
     hostname: "{{ ansible_facts['hostname'] }}.local"
     command: "foreman-rake {{ item.rake }}"
     volume:
@@ -239,7 +251,7 @@
       - bin/rails db:migrate && bin/rails db:seed
     detach: false
     rm: true
-    network: host
+    network: "{{ ['foreman-db'] if database_mode == 'internal' else ['foreman-app'] }}"
     env:
       FOREMAN_ENABLED_PLUGINS: "{{ foreman_plugins | join(' ') }}"
     secrets:

src/roles/foreman/templates/katello.yaml.j2

@@ -3,13 +3,13 @@
   :rest_client_timeout: 3600
 
   :candlepin:
-    :url: https://localhost:23443/candlepin
+    :url: https://candlepin:23443/candlepin
     :oauth_key: "katello"
     :oauth_secret: "{{ candlepin_oauth_secret }}"
     :ca_cert_file: /etc/foreman/katello-default-ca.crt
 
   :candlepin_events:
-    :broker_host: localhost
+    :broker_host: candlepin
     :ssl_cert_file: /etc/foreman/client_cert.pem
     :ssl_key_file: /etc/foreman/client_key.pem
     :ssl_ca_file: /etc/foreman/katello-default-ca.crt

src/roles/foreman/templates/settings.yaml.j2

@@ -10,7 +10,7 @@
 :rails_cache_store:
   :type: redis
   :urls:
-    - redis://localhost:6379/4
+    - redis://redis:6379/4
   :options:
     :compress: true
     :namespace: foreman