From 5c7b305ac44545bf1b6018c3482dc547d19d0da5 Mon Sep 17 00:00:00 2001 From: Sven-Ric Date: Wed, 22 Jul 2026 22:09:23 +0200 Subject: [PATCH] Add support for the pure container version of SONiC --- .github/workflows/base-image.yaml | 27 ++++ .github/workflows/integration.yaml | 44 +++++++ Makefile | 24 +++- README.md | 3 +- deploy_partition.yaml | 3 + files/sonic-container/frr-reload.service | 13 ++ files/sonic-container/leaf01_config_db.json | 68 ++++++++++ files/sonic-container/leaf02_config_db.json | 68 ++++++++++ images/sonic-container/Dockerfile | 76 +++++++++++ images/sonic-container/daemon.json | 3 + images/sonic-container/frr-reload | 2 + images/sonic-container/healthcheck.sh | 20 +++ images/sonic-container/lanemap.ini | 3 + images/sonic-container/lldpd.conf | 4 + images/sonic-container/port_config.ini | 123 ++++++++++++++++++ images/sonic-container/sonic-init.sh | 53 ++++++++ images/sonic-container/systemd/bgp.service | 11 ++ .../sonic-container/systemd/database.service | 11 ++ .../systemd/sonic-init.service | 11 ++ images/sonic-container/systemd/sonic.service | 13 ++ images/sonic-container/systemd/sonic.target | 6 + images/sonic-container/systemd/swss.service | 11 ++ images/sonic-container/systemd/syncd.service | 11 ++ inventories/partition.yaml | 5 + mini-lab.container_sonic.yaml | 80 ++++++++++++ roles/sonic/tasks/container.yaml | 40 ++++++ roles/sonic/tasks/main.yaml | 9 ++ 27 files changed, 740 insertions(+), 2 deletions(-) create mode 100644 files/sonic-container/frr-reload.service create mode 100644 files/sonic-container/leaf01_config_db.json create mode 100644 files/sonic-container/leaf02_config_db.json create mode 100644 images/sonic-container/Dockerfile create mode 100644 images/sonic-container/daemon.json create mode 100644 images/sonic-container/frr-reload create mode 100644 images/sonic-container/healthcheck.sh create mode 100644 images/sonic-container/lanemap.ini create mode 100644 images/sonic-container/lldpd.conf create mode 100644 images/sonic-container/port_config.ini create mode 100644 images/sonic-container/sonic-init.sh create mode 100644 images/sonic-container/systemd/bgp.service create mode 100644 images/sonic-container/systemd/database.service create mode 100644 images/sonic-container/systemd/sonic-init.service create mode 100644 images/sonic-container/systemd/sonic.service create mode 100644 images/sonic-container/systemd/sonic.target create mode 100644 images/sonic-container/systemd/swss.service create mode 100644 images/sonic-container/systemd/syncd.service create mode 100644 mini-lab.container_sonic.yaml create mode 100644 roles/sonic/tasks/container.yaml diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 039e6f01..5b24be40 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -38,3 +38,30 @@ jobs: sbom: true file: images/sonic/base-${{ matrix.versions.name }}/Dockerfile tags: ${{ env.REGISTRY }}/metal-stack/mini-lab-sonic-base:${{ matrix.versions.name }} + + build-mini-lab-sonic-container-base-image: + name: build mini-lab-sonic-container base image + runs-on: ubuntu-latest + + strategy: + matrix: + versions: + - name: "202505" + + steps: + - name: Log in to the container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + + # the docker-sonic-vs artifact is a gzipped `docker save` archive, so it + # is loaded and re-tagged instead of being built from a Dockerfile + - name: Download, load and push docker-sonic-vs + run: | + curl -fL -o docker-sonic-vs.gz \ + "https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=${{ matrix.versions.name }}&platform=vs&target=target%2Fdocker-sonic-vs.gz" + docker load -i docker-sonic-vs.gz + docker tag docker-sonic-vs:latest ${{ env.REGISTRY }}/metal-stack/mini-lab-sonic-container-base:${{ matrix.versions.name }} + docker push ${{ env.REGISTRY }}/metal-stack/mini-lab-sonic-container-base:${{ matrix.versions.name }} diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 448a52da..e94803e9 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -94,12 +94,54 @@ jobs: cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }} cache-to: type=inline + build-mini-lab-sonic-container-image: + name: Build mini-lab-sonic-container image + runs-on: ubuntu-latest + + steps: + - name: Log in to the container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Make tag + run: | + IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest") + SHA_TAG=${COMMIT_SHA::8} + + echo "MINI_LAB_SONIC_CONTAINER_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-container:${IMAGE_TAG}" >> $GITHUB_ENV + echo "MINI_LAB_SONIC_CONTAINER_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-container:${SHA_TAG}" >> $GITHUB_ENV + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push mini-lab-sonic-container container + uses: docker/build-push-action@v6 + with: + context: ./images/sonic-container + pull: true + push: true + sbom: true + tags: | + ${{ env.MINI_LAB_SONIC_CONTAINER_IMAGE }} + ${{ env.MINI_LAB_SONIC_CONTAINER_IMAGE_SHA }} + cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_CONTAINER_IMAGE }} + cache-to: type=inline + test: name: Run tests runs-on: self-hosted needs: - build-mini-lab-vms-image - build-mini-lab-sonic-image + - build-mini-lab-sonic-container-image continue-on-error: true strategy: @@ -108,6 +150,7 @@ jobs: - name: sonic - name: gardener - name: dell_sonic + - name: container_sonic steps: - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211 @@ -141,6 +184,7 @@ jobs: echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV + echo "MINI_LAB_SONIC_CONTAINER_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-container:${IMAGE_TAG}" >> $GITHUB_ENV - name: Run integration tests shell: bash diff --git a/Makefile b/Makefile index 837e44bf..62324bc5 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ ANSIBLE_DISPLAY_SKIPPED_HOSTS=false MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic) MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest) MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest) +MINI_LAB_SONIC_CONTAINER_IMAGE := $(or $(MINI_LAB_SONIC_CONTAINER_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-container:latest) MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) MACHINE_OS=debian-13.0 @@ -34,6 +35,10 @@ MAX_RETRIES := 30 ifeq ($(MINI_LAB_FLAVOR),sonic) LAB_TOPOLOGY=mini-lab.sonic.yaml MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true) +else ifeq ($(MINI_LAB_FLAVOR),container_sonic) +LAB_TOPOLOGY=mini-lab.container_sonic.yaml +MINI_LAB_SONIC_IMAGE=$(MINI_LAB_SONIC_CONTAINER_IMAGE) +MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true) else ifeq ($(MINI_LAB_FLAVOR),dell_sonic) LAB_TOPOLOGY=mini-lab.dell_sonic.yaml MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION) @@ -136,7 +141,10 @@ partition-bake: external_network ifeq ($(CI),true) docker pull $(MINI_LAB_SONIC_IMAGE) endif -ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),$(MINI_LAB_FLAVOR)) +ifeq ($(MINI_LAB_FLAVOR),container_sonic) +# tolerate a locally built image that is not (yet) available in the registry + docker pull $(MINI_LAB_SONIC_IMAGE) || true +else ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),$(MINI_LAB_FLAVOR)) docker pull $(MINI_LAB_SONIC_IMAGE) endif @if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \ @@ -188,6 +196,7 @@ cleanup-partition: mkdir -p clab-mini-lab sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.dell_sonic.yaml sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml + sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.container_sonic.yaml sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.capms.dell_sonic.yaml sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.kamaji.yaml docker network rm --force mini_lab_ext @@ -423,6 +432,19 @@ build-sonic-base: docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505 +# the docker-sonic-vs artifact is a gzipped `docker save` archive, so the base +# image is loaded and re-tagged instead of being built from a Dockerfile +.PHONY: build-sonic-container-base +build-sonic-container-base: + curl -fL -o docker-sonic-vs.gz "https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=202505&platform=vs&target=target%2Fdocker-sonic-vs.gz" + docker load -i docker-sonic-vs.gz + docker tag docker-sonic-vs:latest ghcr.io/metal-stack/mini-lab-sonic-container-base:202505 + rm -f docker-sonic-vs.gz + +.PHONY: build-sonic-container +build-sonic-container: + docker build -t ghcr.io/metal-stack/mini-lab-sonic-container:latest images/sonic-container + ## DEV TARGETS ## .PHONY: dev-env diff --git a/README.md b/README.md index c376a0db..7bae9720 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,8 @@ make up All available mini-lab flavors are listed below: -- `sonic`: runs two Community SONiC switches +- `sonic`: runs two Community SONiC switches (QEMU VMs wrapped in containers) +- `container_sonic`: runs two Community SONiC switches based on the official [docker-sonic-vs](https://github.com/sonic-net/sonic-buildimage/tree/master/platform/vs) container image (no VMs, faster startup and lower resource usage). The image is extended with systemd, sshd and dockerd so that metal-core and the other partition services can be deployed onto the switches just like on the VM-based flavors. - `dell_sonic`: runs two Enterprise SONiC switches with a [locally built vrnetlab image](https://github.com/srl-labs/vrnetlab/tree/master/dell/dell_sonic) - `capms_dell_sonic`: runs the `dell_sonic` flavor but with four instead of two machines (this is used for [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall) - `kamaji`: runs a variation of the `sonic` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`. diff --git a/deploy_partition.yaml b/deploy_partition.yaml index 2ad4de4d..a568d759 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -73,6 +73,9 @@ path: /etc/apt/sources.list search_string: deb [arch=amd64] http://deb.debian.org/debian/ bullseye-backports main contrib non-free line: deb [arch=amd64] http://archive.debian.org/debian/ bullseye-backports main contrib non-free + # container SONiC is based on bookworm. Without the search_string present, + # lineinfile would append the bullseye repository line + when: "'container_sonic' not in group_names" roles: - name: ansible-common tags: always diff --git a/files/sonic-container/frr-reload.service b/files/sonic-container/frr-reload.service new file mode 100644 index 00000000..b6ee26f7 --- /dev/null +++ b/files/sonic-container/frr-reload.service @@ -0,0 +1,13 @@ +# Overrides the frr-reload.service installed by the sonic-config role, which +# runs frr-reload via `docker exec bgp`. On container SONiC there is no bgp +# container and FRR runs directly under supervisord. +[Unit] +Description=Reload FRR + +[Service] +Type=oneshot +ExecStart=/usr/lib/frr/frr-reload +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/files/sonic-container/leaf01_config_db.json b/files/sonic-container/leaf01_config_db.json new file mode 100644 index 00000000..ad4e77f7 --- /dev/null +++ b/files/sonic-container/leaf01_config_db.json @@ -0,0 +1,68 @@ +{ + "AUTO_TECHSUPPORT": { + "GLOBAL": { + "state": "disabled" + } + }, + "DEVICE_METADATA": { + "localhost": { + "docker_routing_config_mode": "split-unified", + "hostname": "leaf01", + "hwsku": "Accton-AS7726-32X", + "mac": "00:00:00:00:00:00", + "platform": "x86_64-kvm_x86_64-r0", + "type": "LeafRouter" + } + }, + "FEATURE": { + "gnmi": { + "state": "disabled" + }, + "mgmt-framework": { + "state": "disabled" + }, + "snmp": { + "state": "disabled" + }, + "teamd": { + "state": "disabled" + } + }, + "PORT": { + "Ethernet0": { + "lanes": "1", + "alias": "Eth1/1", + "index": "1", + "speed": "25000", + "admin_status": "up", + "mtu": "9100" + }, + "Ethernet1": { + "lanes": "2", + "alias": "Eth1/2", + "index": "1", + "speed": "25000", + "admin_status": "up", + "mtu": "9100" + }, + "Ethernet120": { + "lanes": "121,122,123,124", + "alias": "Eth31", + "index": "31", + "speed": "100000", + "admin_status": "up", + "mtu": "9100" + } + }, + "MGMT_PORT": { + "eth0": { + "alias": "eth0", + "admin_status": "up" + } + }, + "VERSIONS": { + "DATABASE": { + "VERSION": "version_202311_03" + } + } +} diff --git a/files/sonic-container/leaf02_config_db.json b/files/sonic-container/leaf02_config_db.json new file mode 100644 index 00000000..dea9465c --- /dev/null +++ b/files/sonic-container/leaf02_config_db.json @@ -0,0 +1,68 @@ +{ + "AUTO_TECHSUPPORT": { + "GLOBAL": { + "state": "disabled" + } + }, + "DEVICE_METADATA": { + "localhost": { + "docker_routing_config_mode": "split-unified", + "hostname": "leaf02", + "hwsku": "Accton-AS7726-32X", + "mac": "00:00:00:00:00:00", + "platform": "x86_64-kvm_x86_64-r0", + "type": "LeafRouter" + } + }, + "FEATURE": { + "gnmi": { + "state": "disabled" + }, + "mgmt-framework": { + "state": "disabled" + }, + "snmp": { + "state": "disabled" + }, + "teamd": { + "state": "disabled" + } + }, + "PORT": { + "Ethernet0": { + "lanes": "1", + "alias": "Eth1/1", + "index": "1", + "speed": "25000", + "admin_status": "up", + "mtu": "9100" + }, + "Ethernet1": { + "lanes": "2", + "alias": "Eth1/2", + "index": "1", + "speed": "25000", + "admin_status": "up", + "mtu": "9100" + }, + "Ethernet120": { + "lanes": "121,122,123,124", + "alias": "Eth31", + "index": "31", + "speed": "100000", + "admin_status": "up", + "mtu": "9100" + } + }, + "MGMT_PORT": { + "eth0": { + "alias": "eth0", + "admin_status": "up" + } + }, + "VERSIONS": { + "DATABASE": { + "VERSION": "version_202311_03" + } + } +} diff --git a/images/sonic-container/Dockerfile b/images/sonic-container/Dockerfile new file mode 100644 index 00000000..cdf008f6 --- /dev/null +++ b/images/sonic-container/Dockerfile @@ -0,0 +1,76 @@ +# Container SONiC (docker-sonic-vs) image for the container_sonic flavor. +# +# The official docker-sonic-vs image runs all SONiC daemons under a single +# supervisord and ships without sshd, dockerd and systemd. mini-lab deploys +# metal-core, dhcp, pixiecore and the monitoring exporters onto the switches +# via Ansible over SSH as systemd-managed docker containers, so this image adds +# - systemd as PID 1 with supervisord wrapped in sonic.service +# - sshd with root key login (authorized_keys installed by sonic-init.sh) +# - dockerd (docker in docker, requires a privileged container) +# - lldpd (docker-sonic-vs ships without any LLDP daemon +# - stub units (bgp/database/syncd/swss.service, sonic.target) so the +# dependencies of metal-core's generated systemd unit resolve +ARG SONIC_BASE_URL=https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=202505&platform=vs +ARG FRR_RELOAD_URL=${SONIC_BASE_URL}&target=target%2Fdebs%2Fbookworm%2Ffrr-pythontools_10.3-sonic-0_all.deb + +FROM ghcr.io/metal-stack/mini-lab-sonic-container-base:202505 + +ARG FRR_RELOAD_URL +ADD "${FRR_RELOAD_URL}" /tmp/frr-pythontools.deb + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install --yes --no-install-recommends \ + dbus \ + docker.io \ + ethtool \ + fuse-overlayfs \ + iptables \ + jq \ + lldpd \ + systemd \ + systemd-sysv \ + && dpkg -i /tmp/frr-pythontools.deb \ + && rm -rf /var/lib/apt/lists/* /tmp/frr-pythontools.deb + +# same port naming as the VM flavor (single-lane Ethernet0..Ethernet119 plus +# Ethernet120/124), so the existing inventory (spine uplink Ethernet120) applies +ARG HWSKU_DIR=/usr/share/sonic/device/x86_64-kvm_x86_64-r0/Accton-AS7726-32X +COPY port_config.ini lanemap.ini ${HWSKU_DIR}/ + +# start.sh only brings up the FRR daemons that SONiC needs unconditionally. +# bgpd is left to the per-vendor bgp container which does not exist here +RUN sed -i '/supervisorctl start staticd/a\\nsupervisorctl start bgpd\n\n[ -f /etc/frr/frr.conf ] \&\& vtysh -b || true' /usr/bin/start.sh + +COPY frr-reload /usr/lib/frr/frr-reload +COPY sonic-init.sh healthcheck.sh /usr/local/bin/ +COPY daemon.json /etc/docker/daemon.json +COPY lldpd.conf /etc/lldpd.d/mini-lab.conf +COPY systemd/ /etc/systemd/system/ + +# sonic-configdb-utils and parts of the SONiC CLI read platform information +# from this file. On the VM flavor it is written by launch.py +RUN printf 'SONIC_VERSION=202505\nPLATFORM=x86_64-kvm_x86_64-r0\nHWSKU=Accton-AS7726-32X\nDEVICE_TYPE=LeafRouter\nASIC_TYPE=vs\n' >/etc/sonic/sonic-environment + +# metal-core and the sonic-config role manage the FRR config in /etc/sonic/frr, +# which on SONiC is bind-mounted into the bgp container as /etc/frr. +# FRR runs directly here, so both paths must point to the same files +RUN mkdir -p /etc/sonic/frr \ + && mv /etc/frr/* /etc/sonic/frr/ \ + && rmdir /etc/frr \ + && ln -s /etc/sonic/frr /etc/frr + +RUN chmod +x /usr/lib/frr/frr-reload /usr/local/bin/sonic-init.sh /usr/local/bin/healthcheck.sh \ + && systemctl enable ssh docker lldpd sonic-init.service sonic.service sonic.target \ + database.service bgp.service swss.service syncd.service \ +# mask units that collide with supervisord-managed services. These are services that are pulled in +# from the debian base image and previously inert SONiC services, that were enabled by pulling in systemd as PID 1 + && systemctl mask getty@tty1.service console-getty.service \ + redis-server.service rsyslog.service frr.service networking.service \ + caclmgrd.service procdockerstatsd.service \ + determine-reboot-cause.service process-reboot-cause.service \ + aaastatsd.timer featured.timer hostcfgd.timer \ + && mkdir -p /root/.ssh && chmod 700 /root/.ssh + +HEALTHCHECK --start-period=60s --interval=5s --retries=60 CMD /usr/local/bin/healthcheck.sh + +ENTRYPOINT ["/sbin/init"] diff --git a/images/sonic-container/daemon.json b/images/sonic-container/daemon.json new file mode 100644 index 00000000..514d027b --- /dev/null +++ b/images/sonic-container/daemon.json @@ -0,0 +1,3 @@ +{ + "storage-driver": "fuse-overlayfs" +} diff --git a/images/sonic-container/frr-reload b/images/sonic-container/frr-reload new file mode 100644 index 00000000..cd3f0f01 --- /dev/null +++ b/images/sonic-container/frr-reload @@ -0,0 +1,2 @@ +#!/bin/sh +exec python3 /usr/lib/frr/frr-reload.py --reload /etc/frr/frr.conf diff --git a/images/sonic-container/healthcheck.sh b/images/sonic-container/healthcheck.sh new file mode 100644 index 00000000..25deb1d6 --- /dev/null +++ b/images/sonic-container/healthcheck.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Healthy once redis answers and every configured port is oper up in APPL_DB. +# Containerlab's healthy stage blocks the deploy (and with it the Ansible +# provisioning) until then, like the LLDP-based readiness check of the VM +# flavor did. +set -eu + +sonic-db-cli PING | grep -q PONG + +ports=$(sonic-db-cli CONFIG_DB KEYS 'PORT|*') +[ -n "${ports}" ] || exit 1 + +for port in ${ports}; do + port=${port#PORT|} + status=$(sonic-db-cli APPL_DB HGET "PORT_TABLE:${port}" oper_status) + if [ "${status}" != "up" ]; then + echo "${port} is not oper up" + exit 1 + fi +done diff --git a/images/sonic-container/lanemap.ini b/images/sonic-container/lanemap.ini new file mode 100644 index 00000000..a75069e0 --- /dev/null +++ b/images/sonic-container/lanemap.ini @@ -0,0 +1,3 @@ +eth1:1 +eth2:2 +eth3:121,122,123,124 diff --git a/images/sonic-container/lldpd.conf b/images/sonic-container/lldpd.conf new file mode 100644 index 00000000..b4235363 --- /dev/null +++ b/images/sonic-container/lldpd.conf @@ -0,0 +1,4 @@ +# metal-hammer only accepts neighbors with port id subtype mac or local. +# The local port ids are configured per port by roles/sonic/tasks/container.yaml +configure system interface pattern Ethernet* +configure lldp tx-interval 10 diff --git a/images/sonic-container/port_config.ini b/images/sonic-container/port_config.ini new file mode 100644 index 00000000..acc1f3d2 --- /dev/null +++ b/images/sonic-container/port_config.ini @@ -0,0 +1,123 @@ +# name lanes alias index speed +Ethernet0 1 Eth1/1 1 25000 +Ethernet1 2 Eth1/2 1 25000 +Ethernet2 3 Eth1/3 1 25000 +Ethernet3 4 Eth1/4 1 25000 +Ethernet4 5 Eth2/1 2 25000 +Ethernet5 6 Eth2/2 2 25000 +Ethernet6 7 Eth2/3 2 25000 +Ethernet7 8 Eth2/4 2 25000 +Ethernet8 9 Eth3/1 3 25000 +Ethernet9 10 Eth3/2 3 25000 +Ethernet10 11 Eth3/3 3 25000 +Ethernet11 12 Eth3/4 3 25000 +Ethernet12 13 Eth4/1 4 25000 +Ethernet13 14 Eth4/2 4 25000 +Ethernet14 15 Eth4/3 4 25000 +Ethernet15 16 Eth4/4 4 25000 +Ethernet16 17 Eth5/1 5 25000 +Ethernet17 18 Eth5/2 5 25000 +Ethernet18 19 Eth5/3 5 25000 +Ethernet19 20 Eth5/4 5 25000 +Ethernet20 21 Eth6/1 6 25000 +Ethernet21 22 Eth6/2 6 25000 +Ethernet22 23 Eth6/3 6 25000 +Ethernet23 24 Eth6/4 6 25000 +Ethernet24 25 Eth7/1 7 25000 +Ethernet25 26 Eth7/2 7 25000 +Ethernet26 27 Eth7/3 7 25000 +Ethernet27 28 Eth7/4 7 25000 +Ethernet28 29 Eth8/1 8 25000 +Ethernet29 30 Eth8/2 8 25000 +Ethernet30 31 Eth8/3 8 25000 +Ethernet31 32 Eth8/4 8 25000 +Ethernet32 33 Eth9/1 9 25000 +Ethernet33 34 Eth9/2 9 25000 +Ethernet34 35 Eth9/3 9 25000 +Ethernet35 36 Eth9/4 9 25000 +Ethernet36 37 Eth10/1 10 25000 +Ethernet37 38 Eth10/2 10 25000 +Ethernet38 39 Eth10/3 10 25000 +Ethernet39 40 Eth10/4 10 25000 +Ethernet40 41 Eth11/1 11 25000 +Ethernet41 42 Eth11/2 11 25000 +Ethernet42 43 Eth11/3 11 25000 +Ethernet43 44 Eth11/4 11 25000 +Ethernet44 45 Eth12/1 12 25000 +Ethernet45 46 Eth12/2 12 25000 +Ethernet46 47 Eth12/3 12 25000 +Ethernet47 48 Eth12/4 12 25000 +Ethernet48 49 Eth13/1 13 25000 +Ethernet49 50 Eth13/2 13 25000 +Ethernet50 51 Eth13/3 13 25000 +Ethernet51 52 Eth13/4 13 25000 +Ethernet52 53 Eth14/1 14 25000 +Ethernet53 54 Eth14/2 14 25000 +Ethernet54 55 Eth14/3 14 25000 +Ethernet55 56 Eth14/4 14 25000 +Ethernet56 57 Eth15/1 15 25000 +Ethernet57 58 Eth15/2 15 25000 +Ethernet58 59 Eth15/3 15 25000 +Ethernet59 60 Eth15/4 15 25000 +Ethernet60 61 Eth16/1 16 25000 +Ethernet61 62 Eth16/2 16 25000 +Ethernet62 63 Eth16/3 16 25000 +Ethernet63 64 Eth16/4 16 25000 +Ethernet64 65 Eth17/1 17 25000 +Ethernet65 66 Eth17/2 17 25000 +Ethernet66 67 Eth17/3 17 25000 +Ethernet67 68 Eth17/4 17 25000 +Ethernet68 69 Eth18/1 18 25000 +Ethernet69 70 Eth18/2 18 25000 +Ethernet70 71 Eth18/3 18 25000 +Ethernet71 72 Eth18/4 18 25000 +Ethernet72 73 Eth19/1 19 25000 +Ethernet73 74 Eth19/2 19 25000 +Ethernet74 75 Eth19/3 19 25000 +Ethernet75 76 Eth19/4 19 25000 +Ethernet76 77 Eth20/1 20 25000 +Ethernet77 78 Eth20/2 20 25000 +Ethernet78 79 Eth20/3 20 25000 +Ethernet79 80 Eth20/4 20 25000 +Ethernet80 81 Eth21/1 21 25000 +Ethernet81 82 Eth21/2 21 25000 +Ethernet82 83 Eth21/3 21 25000 +Ethernet83 84 Eth21/4 21 25000 +Ethernet84 85 Eth22/1 22 25000 +Ethernet85 86 Eth22/2 22 25000 +Ethernet86 87 Eth22/3 22 25000 +Ethernet87 88 Eth22/4 22 25000 +Ethernet88 89 Eth23/1 23 25000 +Ethernet89 90 Eth23/2 23 25000 +Ethernet90 91 Eth23/3 23 25000 +Ethernet91 92 Eth23/4 23 25000 +Ethernet92 93 Eth24/1 24 25000 +Ethernet93 94 Eth24/2 24 25000 +Ethernet94 95 Eth24/3 24 25000 +Ethernet95 96 Eth24/4 24 25000 +Ethernet96 97 Eth25/1 25 25000 +Ethernet97 98 Eth25/2 25 25000 +Ethernet98 99 Eth25/3 25 25000 +Ethernet99 100 Eth25/4 25 25000 +Ethernet100 101 Eth26/1 26 25000 +Ethernet101 102 Eth26/2 26 25000 +Ethernet102 103 Eth26/3 26 25000 +Ethernet103 104 Eth26/4 26 25000 +Ethernet104 105 Eth27/1 27 25000 +Ethernet105 106 Eth27/2 27 25000 +Ethernet106 107 Eth27/3 27 25000 +Ethernet107 108 Eth27/4 27 25000 +Ethernet108 109 Eth28/1 28 25000 +Ethernet109 110 Eth28/2 28 25000 +Ethernet110 111 Eth28/3 28 25000 +Ethernet111 112 Eth28/4 28 25000 +Ethernet112 113 Eth29/1 29 25000 +Ethernet113 114 Eth29/2 29 25000 +Ethernet114 115 Eth29/3 29 25000 +Ethernet115 116 Eth29/4 29 25000 +Ethernet116 117 Eth30/1 30 25000 +Ethernet117 118 Eth30/2 30 25000 +Ethernet118 119 Eth30/3 30 25000 +Ethernet119 120 Eth30/4 30 25000 +Ethernet120 121,122,123,124 Eth31 31 100000 +Ethernet124 125,126,127,128 Eth32 32 100000 diff --git a/images/sonic-container/sonic-init.sh b/images/sonic-container/sonic-init.sh new file mode 100644 index 00000000..9e0b438d --- /dev/null +++ b/images/sonic-container/sonic-init.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +# docker bind-mounts /etc/resolv.conf, /etc/hostname and /etc/hosts, which +# breaks ansible modules that replace files via atomic rename, so +# turn them into regular files +make_regular_file() { + mountpoint -q "$1" || return 0 + cp "$1" "$1.unmounted" + umount "$1" + mv "$1.unmounted" "$1" +} + +wait_for_interfaces() { + # Recovers the number of containerlab interfaces, then waits until all interfaces are ready + # necessary in this image because we replace PID 1 with systemd + clab_intfs=$(tr '\0' '\n' /dev/null | wc -l)" -lt "${expected}" ]; do + sleep 1 + done +} + +# seed the initial config_db. sonic-config regenerates this file later, so it +# must be a copy rather than a bind mount at the target path +seed_config_db() { + [ ! -f /etc/sonic/config_db.json ] || return 0 + [ -f /config_db.json.init ] || return 0 + + mkdir -p /etc/sonic + mac=$(cat /sys/class/net/eth0/address) + jq --arg mac "${mac}" '.DEVICE_METADATA.localhost.mac = $mac' \ + /config_db.json.init >/etc/sonic/config_db.json +} + +install_root_ssh_key() { + [ -f /authorized_keys ] || return 0 + + mkdir -p /root/.ssh + chmod 700 /root/.ssh + cp /authorized_keys /root/.ssh/authorized_keys + chown root:root /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys +} + +for f in /etc/resolv.conf /etc/hostname /etc/hosts; do + make_regular_file "${f}" +done +wait_for_interfaces +seed_config_db +install_root_ssh_key diff --git a/images/sonic-container/systemd/bgp.service b/images/sonic-container/systemd/bgp.service new file mode 100644 index 00000000..c7c99933 --- /dev/null +++ b/images/sonic-container/systemd/bgp.service @@ -0,0 +1,11 @@ +[Unit] +Description=Stub for SONiC bgp.service +PartOf=sonic.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true + +[Install] +WantedBy=sonic.target diff --git a/images/sonic-container/systemd/database.service b/images/sonic-container/systemd/database.service new file mode 100644 index 00000000..b41e29a6 --- /dev/null +++ b/images/sonic-container/systemd/database.service @@ -0,0 +1,11 @@ +[Unit] +Description=Stub for SONiC database.service +PartOf=sonic.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true + +[Install] +WantedBy=sonic.target diff --git a/images/sonic-container/systemd/sonic-init.service b/images/sonic-container/systemd/sonic-init.service new file mode 100644 index 00000000..64f72b34 --- /dev/null +++ b/images/sonic-container/systemd/sonic-init.service @@ -0,0 +1,11 @@ +[Unit] +Description=Prepare SONiC container +Before=sonic.service ssh.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/bin/sonic-init.sh + +[Install] +WantedBy=multi-user.target diff --git a/images/sonic-container/systemd/sonic.service b/images/sonic-container/systemd/sonic.service new file mode 100644 index 00000000..e50ec25e --- /dev/null +++ b/images/sonic-container/systemd/sonic.service @@ -0,0 +1,13 @@ +[Unit] +Description=SONiC services (supervisord) +Requires=sonic-init.service +After=sonic-init.service network.target + +[Service] +# start.sh needs these because the container environment does not reach systemd units +Environment="PLATFORM=x86_64-kvm_x86_64-r0" "HWSKU=Accton-AS7726-32X" +ExecStart=/usr/local/bin/supervisord +Restart=always + +[Install] +WantedBy=sonic.target diff --git a/images/sonic-container/systemd/sonic.target b/images/sonic-container/systemd/sonic.target new file mode 100644 index 00000000..fc06695a --- /dev/null +++ b/images/sonic-container/systemd/sonic.target @@ -0,0 +1,6 @@ +[Unit] +Description=SONiC target +Wants=sonic.service + +[Install] +WantedBy=multi-user.target diff --git a/images/sonic-container/systemd/swss.service b/images/sonic-container/systemd/swss.service new file mode 100644 index 00000000..8f00d307 --- /dev/null +++ b/images/sonic-container/systemd/swss.service @@ -0,0 +1,11 @@ +[Unit] +Description=Stub for SONiC swss.service +PartOf=sonic.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true + +[Install] +WantedBy=sonic.target diff --git a/images/sonic-container/systemd/syncd.service b/images/sonic-container/systemd/syncd.service new file mode 100644 index 00000000..0a29788e --- /dev/null +++ b/images/sonic-container/systemd/syncd.service @@ -0,0 +1,11 @@ +[Unit] +Description=Stub for SONiC syncd.service +PartOf=sonic.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true + +[Install] +WantedBy=sonic.target diff --git a/inventories/partition.yaml b/inventories/partition.yaml index fb3aa8a3..8d0bb0e4 100644 --- a/inventories/partition.yaml +++ b/inventories/partition.yaml @@ -7,9 +7,14 @@ partition: localhost: ansible_python_interpreter: "{{ ansible_playbook_python }}" children: + container_sonic: dell_sonic: leaves: +container_sonic: + vars: + ansible_group_priority: 10 + dell_sonic: vars: ansible_group_priority: 10 diff --git a/mini-lab.container_sonic.yaml b/mini-lab.container_sonic.yaml new file mode 100644 index 00000000..32b8ea13 --- /dev/null +++ b/mini-lab.container_sonic.yaml @@ -0,0 +1,80 @@ +name: mini-lab +prefix: "" + +mgmt: + network: bridge + +topology: + defaults: + kind: linux + nodes: + exit: + image: quay.io/frrouting/frr:10.3.0 + network-mode: none + binds: + - files/exit/daemons:/etc/frr/daemons + - files/exit/frr.conf:/etc/frr/frr.conf + - files/exit/vtysh.conf:/etc/frr/vtysh.conf + - files/exit/network.sh:/root/network.sh + exec: + - sh /root/network.sh + external_service: + image: docker.io/library/nginx:alpine-slim + network-mode: none + binds: + - files/external_service/network.sh:/root/network.sh + exec: + - sh /root/network.sh + mini_lab_ext: + kind: bridge + leaf01: + group: leaves + image: ${MINI_LAB_SONIC_IMAGE} + labels: + ansible-group: container_sonic + binds: + - files/ssh/id_ed25519.pub:/authorized_keys:ro + - files/sonic-container/leaf01_config_db.json:/config_db.json.init:ro + stages: + healthy: + exec: + # date is a dummy command. To run an on-exit exec + # stage, containerlab must wait for the docker healthcheck to pass + # so the deploy blocks until the switch is actually ready. This is + # needed because sshd answers way before the switch is ready + - command: date + phase: on-exit + leaf02: + group: leaves + image: ${MINI_LAB_SONIC_IMAGE} + labels: + ansible-group: container_sonic + binds: + - files/ssh/id_ed25519.pub:/authorized_keys:ro + - files/sonic-container/leaf02_config_db.json:/config_db.json.init:ro + stages: + healthy: + exec: + - command: date + phase: on-exit + machine01: + group: machines + image: ${MINI_LAB_VM_IMAGE} + env: + UUID: 00000000-0000-0000-0000-000000000001 + machine02: + group: machines + image: ${MINI_LAB_VM_IMAGE} + env: + UUID: 00000000-0000-0000-0000-000000000002 + links: + - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"] + mtu: 9000 + - endpoints: ["external_service:mini_lab_ext", "mini_lab_ext:external_service"] + mtu: 9000 + - endpoints: ["leaf01:eth1", "machine01:lan0"] # Ethernet0 + - endpoints: ["leaf02:eth1", "machine01:lan1"] # Ethernet0 + - endpoints: ["leaf01:eth2", "machine02:lan0"] # Ethernet1 + - endpoints: ["leaf02:eth2", "machine02:lan1"] # Ethernet1 + - endpoints: ["leaf01:eth3", "exit:eth1"] # Ethernet120 + - endpoints: ["leaf02:eth3", "exit:eth2"] # Ethernet120 diff --git a/roles/sonic/tasks/container.yaml b/roles/sonic/tasks/container.yaml new file mode 100644 index 00000000..d826c416 --- /dev/null +++ b/roles/sonic/tasks/container.yaml @@ -0,0 +1,40 @@ +--- +# the sonic-config role installs a frr-reload.service that reloads FRR via +# `docker exec bgp`. On container SONiC FRR runs under supervisord, so +# override the unit with one that calls frr-reload directly +- name: Override frr-reload.service + ansible.builtin.copy: + src: "{{ playbook_dir }}/files/sonic-container/frr-reload.service" + dest: /etc/systemd/system/frr-reload.service + mode: "0644" + +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: true + +# metal-hammer only accepts LLDP neighbors with port id subtype mac or local, +# and metal-api matches the advertised port id against the switch port alias. +# On the VM flavor SONiC's lldpmgrd derives this configuration from the PORT +# table in the CONFIG_DB, so do the same here +- name: Read port aliases from CONFIG_DB + ansible.builtin.shell: | + set -o pipefail + for port in $(sonic-db-cli CONFIG_DB KEYS 'PORT|*'); do + alias=$(sonic-db-cli CONFIG_DB HGET "${port}" alias) + if [ -n "${alias}" ]; then + echo "${port#PORT|} ${alias}" + fi + done + args: + executable: /bin/bash + register: sonic_port_aliases + changed_when: false + +- name: Configure LLDP port IDs + ansible.builtin.command: "lldpcli configure ports {{ item.split(' ')[0] }} lldp portidsubtype local {{ item.split(' ')[1] }}" + retries: 10 + delay: 3 + register: result + until: result.rc == 0 + changed_when: true + with_items: "{{ sonic_port_aliases.stdout_lines }}" diff --git a/roles/sonic/tasks/main.yaml b/roles/sonic/tasks/main.yaml index c8ee8460..1c7866b3 100644 --- a/roles/sonic/tasks/main.yaml +++ b/roles/sonic/tasks/main.yaml @@ -1,6 +1,13 @@ --- +# on container SONiC frr-pythontools and the frr-reload script are baked into +# the image because FRR runs under supervisord instead of a bgp container - name: Install frr-pythontools ansible.builtin.import_tasks: frr-reload.yaml + when: "'container_sonic' not in group_names" + +- name: Adapt services for container SONiC + ansible.builtin.import_tasks: container.yaml + when: "'container_sonic' in group_names" - name: Fix Network Performance ansible.builtin.import_tasks: fix-network-performance.yaml @@ -31,7 +38,9 @@ ansible.builtin.import_tasks: mock-platform.yaml # ntp restarting for monitoring -> otherwise some NodeTimeOutOfSync error +# container SONiC has no chrony and the clock is shared with the host kernel - name: restart chrony systemd: name: chrony state: restarted + when: "'container_sonic' not in group_names"