Firmware-CI testing

.firmwareci/.jinja2_templates/defaults.j2

@@ -0,0 +1,14 @@
+## Can be changed depending on the FWCI tester device used
+{%- set RPI_HOSTNAME = "fwci-dutctl-tester" %}
+{%- set RPI_FQDN = RPI_HOSTNAME ~ ".sec.9e.network" %}
+## Do not change values below, unless you know what you are doing
+{%- set PKG_NAME = "dutctl" %}
+{%- set DUTCTL_CONFIG_FILE = "/etc/dutctl.yaml" %}
+{%- set SYSTEMD_FILE = "dutagent.service" %}
+{%- set DUTAGENT_PORT = "2024" %}
+{%- set DUTCTL_ARGS = '"-s", "' ~ RPI_HOSTNAME ~ ':' ~ DUTAGENT_PORT ~ '"' %}
+{%- set DUTCTL_ARGS_STR = '-s ' ~ RPI_HOSTNAME ~ ':' ~ DUTAGENT_PORT %}
+## Values below should not change
+{%- set GPIO_PIN_TEST_OLIMEX = "21" %}
+{%- set FAKE_SERIAL_INPUT = "/home/oscar/ttyS1" %}
+{%- set FAKE_SERIAL_OUTPUT = "/home/oscar/ttyS2" %}

.firmwareci/.jinja2_templates/defaults.yaml

@@ -0,0 +1,15 @@
+transport_oscar: &transport_oscar
+  proto: ssh
+  options:
+    host: "[[attributes.Host]]"
+    user: oscar
+    identity_file: "/root/.ssh/fwci"
+    # TODO: The ssh keys (identity_file) are hard-coded in the FWCI docker at the moment
+
+transport_root: &transport_root
+  proto: ssh
+  options:
+    host: "[[attributes.Host]]"
+    user: root
+    identity_file: "/root/.ssh/fwci"
+    # TODO: The ssh keys (identity_file) are hard-coded in the FWCI docker at the moment

.firmwareci/.jinja2_templates/post.yaml.j2

@@ -0,0 +1,33 @@
+post-stage:
+  # ===========================================
+  #  Delete the files copied over for the test
+  # ===========================================
+  - cmd: shell
+    name: Stop remotelab-server systemd service
+    transport: *transport_root
+    parameters:
+      command: "systemctl stop {{ defaults.SYSTEMD_FILE }} || true"
+
+  - cmd: shell
+    name: Uninstall the package
+    transport: *transport_root
+    parameters:
+      command: "apt remove --assume-yes {{ defaults.PKG_NAME }} || true"
+
+  - cmd: shell
+    name: Delete testing configuration file
+    transport: *transport_root
+    parameters:
+      command: "rm {{ defaults.DUTCTL_CONFIG_FILE }} || true"
+
+  - cmd: shell
+    name: Delete package
+    transport: *transport_root
+    parameters:
+      command: "rm {{ defaults.PKG_NAME }}.deb || true"
+
+  - cmd: shell
+    name: Shutdown fake serial
+    transport: *transport_oscar
+    parameters:
+      command: "pkill socat || true"

.firmwareci/.jinja2_templates/pre.yaml.j2

@@ -0,0 +1,76 @@
+pre-stage:
+  # Fix permissions
+  - cmd: cmd
+    name: chmod dutctl
+    transport:
+      proto: local
+    parameters:
+      executable: chmod
+      args: ["755", "[[input.DUTCTL]]"]
+
+  # ========================
+  #  Copy files over to RPI
+  # ========================
+  - cmd: copy
+    name: Copy over {{ defaults.PKG_NAME }} debian package
+    transport: *transport_root
+    parameters:
+      source: "[[input.REMOTE_PKG]]"
+      destination: "{{ defaults.PKG_NAME }}.deb"
+
+  - cmd: cmd
+    name: Install {{ defaults.PKG_NAME }} package
+    transport: *transport_root
+    parameters:
+      executable: dpkg
+      args: ["-i", "{{ defaults.PKG_NAME }}.deb"]
+
+  - cmd: copy
+    name: Copy over test config file
+    transport: *transport_root
+    parameters:
+      source: "[[input.CONFIG]]"
+      destination: {{ defaults.DUTCTL_CONFIG_FILE }}
+
+  - cmd: cmd
+    name: chmod config file
+    transport: *transport_root
+    parameters:
+      executable: chmod
+      args: ["644", {{ defaults.DUTCTL_CONFIG_FILE }}]
+
+  # ==============
+  #  Start server
+  # ==============
+  - cmd: cmd
+    name: Reload systemctl daemon
+    transport: *transport_root
+    parameters:
+      executable: systemctl
+      args: ["daemon-reload"]
+
+  - cmd: cmd
+    name: Start systemd service
+    transport: *transport_root
+    parameters:
+      executable: systemctl
+      args: ["start", {{ defaults.SYSTEMD_FILE }}]
+
+  - cmd: shell
+    name: Status systemd service
+    transport: *transport_root
+    parameters:
+      command: "systemctl status --no-pager {{ defaults.SYSTEMD_FILE }}"
+      expect:
+        - regex: "Active: active \\(running\\)"
+
+  # ==========================
+  #  Test dutctl help command
+  # ==========================
+  - cmd: cmd
+    name: dutctl help
+    transport:
+      proto: local
+    parameters:
+      executable: "[[input.DUTCTL]]"
+      args: ["--help"]

.firmwareci/README.md

@@ -0,0 +1,108 @@
+# FirmwareCI configuration for dutctl testing
+
+This directory contains FirmwareCI configurations and tests.
+
+For detailed information on FirmwareCI, please refer to the [official documentation](https://docs.firmware-ci.com/).
+
+
+## Requirements
+
+- Parametric and re-usable testing
+- Easy to expand
+- Reliable
+- Can be used in CI/CD to verify each pull request
+
+
+## Design decisions
+
+### Jinja templating
+We want to test multiple features (flashing, power control, serial, etc). And to keep everything nicely organised we need to have each feature-specific test as it's own test.
+
+However, the setup and tear-down of the tests is rather complex:
+- copy over the compiled binaries
+- copy over configuration files
+- spin up the server (agent)
+- execute the test
+- shut-down the server
+- delete the copied binaries
+- delete configuration files
+
+In case of the serial feature, it also means spinning-up "fake" / "dummy" serial that can be used for the testing.
+
+It would be stupid to copy-paste all this setup into each test file, especially because keeping all of them in sync would be a nightmare.
+
+For this reason I have decided to use [jinja2 templates](https://jinja.palletsprojects.com/en/stable/api/) to automate this menial and error-prone work. To make it even simpler, I have also included [Taskfile](https://taskfile.dev/docs/guide) which has already all the jobs written there, so to generate all of the templates is as easy as simply calling `task jinja2:templates`.
+
+Thanks to jinja templating, and how the tests are structured, tests are parametric and easy to expand.
+
+
+### Debian packaging magic
+As you might notice, instead of simply copying over the compiled binaries (`dutctl` / `dutagent` / ...) we use Debian packages.
+
+The biggest motivator here to do this was to accommodate for setup and tear-down, and the differences between tested versions (future-proofing). This way, we leverage the power of package manager to make sure that the cleanup (uninstalling) of old file is complete and that no files are left behind! This makes sure that the environment is always pristine.
+
+Another huge advantage is, that with this approach, that run-time dependencies are also handled by package manager. Because of this, the test hardware (Raspberry Pi) has minimal (if any) setup required for use in the FWCI tests. Just install package, test and uninstall! Easy!
+
+
+## Project structure
+
+The most important files / directories are:
+- `.firmwareci/Taskfile.yml`
+- `.firmwareci/.jinja2_templates/`
+- `.firmwareci/workflows/`
+
+### Taskfile
+The `.firmwareci/Taskfile.yml` is there to help and automate some mundane and repetitive jobs.
+
+Contains task `jinja2:templates` which finds all jinja tempaltes in `.firmwareci/workflows/` and `.firmwareci/duts/` and runs a tempalting engine on them.
+
+The list of the templates is created dynamically with the shell command:
+```bash
+find './workflows/' './duts/' -type f -name '*.yaml.j2'
+```
+
+This list is then iterated over in a for-loop, such as that it takes `./duts/dut-rpi-fti-tester/dut.yaml.j2` as input, and produces `./duts/dut-rpi-fti-tester/dut.yaml`.
+
+
+### jinja2_templates
+The `.firmwareci/.jinja2_templates` contains the "common" building-blocks that are shared across templates. The `defaults.j2` and `defaults.yaml` are for storing some basics (they are intentionally separate):
+- `defaults.j2`
+    - this is pure jinja2 syntax, and is imported into template with
+      ```j2
+      {%- import ".jinja2_templates/defaults.j2" as defaults %}
+      ```
+    - import means that the variables will become accessible, and can then be accessed in the template with:
+      ```j2
+      {{ defaults.DUTCTL_ARGS }}
+      ```
+- `defaults.yaml`
+    - this is pure YAML file to be `include`d into a template with
+	  ```j2
+	  {% include ".jinja2_templates/defaults.yaml" with context %}
+	  ```
+	- essentially this is to just simply render the content of the file, ignoring any variable assignments or macros within it
+	- technically speaking it could be renamed to `defaults.yaml.j2`, but it does not contain any jinja2 code and renaming it would have no effect
+- `pre.yaml.j2` and `post.yaml.j2`
+    - for pre-stage and post-stage respectively
+    - they are copy-pasted into each test (they are `include`d in tests, meaning that their content will be rendered)
+
+
+### workflows
+The `workflows` directory is where the tests live.
+
+### Systemd service
+The `dutagent.service` file is a systemd service file to start a dutagent on dutworker.
+
+At the moment it is only used for firmware-ci testing.
+
+
+## Adding completely new test
+To add a completely new tests from scratch, create a new `.yaml.j2` file in `.firmwareci/workflows/workflow-rpi-dutctl-tester/`. I would recommend to copy-paste some existing test (for example the `dummy-modules.yaml.j2` is rather simple) and use it as basis.
+
+Then update the test according to your needs.
+
+Run `task fwci:jinja2:templates` and it will automatically generate a complete tests. That is it. It is as simple as that.
+
+Next, I recommend to run `fwci fwci:validate`.
+
+As the last step, add all new files (even those jinja2 generated) into git commit. This is because FirmwareCI cannot pre-process the tests on it's own, which is why we have to include even the "generated code".

.firmwareci/Taskfile.yml

@@ -0,0 +1,50 @@
+---
+version: "3"
+
+tasks:
+  jinja2:templates:
+    desc: Run jinja2 on template files
+    cmds:
+      - task: jinja2:find-and-process-all-tempaltes
+
+  fwci:validate:
+    desc: Validate FWCI files
+    cmds:
+      - fwci validate
+
+  #================
+  # Internal tasks
+  #================
+
+  jinja2:find-and-process-all-tempaltes:
+    desc: Run jinja2 on jinja2 templates
+    internal: true
+    label: 'jinja2-tests-{{.REPO}}'
+    vars:
+      WORKFLOWS:
+        sh: find './workflows/' './duts/' -type f -name '*.yaml.j2'
+    sources:
+      - .jinja2_templates/*
+      - duts/**/*.yaml.j2
+      - workflows/workflow-*/defaults.j2
+      - workflows/workflow-*/**/*.yaml.j2
+    cmds:
+      - for:
+          var: WORKFLOWS
+        task: jinja2:run-on-file
+        vars:
+          INPUTFILE:
+            sh: echo "{{.ITEM}}"
+          OUTPUTFILE:
+            sh: echo "{{.ITEM}}" | sed -E 's/\.j2$//g'
+
+  jinja2:run-on-file:
+    desc: Run jinja2 on a file
+    internal: true
+    cmds:
+      - pwd
+      - echo 'INPUTFILE = {{.INPUTFILE}}'
+      - echo 'OUTPUTFILE = {{.OUTPUTFILE}}'
+      - j2 --format env -o '{{.OUTPUTFILE}}' '{{.INPUTFILE}}'
+    requires:
+      vars: ["INPUTFILE", "OUTPUTFILE"]

.firmwareci/dutagent.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=dutagent to run on SBC attached to a devices under test
+Documentation=https://github.com/BlindspotSoftware/dutctl
+Wants=network-online.target
+After=network-online.target
+StartLimitInterval=15
+StartLimitBurst=3
+
+[Service]
+Restart=always
+RestartSec=3s
+ExecStart=/usr/bin/dutagent -a 0.0.0.0:2024 -c /etc/dutctl.yaml
+
+[Install]
+WantedBy=multi-user.target

.firmwareci/duts/dut-rpi-dutctl-tester/dut.yaml.j2

@@ -0,0 +1,7 @@
+---
+# Device-under-Test Configuration for RaspberryPi acting as dutctl tester
+{%- import ".jinja2_templates/defaults.j2" as defaults %}
+name: {{ defaults.RPI_HOSTNAME }}
+label: {{ defaults.RPI_HOSTNAME }}
+attributes:
+  Host: "{{ defaults.RPI_FQDN }}"