From 3923d29ea8424c50533f1030c0a92f76d00c20a4 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Sun, 15 Jun 2025 16:31:01 +0000 Subject: [PATCH] chore: support pegboard container runner inside docker compose --- .github/workflows/release.yaml | 28 -- Cargo.lock | 22 ++ docker/dev-full/docker-compose.yml | 15 +- docker/dev-full/rivet-client/config.jsonc | 6 +- docker/dev-full/rivet-client/entrypoint.sh | 320 ++++++++++++++++++ .../rivet-client/rivet-actor.conflist | 42 +++ .../rivet-client/rivet-setup-networking.sh | 270 +++++++++++++++ docker/universal/Dockerfile | 27 +- justfile | 4 + .../files/pegboard_configure.sh | 2 + .../edge/infra/client/config/src/manager.rs | 13 + .../client/manager/src/actor/oci_config.rs | 41 ++- .../infra/client/manager/src/actor/setup.rs | 1 + 13 files changed, 729 insertions(+), 62 deletions(-) create mode 100755 docker/dev-full/rivet-client/entrypoint.sh create mode 100644 docker/dev-full/rivet-client/rivet-actor.conflist create mode 100644 docker/dev-full/rivet-client/rivet-setup-networking.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6084b2fa10..6b98fb26d1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -248,34 +248,6 @@ jobs: secret-files: | netrc=${{ runner.temp }}/netrc - - name: Build & Push (rivetgg/client:isolate-v8-runner) - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: rivetgg/rivet-client:isolate-v8-runner-${{ steps.vars.outputs.sha_short }}${{ matrix.arch_suffix }} - file: docker/universal/Dockerfile - target: isolate-v8-runner - platforms: ${{ matrix.platform }} - secrets: | - fontawesome_package_token=${{ secrets.FONTAWESOME_PACKAGE_TOKEN }} - secret-files: | - netrc=${{ runner.temp }}/netrc - - - name: Build & Push (rivetgg/client:container-runner) - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: rivetgg/rivet-client:container-runner-${{ steps.vars.outputs.sha_short }}${{ matrix.arch_suffix }} - file: docker/universal/Dockerfile - target: container-runner - platforms: ${{ matrix.platform }} - secrets: | - fontawesome_package_token=${{ secrets.FONTAWESOME_PACKAGE_TOKEN }} - secret-files: | - netrc=${{ runner.temp }}/netrc - - name: Build & Push (rivetgg/rivet:monolith) uses: docker/build-push-action@v4 with: diff --git a/Cargo.lock b/Cargo.lock index 49bf99e734..29e970c310 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,9 +717,11 @@ version = "25.4.2" dependencies = [ "api-helper", "async-trait", + "build", "chirp-client", "chirp-workflow", "chrono", + "cluster", "fdb-util", "foundationdb", "http 0.2.12", @@ -736,6 +738,7 @@ dependencies = [ "rivet-health-checks", "rivet-operation", "rivet-pools", + "s3-util", "serde", "serde_json", "thiserror 1.0.69", @@ -10767,6 +10770,7 @@ dependencies = [ "reqwest 0.12.12", "rivet-logs", "rivet-util", + "scc", "serde", "serde_json", "serde_yaml", @@ -10792,6 +10796,7 @@ dependencies = [ "build", "chirp-client", "chirp-workflow", + "cluster", "fdb-util", "foundationdb", "pegboard", @@ -12873,12 +12878,14 @@ dependencies = [ "chrono", "clap", "colored_json", + "fdb-util", "foundationdb", "futures-util", "global-error", "hex", "include_dir", "indoc 2.0.5", + "lz4_flex", "reqwest 0.12.12", "rivet-api", "rivet-config", @@ -13515,6 +13522,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.27" @@ -13604,6 +13620,12 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "sdd" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "584e070911c7017da6cb2eb0788d09f43d789029b5877d3e5ecc8acf86ceee21" + [[package]] name = "sealed" version = "0.4.0" diff --git a/docker/dev-full/docker-compose.yml b/docker/dev-full/docker-compose.yml index 6d0f064868..101eae25f3 100644 --- a/docker/dev-full/docker-compose.yml +++ b/docker/dev-full/docker-compose.yml @@ -13,7 +13,7 @@ services: - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_ENDPOINT=http://otel-collector:4317 - # - RUST_LOG=debug,hyper=info + - RUST_LOG=debug,hyper=info stop_grace_period: 0s ports: # API @@ -44,6 +44,8 @@ services: interval: 2s timeout: 10s retries: 10 + # Initial SQL migrations sometimes take time to run + start_period: 30s rivet-edge-server: build: @@ -60,7 +62,7 @@ services: - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_SERVICE_NAME=rivet-edge - RIVET_OTEL_ENDPOINT=http://otel-collector:4317 - # - RUST_LOG=debug,hyper=info + - RUST_LOG=debug,hyper=info stop_grace_period: 0s ports: # API @@ -138,7 +140,7 @@ services: - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_SERVICE_NAME=rivet-guard - RIVET_OTEL_ENDPOINT=http://otel-collector:4317 - # - RUST_LOG=debug,hyper=info + - RUST_LOG=debug,hyper=info stop_grace_period: 0s ports: # HTTP @@ -198,6 +200,9 @@ services: platform: linux/amd64 restart: unless-stopped command: -c /etc/rivet-client/config.jsonc + # entrypoint: entrypoint.sh + # Required for runc & cnitool to operate + privileged: true environment: - RUST_BACKTRACE=1 - RUST_LOG_ANSI_COLOR=1 @@ -205,7 +210,7 @@ services: - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_SERVICE_NAME=rivet-client - RIVET_OTEL_ENDPOINT=http://otel-collector:4317 - # - RUST_LOG=debug,hyper=info + - RUST_LOG=debug,hyper=info stop_grace_period: 0s depends_on: rivet-edge-server: @@ -213,7 +218,9 @@ services: foundationdb: condition: service_healthy volumes: + # - ./rivet-client/entrypoint.sh:/usr/local/bin/entrypoint.sh:ro - ./rivet-client/config.jsonc:/etc/rivet-client/config.jsonc:ro + - ./rivet-client/rivet-actor.conflist:/opt/cni/config/rivet-actor.conflist:ro - client-data:/var/lib/rivet-client ports: # Enable host networking for actors diff --git a/docker/dev-full/rivet-client/config.jsonc b/docker/dev-full/rivet-client/config.jsonc index 9795890af2..4ce78caad8 100644 --- a/docker/dev-full/rivet-client/config.jsonc +++ b/docker/dev-full/rivet-client/config.jsonc @@ -1,9 +1,8 @@ { "client": { "runner": { - "flavor": "isolate", - // Enables running in non-privileged Docker containers - "use_mounts": false + "flavor": "container", + "use_resource_constraints": false }, "cluster": { // This is safe to hardcode @@ -15,6 +14,7 @@ ] } }, + // TODO: Compare this to network in cluster config "network": { "bind_ip": "127.0.0.1", // Point to DNS name inside Docker container diff --git a/docker/dev-full/rivet-client/entrypoint.sh b/docker/dev-full/rivet-client/entrypoint.sh new file mode 100755 index 0000000000..81a054e1d3 --- /dev/null +++ b/docker/dev-full/rivet-client/entrypoint.sh @@ -0,0 +1,320 @@ +#!/bin/sh +set -euf + +# SEE ALSO: packages/core/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# +# DO NOT CHANGE NETWORKING CONFIGS WITHOUT MANUALLY RE-TESTING RULES +# +# There are no automated tests to validate that iptables +# and tc correctly marks traffic priorities. Manually +# check the following if you change this file. +# +# 1. Restart the game node(s) to make sure there are +# fresh iptables & tc rules. +# 2. Run `tc -s class show dev eth1` and start a lobby. +# Make sure packets are passing through 1:10 (Game +# Guard traffic) and 1:20 (ATS traffic). +# 3. Run `iptables -L -v` and validate that packets are +# flowing through *ALL* rules in the RIVET-ADMIN chain +# (for game traffic) and the RIVET-INPUT chain (for ATS +# traffic). +# 4. Run `iptables -L -v -t mangle` and validate that +# packets are flowing through *ALL* the rules in +# RIVET-ADMIN and RIVET-INPUT. +# 5. Obviously, make sure both bridge and host networking +# works. The lobby connectivity tests cover this. +# +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# Create admin chain that only accepts traffic from the GG subnet +# +# See Nomad equivalent: https://github.com/hashicorp/nomad/blob/a8f0f2612ef9d283ed903721f8453a0c0c3f51c5/client/allocrunner/networking_bridge_linux.go#L73 +# ADMIN_CHAIN="RIVET-ADMIN" +SUBNET_IPV4="172.26.64.0/20" +SUBNET_IPV6="fd00:db8:2::/64" +# PUBLIC_IFACE="eth0" +# VLAN_IFACE="eth1" +# GG_VLAN_SUBNET="0.0.0.0/0" + +# # MARK: Linux Traffic Control +# for iface in $PUBLIC_IFACE $VLAN_IFACE; do +# # Check if the HTB qdisc already exists +# if ! tc qdisc show dev \$iface | grep -q "htb 1:"; then +# +# # Set up a HTB queuing discipline. +# # +# # This will help prioritize traffic in the case of congestion. +# # +# # HTB was chosen over QCB because it allows for more flexibility in the future. +# # +# # Manually test that traffic is getting routed correctly by running: +# # tc -s class show dev eth1 +# # +# # Read more: https://lartc.org/howto/lartc.qdisc.classful.html#AEN1071 +# tc qdisc add dev \$iface \ +# root \ +# handle 1: \ +# htb \ +# default 10 +# +# # Create a root class with a max bandwidth +# tc class add dev \$iface \ +# parent 1: \ +# classid 1:1 \ +# htb \ +# rate 10Gbit +# +# # Game traffic class with high priority +# # +# # Low bandwidth limit = game servers are not expected to use much bandwidth +# # High priority = packets take priority in the case of congestion +# tc class add dev \$iface \ +# parent 1:1 \ +# classid 1:10 \ +# htb \ +# rate 100Mbit \ +# prio 0 +# +# # Background traffic class with lower priority +# # +# # High bandwidth = peak performance when there is no network congestion +# # Low priority = packets are dropped first in the case of congestion +# tc class add dev \$iface \ +# parent 1:1 \ +# classid 1:20 \ +# htb \ +# rate 1000Mbit \ +# prio 1 +# +# # Forward packets with different marks to the appropriate classes. +# # +# # prio x = sets filter priority +# # handle x = handle packets marked x by iptables +# # fw classid x = send matched packets to class x +# # action change dsfield set x = set the packet's TOS (0x10 = low delay, 0x8 = high throughput) +# tc filter add dev \$iface \ +# protocol ip \ +# parent 1:0 \ +# prio 1 \ +# handle 1 \ +# fw classid 1:10 +# tc filter add dev \$iface \ +# protocol ip \ +# parent 1:0 \ +# prio 2 \ +# handle 2 \ +# fw classid 1:20 +# +# echo "HTB qdisc and class rules added." +# +# else +# echo "HTB qdisc and class rules already exist." +# fi +# done +# +# # MARK: iptables +# add_ipt_chain() { +# local ipt="\$1" +# local table="\$2" +# local chain="\$3" +# +# if ! "\$ipt" -t "\$table" -L "\$chain" &>/dev/null; then +# "\$ipt" -t "\$table" -N "\$chain" +# echo "Created \$ipt \$table chain: \$chain" +# else +# echo "Chain already exists in \$ipt \$table: \$chain" +# fi +# } +# +# add_ipt_rule() { +# local ipt="\$1" +# local table="\$2" +# local chain="\$3" +# local rule="\$4" +# +# if ! "\$ipt" -t \$table -C "\$chain" \$rule &>/dev/null; then +# "\$ipt" -t \$table -A "\$chain" \$rule +# echo "Added \$ipt \$table \$chain rule: \$rule" +# else +# echo "Rule already exists in \$ipt \$table \$chain: \$rule" +# fi +# } +# +# for ipt in iptables ip6tables; do +# # Define SUBNET_VAR based on iptables version +# if [ "\$ipt" == "iptables" ]; then +# SUBNET_VAR="$SUBNET_IPV4" +# else +# SUBNET_VAR="$SUBNET_IPV6" +# fi +# +# # MARK: Chains +# add_ipt_chain "\$ipt" "filter" "$ADMIN_CHAIN" +# +# add_ipt_chain "\$ipt" "mangle" "RIVET-FORWARD" +# add_ipt_rule "\$ipt" "mangle" "FORWARD" "-j RIVET-FORWARD" +# +# add_ipt_chain "\$ipt" "filter" "RIVET-INPUT" +# add_ipt_rule "\$ipt" "filter" "INPUT" "-j RIVET-INPUT" +# +# add_ipt_chain "\$ipt" "mangle" "RIVET-INPUT" +# add_ipt_rule "\$ipt" "mangle" "INPUT" "-j RIVET-INPUT" +# +# # MARK: Create GG TOS +# # +# # Sets the TOS to minimize delay if not already set. +# if ! "\$ipt" -t mangle -L "RIVET-TOS-GG" &>/dev/null; then +# "\$ipt" -t mangle -N "RIVET-TOS-GG" +# echo "Created \$ipt chain: RIVET-TOS-GG" +# else +# echo "Chain already exists in \$ipt: RIVET-TOS-GG" +# fi +# add_ipt_rule "\$ipt" "mangle" "RIVET-TOS-GG" "-m tos ! --tos 0x0 -j RETURN" +# add_ipt_rule "\$ipt" "mangle" "RIVET-TOS-GG" "-j TOS --set-tos 0x10" +# +# # VLAN only applicable to IPv4 +# if [ "\$ipt" == "iptables" ]; then +# # MARK: GG TOS +# add_ipt_rule "\$ipt" "mangle" "RIVET-FORWARD" "-s $GG_VLAN_SUBNET -d \$SUBNET_VAR -j RIVET-TOS-GG" +# add_ipt_rule "\$ipt" "mangle" "RIVET-FORWARD" "-s \$SUBNET_VAR -d $GG_VLAN_SUBNET -j RIVET-TOS-GG" +# +# # MARK: GG ingress +# # Prioritize traffic +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s $GG_VLAN_SUBNET -d \$SUBNET_VAR -j MARK --set-mark 1" +# # Accept traffic +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s $GG_VLAN_SUBNET -d \$SUBNET_VAR -j ACCEPT" +# +# # MARK: GG egress +# # Prioritize response traffic +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -m conntrack --ctstate NEW,ESTABLISHED -j MARK --set-mark 1" +# # Enable conntrack to allow traffic to flow back to the GG subnet +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT" +# +# # MARK: ATS ingress +# # Maximize throughput from ATS +# add_ipt_rule "\$ipt" "mangle" "RIVET-INPUT" "-s __ATS_VLAN_SUBNET__ -j TOS --set-tos Maximize-Throughput" +# # Deprioritize traffic so game traffic takes priority +# add_ipt_rule "\$ipt" "filter" "RIVET-INPUT" "-s __ATS_VLAN_SUBNET__ -j MARK --set-mark 2" +# fi +# +# # MARK: Public egress +# # Prioritize traffic +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -o $PUBLIC_IFACE -j MARK --set-mark 1" +# # Allow egress traffic +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -o $PUBLIC_IFACE -j ACCEPT" +# +# # Allow public ingress traffic on all ports because this is already mapped by CNI +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-p tcp -d \$SUBNET_VAR -i $PUBLIC_IFACE -j ACCEPT" +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-p udp -d \$SUBNET_VAR -i $PUBLIC_IFACE -j ACCEPT" +# +# # MARK: Deny +# # Deny all other egress traffic +# add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -j DROP" +# done + +# MARK: CNI +# +# Dual-stack CNI config +# +# We use ptp instead of bridge networking in order to isolate the pod's traffic. It's also more performant than bridge networking. +# +# See default Nomad configuration: https://github.com/hashicorp/nomad/blob/a8f0f2612ef9d283ed903721f8453a0c0c3f51c5/client/allocrunner/networking_bridge_linux.go#L152 +# cat << EOF > /opt/cni/config/rivet-actor.conflist +# { +# "cniVersion": "0.4.0", +# "name": "rivet-actor", +# "plugins": [ +# { +# "type": "loopback" +# }, +# { +# "type": "ptp", +# "ipMasq": true, +# "ipam": { +# "type": "host-local", +# "ranges": [ +# [ +# { "subnet": "$SUBNET_IPV4" } +# ], +# [ +# { "subnet": "$SUBNET_IPV6" } +# ] +# ], +# "routes": [ +# { "dst": "0.0.0.0/0" }, +# { "dst": "::/0" } +# ] +# }, +# "dns": { +# "nameservers": [ +# "8.8.8.8", +# "8.8.4.4", +# "2001:4860:4860::8888", +# "2001:4860:4860::8844" +# ], +# "options": ["rotate", "edns0", "attempts:2"] +# } +# }, +# { +# "type": "firewall", +# "backend": "iptables", +# "iptablesAdminChainName": "$ADMIN_CHAIN" +# }, +# { +# "type": "portmap", +# "capabilities": { "portMappings": true }, +# "snat": true +# } +# ] +# } +# EOF +cat << EOF > /opt/cni/config/rivet-actor.conflist +{ + "cniVersion": "0.4.0", + "name": "rivet-actor", + "plugins": [ + { + "type": "loopback" + }, + { + "type": "ptp", + "ipMasq": true, + "ipam": { + "type": "host-local", + "ranges": [ + [ + { "subnet": "$SUBNET_IPV4" } + ], + [ + { "subnet": "$SUBNET_IPV6" } + ] + ], + "routes": [ + { "dst": "0.0.0.0/0" }, + { "dst": "::/0" } + ] + }, + "dns": { + "nameservers": [ + "8.8.8.8", + "8.8.4.4", + "2001:4860:4860::8888", + "2001:4860:4860::8844" + ], + "options": ["rotate", "edns0", "attempts:2"] + } + }, + { + "type": "portmap", + "capabilities": { "portMappings": true }, + "snat": true + } + ] +} +EOF + +# MARK: Entrypoint +/usr/bin/tini -- rivet-client -c /etc/rivet-client/config.jsonc + diff --git a/docker/dev-full/rivet-client/rivet-actor.conflist b/docker/dev-full/rivet-client/rivet-actor.conflist new file mode 100644 index 0000000000..7d48a3282f --- /dev/null +++ b/docker/dev-full/rivet-client/rivet-actor.conflist @@ -0,0 +1,42 @@ +{ + "cniVersion": "0.4.0", + "name": "rivet-actor", + "plugins": [ + { + "type": "loopback" + }, + { + "type": "ptp", + "ipMasq": true, + "ipam": { + "type": "host-local", + "ranges": [ + [ + { "subnet": "172.26.64.0/20" } + ], + [ + { "subnet": "fd00:db8:2::/64" } + ] + ], + "routes": [ + { "dst": "0.0.0.0/0" }, + { "dst": "::/0" } + ] + }, + "dns": { + "nameservers": [ + "8.8.8.8", + "8.8.4.4", + "2001:4860:4860::8888", + "2001:4860:4860::8844" + ], + "options": ["rotate", "edns0", "attempts:2"] + } + }, + { + "type": "portmap", + "capabilities": { "portMappings": true }, + "snat": true + } + ] +} diff --git a/docker/dev-full/rivet-client/rivet-setup-networking.sh b/docker/dev-full/rivet-client/rivet-setup-networking.sh new file mode 100644 index 0000000000..c3d7df5d6c --- /dev/null +++ b/docker/dev-full/rivet-client/rivet-setup-networking.sh @@ -0,0 +1,270 @@ +#!/bin/sh +set -euf + +# SEE ALSO: packages/core/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# +# DO NOT CHANGE NETWORKING CONFIGS WITHOUT MANUALLY RE-TESTING RULES +# +# There are no automated tests to validate that iptables +# and tc correctly marks traffic priorities. Manually +# check the following if you change this file. +# +# 1. Restart the game node(s) to make sure there are +# fresh iptables & tc rules. +# 2. Run `tc -s class show dev eth1` and start a lobby. +# Make sure packets are passing through 1:10 (Game +# Guard traffic) and 1:20 (ATS traffic). +# 3. Run `iptables -L -v` and validate that packets are +# flowing through *ALL* rules in the RIVET-ADMIN chain +# (for game traffic) and the RIVET-INPUT chain (for ATS +# traffic). +# 4. Run `iptables -L -v -t mangle` and validate that +# packets are flowing through *ALL* the rules in +# RIVET-ADMIN and RIVET-INPUT. +# 5. Obviously, make sure both bridge and host networking +# works. The lobby connectivity tests cover this. +# +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# Create admin chain that only accepts traffic from the GG subnet +# +# See Nomad equivalent: https://github.com/hashicorp/nomad/blob/a8f0f2612ef9d283ed903721f8453a0c0c3f51c5/client/allocrunner/networking_bridge_linux.go#L73 +ADMIN_CHAIN="RIVET-ADMIN" +SUBNET_IPV4="172.26.64.0/20" +SUBNET_IPV6="fd00:db8:2::/64" + +# MARK: Linux Traffic Control +for iface in __PUBLIC_IFACE__ __VLAN_IFACE__; do + # Check if the HTB qdisc already exists + if ! tc qdisc show dev \$iface | grep -q "htb 1:"; then + + # Set up a HTB queuing discipline. + # + # This will help prioritize traffic in the case of congestion. + # + # HTB was chosen over QCB because it allows for more flexibility in the future. + # + # Manually test that traffic is getting routed correctly by running: + # tc -s class show dev eth1 + # + # Read more: https://lartc.org/howto/lartc.qdisc.classful.html#AEN1071 + tc qdisc add dev \$iface \ + root \ + handle 1: \ + htb \ + default 10 + + # Create a root class with a max bandwidth + tc class add dev \$iface \ + parent 1: \ + classid 1:1 \ + htb \ + rate 10Gbit + + # Game traffic class with high priority + # + # Low bandwidth limit = game servers are not expected to use much bandwidth + # High priority = packets take priority in the case of congestion + tc class add dev \$iface \ + parent 1:1 \ + classid 1:10 \ + htb \ + rate 100Mbit \ + prio 0 + + # Background traffic class with lower priority + # + # High bandwidth = peak performance when there is no network congestion + # Low priority = packets are dropped first in the case of congestion + tc class add dev \$iface \ + parent 1:1 \ + classid 1:20 \ + htb \ + rate 1000Mbit \ + prio 1 + + # Forward packets with different marks to the appropriate classes. + # + # prio x = sets filter priority + # handle x = handle packets marked x by iptables + # fw classid x = send matched packets to class x + # action change dsfield set x = set the packet's TOS (0x10 = low delay, 0x8 = high throughput) + tc filter add dev \$iface \ + protocol ip \ + parent 1:0 \ + prio 1 \ + handle 1 \ + fw classid 1:10 + tc filter add dev \$iface \ + protocol ip \ + parent 1:0 \ + prio 2 \ + handle 2 \ + fw classid 1:20 + + echo "HTB qdisc and class rules added." + + else + echo "HTB qdisc and class rules already exist." + fi +done + +# MARK: iptables +add_ipt_chain() { + local ipt="\$1" + local table="\$2" + local chain="\$3" + + if ! "\$ipt" -t "\$table" -L "\$chain" &>/dev/null; then + "\$ipt" -t "\$table" -N "\$chain" + echo "Created \$ipt \$table chain: \$chain" + else + echo "Chain already exists in \$ipt \$table: \$chain" + fi +} + +add_ipt_rule() { + local ipt="\$1" + local table="\$2" + local chain="\$3" + local rule="\$4" + + if ! "\$ipt" -t \$table -C "\$chain" \$rule &>/dev/null; then + "\$ipt" -t \$table -A "\$chain" \$rule + echo "Added \$ipt \$table \$chain rule: \$rule" + else + echo "Rule already exists in \$ipt \$table \$chain: \$rule" + fi +} + +for ipt in iptables ip6tables; do + # Define SUBNET_VAR based on iptables version + if [ "\$ipt" == "iptables" ]; then + SUBNET_VAR="$SUBNET_IPV4" + else + SUBNET_VAR="$SUBNET_IPV6" + fi + + # MARK: Chains + add_ipt_chain "\$ipt" "filter" "$ADMIN_CHAIN" + + add_ipt_chain "\$ipt" "mangle" "RIVET-FORWARD" + add_ipt_rule "\$ipt" "mangle" "FORWARD" "-j RIVET-FORWARD" + + add_ipt_chain "\$ipt" "filter" "RIVET-INPUT" + add_ipt_rule "\$ipt" "filter" "INPUT" "-j RIVET-INPUT" + + add_ipt_chain "\$ipt" "mangle" "RIVET-INPUT" + add_ipt_rule "\$ipt" "mangle" "INPUT" "-j RIVET-INPUT" + + # MARK: Create GG TOS + # + # Sets the TOS to minimize delay if not already set. + if ! "\$ipt" -t mangle -L "RIVET-TOS-GG" &>/dev/null; then + "\$ipt" -t mangle -N "RIVET-TOS-GG" + echo "Created \$ipt chain: RIVET-TOS-GG" + else + echo "Chain already exists in \$ipt: RIVET-TOS-GG" + fi + add_ipt_rule "\$ipt" "mangle" "RIVET-TOS-GG" "-m tos ! --tos 0x0 -j RETURN" + add_ipt_rule "\$ipt" "mangle" "RIVET-TOS-GG" "-j TOS --set-tos 0x10" + + # VLAN only applicable to IPv4 + if [ "\$ipt" == "iptables" ]; then + # MARK: GG TOS + add_ipt_rule "\$ipt" "mangle" "RIVET-FORWARD" "-s __GG_VLAN_SUBNET__ -d \$SUBNET_VAR -j RIVET-TOS-GG" + add_ipt_rule "\$ipt" "mangle" "RIVET-FORWARD" "-s \$SUBNET_VAR -d __GG_VLAN_SUBNET__ -j RIVET-TOS-GG" + + # MARK: GG ingress + # Prioritize traffic + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s __GG_VLAN_SUBNET__ -d \$SUBNET_VAR -j MARK --set-mark 1" + # Accept traffic + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s __GG_VLAN_SUBNET__ -d \$SUBNET_VAR -j ACCEPT" + + # MARK: GG egress + # Prioritize response traffic + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -m conntrack --ctstate NEW,ESTABLISHED -j MARK --set-mark 1" + # Enable conntrack to allow traffic to flow back to the GG subnet + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT" + + # MARK: ATS ingress + # Maximize throughput from ATS + add_ipt_rule "\$ipt" "mangle" "RIVET-INPUT" "-s __ATS_VLAN_SUBNET__ -j TOS --set-tos Maximize-Throughput" + # Deprioritize traffic so game traffic takes priority + add_ipt_rule "\$ipt" "filter" "RIVET-INPUT" "-s __ATS_VLAN_SUBNET__ -j MARK --set-mark 2" + fi + + # MARK: Public egress + # Prioritize traffic + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -o __PUBLIC_IFACE__ -j MARK --set-mark 1" + # Allow egress traffic + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -o __PUBLIC_IFACE__ -j ACCEPT" + + # Allow public ingress traffic on all ports because this is already mapped by CNI + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-p tcp -d \$SUBNET_VAR -i __PUBLIC_IFACE__ -j ACCEPT" + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-p udp -d \$SUBNET_VAR -i __PUBLIC_IFACE__ -j ACCEPT" + + # MARK: Deny + # Deny all other egress traffic + add_ipt_rule "\$ipt" "filter" "$ADMIN_CHAIN" "-s \$SUBNET_VAR -j DROP" +done + +# MARK: CNI +# +# Dual-stack CNI config +# +# We use ptp instead of bridge networking in order to isolate the pod's traffic. It's also more performant than bridge networking. +# +# See default Nomad configuration: https://github.com/hashicorp/nomad/blob/a8f0f2612ef9d283ed903721f8453a0c0c3f51c5/client/allocrunner/networking_bridge_linux.go#L152 +cat << EOF > /opt/cni/config/rivet-actor.conflist +{ + "cniVersion": "0.4.0", + "name": "rivet-actor", + "plugins": [ + { + "type": "loopback" + }, + { + "type": "ptp", + "ipMasq": true, + "ipam": { + "type": "host-local", + "ranges": [ + [ + { "subnet": "$SUBNET_IPV4" } + ], + [ + { "subnet": "$SUBNET_IPV6" } + ] + ], + "routes": [ + { "dst": "0.0.0.0/0" }, + { "dst": "::/0" } + ] + }, + "dns": { + "nameservers": [ + "8.8.8.8", + "8.8.4.4", + "2001:4860:4860::8888", + "2001:4860:4860::8844" + ], + "options": ["rotate", "edns0", "attempts:2"] + } + }, + { + "type": "firewall", + "backend": "iptables", + "iptablesAdminChainName": "$ADMIN_CHAIN" + }, + { + "type": "portmap", + "capabilities": { "portMappings": true }, + "snat": true + } + ] +} +EOF + diff --git a/docker/universal/Dockerfile b/docker/universal/Dockerfile index a525da1143..6b2920b075 100644 --- a/docker/universal/Dockerfile +++ b/docker/universal/Dockerfile @@ -122,19 +122,24 @@ RUN apt-get update -y && \ # MARK: Runner (Full) FROM --platform=linux/amd64 base-runner AS client-full -COPY --from=builder /app/dist/rivet-client /app/dist/rivet-isolate-v8-runner /app/dist/rivet-container-runner /usr/local/bin/ +ARG CNI_PLUGINS_VERSION=1.3.0 +RUN apt-get install -y skopeo iproute2 runc && \ + echo "Downloading lz4" && \ + curl -L https://releases.rivet.gg/tools/lz4/1.10.0/debian11-amd64/lz4 -o /usr/local/bin/lz4 && \ + chmod +x /usr/local/bin/lz4 && \ + echo "Downloading umoci" && \ + curl -Lf -o /usr/bin/umoci "https://github.com/opencontainers/umoci/releases/download/v0.4.7/umoci.amd64" && \ + chmod +x /usr/bin/umoci && \ + echo "Downloading cnitool" && \ + curl -Lf -o /usr/bin/cnitool "https://github.com/rivet-gg/cni/releases/download/v1.1.2-build3/cnitool" && \ + chmod +x /usr/bin/cnitool && \ + echo "Downloading cni-plugins" && \ + mkdir -p /opt/cni/bin /opt/cni/config && \ + curl -L https://github.com/containernetworking/plugins/releases/download/v${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-v${CNI_PLUGINS_VERSION}.tgz | \ + tar -xz -C /opt/cni/bin +COPY --from=builder /app/dist/rivet-client /app/dist/rivet-container-runner /usr/local/bin/ ENTRYPOINT ["/usr/bin/tini", "--", "rivet-client"] -# MARK: Runner (Isolate V8) -FROM --platform=linux/amd64 base-runner AS isolate-v8-runner -COPY --from=builder /app/dist/rivet-isolate-v8-runner /usr/local/bin/ -ENTRYPOINT ["rivet-client"] - -# MARK: Runner (Container) -FROM --platform=linux/amd64 base-runner AS container-runner -COPY --from=builder /app/dist/rivet-container-runner /usr/local/bin/ -ENTRYPOINT ["rivet-client"] - # MARK: Monlith FROM --platform=linux/amd64 debian:12.9-slim AS monolith ENV DEBIAN_FRONTEND=noninteractive diff --git a/justfile b/justfile index dff775d41c..00a3c36b20 100644 --- a/justfile +++ b/justfile @@ -38,6 +38,10 @@ dev-logs CONTAINER: dev-logs-client: {{docker_compose}} exec rivet-client sh -c 'tail -f -n 100 /var/lib/rivet-client/logs/*' +[group('dev')] +dev-logs-client-crashed: + docker run --rm -it -v dev-full_client-data:/var/lib/rivet-client busybox sh -c 'cat /var/lib/rivet-client/logs/*' + [group('dev')] dev-logs-runner: {{docker_compose}} exec rivet-client sh -c 'tail -f -n 100 /var/lib/rivet-client/runner/logs/*' diff --git a/packages/core/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh b/packages/core/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh index 28e4df9a09..285f41d62c 100644 --- a/packages/core/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh +++ b/packages/core/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh @@ -1,3 +1,5 @@ +# SEE ALSO: docker/dev-full/rivet-client/pegboard-configure.sh + PUBLIC_IP=$(ip -4 route get 1.0.0.0 | awk '{print $7; exit}') # MARK: Pegboard config diff --git a/packages/edge/infra/client/config/src/manager.rs b/packages/edge/infra/client/config/src/manager.rs index 99b8ea23b8..2dfa33f438 100644 --- a/packages/edge/infra/client/config/src/manager.rs +++ b/packages/edge/infra/client/config/src/manager.rs @@ -87,6 +87,15 @@ pub struct Runner { /// Whether or not to use a mount for actor file systems. pub use_mounts: Option, + /// Whether or not to use resource constraints on containers. + /// + /// You should enable this if you see this error in development: + /// + /// ``` + /// cannot enter cgroupv2 "/sys/fs/cgroup/test" with domain controllers -- it is in an invalid state + /// ```` + pub use_resource_constraints: Option, + /// WebSocket Port for runners on this machine to connect to. pub port: Option, @@ -99,6 +108,10 @@ impl Runner { self.use_mounts.unwrap_or(true) } + pub fn use_resource_constraints(&self) -> bool { + self.use_resource_constraints.unwrap_or(true) + } + pub fn port(&self) -> u16 { self.port.unwrap_or(6080) } diff --git a/packages/edge/infra/client/manager/src/actor/oci_config.rs b/packages/edge/infra/client/manager/src/actor/oci_config.rs index 5e4b63fcab..43845108cc 100644 --- a/packages/edge/infra/client/manager/src/actor/oci_config.rs +++ b/packages/edge/infra/client/manager/src/actor/oci_config.rs @@ -11,6 +11,7 @@ pub struct ConfigOpts<'a> { pub env: Vec, pub user: PartialOciConfigUser, pub cwd: String, + pub use_resource_constraints: bool, pub cpu: u64, pub memory: u64, pub memory_max: u64, @@ -81,24 +82,32 @@ pub fn config(opts: ConfigOpts) -> Result { "linux": { "resources": { "devices": linux_resources_devices(), - "cpu": { - "shares": cpu_shares, - // If `quota` is greater than `period`, it is allowed to use multiple cores. - // - // Read more: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/sec-cpu - // "quota": CPU_PERIOD * cpu / 1_000, - // "period": CPU_PERIOD, - // Use the env var for the CPU since Nomad handles assigning CPUs to each task - // "cpus": if cpu >= 1_000 { - // Some(template_env_var("NOMAD_CPU_CORES")) - // } else { - // None - // } + "cpu": if opts.use_resource_constraints { + Some(json!({ + "shares": cpu_shares, + // If `quota` is greater than `period`, it is allowed to use multiple cores. + // + // Read more: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/sec-cpu + // "quota": CPU_PERIOD * cpu / 1_000, + // "period": CPU_PERIOD, + // Use the env var for the CPU since Nomad handles assigning CPUs to each task + // "cpus": if cpu >= 1_000 { + // Some(template_env_var("NOMAD_CPU_CORES")) + // } else { + // None + // } + })) + } else { + None }, // Docker: https://github.com/moby/moby/blob/777e9f271095685543f30df0ff7a12397676f938/daemon/daemon_unix.go#L75 - "memory": { - "reservation": opts.memory, - "limit": opts.memory_max, + "memory": if opts.use_resource_constraints { + Some(json!({ + "reservation": opts.memory, + "limit": opts.memory_max, + })) + } else { + None }, // TODO: network diff --git a/packages/edge/infra/client/manager/src/actor/setup.rs b/packages/edge/infra/client/manager/src/actor/setup.rs index 752ef406ab..27cb4deb63 100644 --- a/packages/edge/infra/client/manager/src/actor/setup.rs +++ b/packages/edge/infra/client/manager/src/actor/setup.rs @@ -218,6 +218,7 @@ impl Actor { env, user: user_config.process.user, cwd: user_config.process.cwd, + use_resource_constraints: ctx.config().runner.use_resource_constraints(), cpu: self.config.resources.cpu, memory: self.config.resources.memory, memory_max: self.config.resources.memory_max,