Turbo quant rest experiments

.github/workflows/manual-benchmarks-cascade.yaml

@@ -76,7 +76,7 @@ jobs:
           CONFIG_FILE="benchmark_cascade/benchmark-configs.json"
           BENCHMARK_SET="${{ inputs.benchmark_set }}"
           CURRENT_BATCH=${{ inputs.current_batch || 0 }}
-          BATCH_SIZE=10
+          BATCH_SIZE=12
 
           echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT
 
@@ -102,6 +102,9 @@ jobs:
             exit 1
           fi
 
+          # Expand per-item "repeat" field: each item with "repeat": N becomes N copies (default 1)
+          BENCHMARK_CONFIGS=$(jq '[.[] | . as $c | range($c.repeat // 1) | $c | del(.repeat)]' <<< "$BENCHMARK_CONFIGS")
+
           # Apply params_override if provided
           PARAMS_OVERRIDE='${{ inputs.params_override }}'
           if [ "$PARAMS_OVERRIDE" != "{}" ]; then
@@ -111,16 +114,31 @@ jobs:
             OVERRIDE_PARAMS=$(echo "$PARAMS_OVERRIDE" | jq -r '.params // {}')
 
             if [ "$OVERRIDE_PARAMS" != "{}" ]; then
+              # Extract and remove repeat before passing to generate_configs.py
+              OVERRIDE_REPEAT=$(echo "$OVERRIDE_PARAMS" | jq -r '.repeat // 1')
+              OVERRIDE_PARAMS=$(echo "$OVERRIDE_PARAMS" | jq 'del(.repeat)')
+
               # Generate all parameter combinations using separate Python script
               if ! BENCHMARK_CONFIGS=$(python3 benchmark_cascade/generate_configs.py "$OVERRIDE_PARAMS" "$BENCHMARK_CONFIGS"); then
                 echo "Error: Failed to generate benchmark configurations"
                 exit 1
               fi
 
               echo "Generated $(echo "$BENCHMARK_CONFIGS" | jq length) configurations after applying params_override"
+
+              # Apply override repeat to all generated configs
+              if [ "$OVERRIDE_REPEAT" -gt 1 ]; then
+                BENCHMARK_CONFIGS=$(jq --argjson n "$OVERRIDE_REPEAT" '[.[] as $c | range($n) | $c]' <<< "$BENCHMARK_CONFIGS")
+                echo "Applied repeat=$OVERRIDE_REPEAT, total configs: $(echo "$BENCHMARK_CONFIGS" | jq length)"
+              fi
             fi
           fi
 
+          # Shuffle so that repeated configs land on different machines/batches
+          # Use the first workflow run ID as seed for deterministic shuffle across batches
+          SHUFFLE_SEED=$(echo "$WORKFLOW_RUN_IDS" | cut -d',' -f1)
+          BENCHMARK_CONFIGS=$(python3 -c "import json, random, sys; d=json.load(sys.stdin); random.seed(int(sys.argv[1])); random.shuffle(d); print(json.dumps(d))" "$SHUFFLE_SEED" <<< "$BENCHMARK_CONFIGS")
+
           TOTAL_CONFIGS=$(echo "$BENCHMARK_CONFIGS" | jq length)
           START_INDEX=$((CURRENT_BATCH * BATCH_SIZE))
           END_INDEX=$((START_INDEX + BATCH_SIZE - 1))
@@ -184,7 +202,7 @@ jobs:
             MACHINES_FIRST=false
 
             SERVER_NAME="benchmark-cascade-server-${i}"
-            CLIENT_NAME="benchmark-cascade-client-${i}"
+            CLIENT_NAME="benchmark-cascade-server-${i}"
             MACHINES_INFO="${MACHINES_INFO}{\"server_name\":\"$SERVER_NAME\",\"client_name\":\"$CLIENT_NAME\"}"
           done
 
@@ -234,7 +252,7 @@ jobs:
     if: inputs.process_results == false
     runs-on: ubuntu-latest
     strategy:
-      max-parallel: 10
+      max-parallel: 12
       fail-fast: false
       matrix:
         config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }}
@@ -273,7 +291,7 @@ jobs:
           max_retries: 5
 
       - name: Create Client
-        if: ${{ inputs.machines_per_bench == true || inputs.current_batch == 0 }}
+        if: ${{ (inputs.machines_per_bench == true || inputs.current_batch == 0) && steps.extract_names.outputs.client_name != steps.extract_names.outputs.server_name }}
         uses: ./.github/workflows/actions/create-server-with-retry
         with:
           server_name: ${{ steps.extract_names.outputs.client_name }}
@@ -289,7 +307,7 @@ jobs:
       contents: read
       packages: read
     strategy:
-      max-parallel: 10
+      max-parallel: 12
       fail-fast: false
       matrix:
         config: ${{ fromJSON(needs.prepareBenchmarks.outputs.matrix) }}
@@ -329,6 +347,13 @@ jobs:
           fi
 
           bash -x tools/setup_ci.sh
+
+          # Clear previous results on remote client to avoid mixing data from different runs
+          source tools/ssh.sh
+          CLOUD_NAME=${CLOUD_NAME:-"hetzner"}
+          IP_OF_THE_CLIENT=$(bash "tools/${CLOUD_NAME}/get_public_ip.sh" "$CLIENT_NAME")
+          ssh_with_retry -o StrictHostKeyChecking=no "root@${IP_OF_THE_CLIENT}" "rm -rf ~/results" || true
+
           bash -x tools/run_ci.sh
 
       - name: Upload benchmark results