Fix/ci flakiness

tests/samourai-crew/run_tests.sh

@@ -19,6 +19,28 @@ export KEY="runner"
 export PASSWORD="runner1234"
 export KEY_ADDR="${RUNNER_ADDR}"
 
+# Poll auth/accounts until the account sequence for ADDR is >= EXPECTED.
+# Exits 0 (non-fatal) after 30s even if not reached — tests will surface the real error.
+wait_for_sequence_gte() {
+    ADDR="$1"
+    EXPECTED="$2"
+    RETRIES=30
+    printf "  Waiting for account sequence >= %s ..." "$EXPECTED"
+    while [ "$RETRIES" -gt 0 ]; do
+        CURR=$(gnokey query "auth/accounts/$ADDR" \
+            -remote "$REMOTE" 2>/dev/null \
+            | grep -oE '"sequence"[^,}0-9]*[0-9]+' | grep -oE '[0-9]+$')
+        if [ -n "$CURR" ] && [ "$CURR" -ge "$EXPECTED" ]; then
+            echo " done (seq=$CURR)"
+            return 0
+        fi
+        RETRIES=$((RETRIES - 1))
+        sleep 1
+    done
+    echo " WARNING: timed out waiting for sequence >= $EXPECTED, continuing"
+    return 0
+}
+
 echo "Remote : $REMOTE"
 echo "Chain  : $CHAINID"
 echo "Mode   : $MODE"
@@ -39,19 +61,31 @@ while [ "$RETRIES" -gt 0 ]; do
 done
 
 # --- import all keys before CLA signing ---
-printf "%s\n%s\n%s\n" "$RUNNER_MNEMONIC" "$PASSWORD" "$PASSWORD" | \
-    gnokey add "$KEY" -recover -insecure-password-stdin=true \
-    -home "$GNOKEY_HOME" > /dev/null 2>&1
+import_key() {
+    IK_NAME="$1"
+    IK_MNEMONIC="$2"
+    printf "  Importing key %s ... " "$IK_NAME"
+    OUT=$(printf "%s\n%s\n%s\n" "$IK_MNEMONIC" "$PASSWORD" "$PASSWORD" | \
+        gnokey add "$IK_NAME" -recover -insecure-password-stdin=true \
+        -home "$GNOKEY_HOME" 2>&1)
+    if echo "$OUT" | grep -qiE "already exists"; then
+        echo "already exists, skipping"
+    elif echo "$OUT" | grep -qiE "error:|failed to|panic"; then
+        echo "FAILED"
+        echo "$OUT"
+        exit 1
+    else
+        echo "OK"
+    fi
+}
+
+import_key "$KEY" "$RUNNER_MNEMONIC"
 
 if [ -n "$STRESS_MNEMONIC_2" ] && [ "$STRESS_MNEMONIC_2" != "TODO_REPLACE_STRESS_MNEMONIC_2" ]; then
-    printf "%s\n%s\n%s\n" "$STRESS_MNEMONIC_2" "$PASSWORD" "$PASSWORD" | \
-        gnokey add "stress_2" -recover -insecure-password-stdin=true \
-        -home "$GNOKEY_HOME" > /dev/null 2>&1
+    import_key "stress_2" "$STRESS_MNEMONIC_2"
 fi
 if [ -n "$STRESS_MNEMONIC_3" ] && [ "$STRESS_MNEMONIC_3" != "TODO_REPLACE_STRESS_MNEMONIC_3" ]; then
-    printf "%s\n%s\n%s\n" "$STRESS_MNEMONIC_3" "$PASSWORD" "$PASSWORD" | \
-        gnokey add "stress_3" -recover -insecure-password-stdin=true \
-        -home "$GNOKEY_HOME" > /dev/null 2>&1
+    import_key "stress_3" "$STRESS_MNEMONIC_3"
 fi
 
 # --- sign CLA if required by the network ---
@@ -77,20 +111,35 @@ sign_cla() {
         -insecure-password-stdin=true \
         -home "$GNOKEY_HOME" \
         "$SIGNER_KEY" 2>&1)
-    if echo "$OUT" | grep -q "OK\|already signed\|TX HASH"; then
-        echo "OK"
+    if echo "$OUT" | grep -q "OK!\|TX HASH"; then
+        echo "signed"
+        CLA_SENT=1
+    elif echo "$OUT" | grep -qiE "already signed"; then
+        echo "already signed"
     else
-        echo "signed or skipped"
+        echo "ERROR"
+        echo "$OUT"
+        exit 1
     fi
 }
 
 if [ -n "$CLA_HASH" ]; then
     echo "Signing CLA (hash: $CLA_HASH)..."
+    SEQ_BEFORE=$(gnokey query "auth/accounts/$KEY_ADDR" \
+        -remote "$REMOTE" 2>/dev/null \
+        | grep -oE '"sequence"[^,}0-9]*[0-9]+' | grep -oE '[0-9]+$')
+    SEQ_BEFORE="${SEQ_BEFORE:-0}"
+
+    CLA_SENT=0
     sign_cla "$KEY"
+
     gnokey list -home "$GNOKEY_HOME" 2>/dev/null | grep -oE '^[0-9]+\. [^ ]+' | awk '{print $2}' | while read -r k; do
         [ "$k" != "$KEY" ] && sign_cla "$k"
     done
-    sleep 10
+
+    if [ "$CLA_SENT" -eq 1 ]; then
+        wait_for_sequence_gte "$KEY_ADDR" $((SEQ_BEFORE + 1))
+    fi
 fi
 echo ""
 

tests/samourai-crew/stress/common.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+# Shared helpers for stress test scripts.
+# REMOTE must be exported by the caller (run_tests.sh via env).
+
+# Poll vm/qeval until the package is accessible (max 30s).
+# Non-fatal on timeout — the subsequent transactions will surface the real error.
+wait_for_package() {
+    PKG="$1"
+    RETRIES=30
+    printf "  Waiting for package to be indexed ..."
+    while [ "$RETRIES" -gt 0 ]; do
+        if gnokey query "vm/qeval" -remote "$REMOTE" \
+            -data "${PKG}.Render(\"\")" > /dev/null 2>&1; then
+            echo " ready"
+            return 0
+        fi
+        RETRIES=$((RETRIES - 1))
+        sleep 1
+    done
+    echo " WARNING: timed out after 30s, continuing"
+    return 0
+}
+
+# Return the current committed sequence for ADDR (empty string on error).
+# Handles both compact ("sequence":"5") and spaced ("sequence": "5") JSON,
+# as well as integer values ("sequence": 5).
+get_sequence() {
+    ADDR="$1"
+    gnokey query "auth/accounts/$ADDR" \
+        -remote "$REMOTE" 2>/dev/null \
+        | grep -oE '"sequence"[^,}0-9]*[0-9]+' | grep -oE '[0-9]+$'
+}
+
+# Poll auth/accounts until the account sequence for ADDR is >= EXPECTED (max 30s).
+# Non-fatal on timeout — subsequent transactions will surface the real error.
+wait_for_sequence_gte() {
+    ADDR="$1"
+    EXPECTED="$2"
+    RETRIES=30
+    printf "  Waiting for account sequence >= %s ..." "$EXPECTED"
+    while [ "$RETRIES" -gt 0 ]; do
+        CURR=$(get_sequence "$ADDR")
+        if [ -n "$CURR" ] && [ "$CURR" -ge "$EXPECTED" ]; then
+            echo " done (seq=$CURR)"
+            return 0
+        fi
+        RETRIES=$((RETRIES - 1))
+        sleep 1
+    done
+    echo " WARNING: timed out waiting for sequence >= $EXPECTED, continuing"
+    return 0
+}

tests/samourai-crew/stress/sybil_chaos.sh

@@ -7,6 +7,8 @@
 #   stress_1 (=KEY), stress_2, stress_3 keys imported in GNOKEY_HOME
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=common.sh
+. "$SCRIPT_DIR/common.sh"
 TX_PER_ACCOUNT="${TX_PER_ACCOUNT:-10}"
 SUFFIX=$(date +%s)
 COUNTER_PKGPATH="gno.land/r/${KEY_ADDR}/stress/chaos${SUFFIX}"
@@ -31,13 +33,17 @@ echo ""
 echo "Deploying counter realm..."
 cp "$SCRIPT_DIR/../realms/counter/counter.gno" "$TMPDIR/counter.gno"
 printf 'module = "%s"\ngno = "0.9"\n' "$COUNTER_PKGPATH" > "$TMPDIR/gnomod.toml"
+SEQ_BEFORE=$(get_sequence "$KEY_ADDR")
+SEQ_BEFORE="${SEQ_BEFORE:-0}"
 echo "$PASSWORD" | gnokey maketx addpkg \
     -pkgpath "$COUNTER_PKGPATH" \
     -pkgdir "$TMPDIR" \
     -gas-fee 1000000ugnot -gas-wanted 10000000 \
     -broadcast -chainid "$CHAINID" -remote "$REMOTE" \
     -insecure-password-stdin=true -home "$GNOKEY_HOME" \
     "$KEY" > /dev/null || { echo "FAIL: could not deploy counter"; exit 1; }
+wait_for_package "$COUNTER_PKGPATH"
+wait_for_sequence_gte "$KEY_ADDR" $((SEQ_BEFORE + 1))
 
 cat > "$TMPDIR/increment.gno" << EOF
 package main

tests/samourai-crew/stress/sybil_precision.sh

@@ -3,6 +3,8 @@
 # sends txs sequentially with a small delay.
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=common.sh
+. "$SCRIPT_DIR/common.sh"
 TX_PER_ACCOUNT="${TX_PER_ACCOUNT:-10}"
 TX_DELAY="${TX_DELAY:-0.8}"
 SUFFIX=$(date +%s)
@@ -27,13 +29,17 @@ echo ""
 echo "Deploying counter realm..."
 cp "$SCRIPT_DIR/../realms/counter/counter.gno" "$TMPDIR/counter.gno"
 printf 'module = "%s"\ngno = "0.9"\n' "$COUNTER_PKGPATH" > "$TMPDIR/gnomod.toml"
+SEQ_BEFORE=$(get_sequence "$KEY_ADDR")
+SEQ_BEFORE="${SEQ_BEFORE:-0}"
 echo "$PASSWORD" | gnokey maketx addpkg \
     -pkgpath "$COUNTER_PKGPATH" \
     -pkgdir "$TMPDIR" \
     -gas-fee 1000000ugnot -gas-wanted 10000000 \
     -broadcast -chainid "$CHAINID" -remote "$REMOTE" \
     -insecure-password-stdin=true -home "$GNOKEY_HOME" \
     "$KEY" > /dev/null || { echo "FAIL: could not deploy counter"; exit 1; }
+wait_for_package "$COUNTER_PKGPATH"
+wait_for_sequence_gte "$KEY_ADDR" $((SEQ_BEFORE + 1))
 
 cat > "$TMPDIR/increment.gno" << EOF
 package main

tests/samourai-crew/stress/sybil_salted_chaos.sh

@@ -3,6 +3,8 @@
 # salt per tx to prevent transaction deduplication.
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=common.sh
+. "$SCRIPT_DIR/common.sh"
 TX_PER_ACCOUNT="${TX_PER_ACCOUNT:-10}"
 SUFFIX=$(date +%s)
 COUNTER_PKGPATH="gno.land/r/${KEY_ADDR}/stress/salted${SUFFIX}"
@@ -26,13 +28,17 @@ echo ""
 echo "Deploying counter realm..."
 cp "$SCRIPT_DIR/../realms/counter/counter.gno" "$TMPDIR/counter.gno"
 printf 'module = "%s"\ngno = "0.9"\n' "$COUNTER_PKGPATH" > "$TMPDIR/gnomod.toml"
+SEQ_BEFORE=$(get_sequence "$KEY_ADDR")
+SEQ_BEFORE="${SEQ_BEFORE:-0}"
 echo "$PASSWORD" | gnokey maketx addpkg \
     -pkgpath "$COUNTER_PKGPATH" \
     -pkgdir "$TMPDIR" \
     -gas-fee 1000000ugnot -gas-wanted 10000000 \
     -broadcast -chainid "$CHAINID" -remote "$REMOTE" \
     -insecure-password-stdin=true -home "$GNOKEY_HOME" \
     "$KEY" > /dev/null || { echo "FAIL: could not deploy counter"; exit 1; }
+wait_for_package "$COUNTER_PKGPATH"
+wait_for_sequence_gte "$KEY_ADDR" $((SEQ_BEFORE + 1))
 
 cat > "$TMPDIR/increment.gno" << EOF
 package main