From b060fd218a7f984194507d821a369cf30ffefea6 Mon Sep 17 00:00:00 2001 From: Srikanth Muppandam Date: Sun, 31 May 2026 20:16:20 +0530 Subject: [PATCH 1/2] Bluetooth: improve failure diagnostics for controller state issues Enhance btloghcidiag() to support lightweight and failure diagnostic modes. The basic mode now keeps controller visibility diagnostics concise, while failure mode collects deeper evidence including hciconfig -a, bluetoothctl show/list, rfkill state, bluetooth.service status, Bluetooth journal logs, Bluetooth-related dmesg, scan_dmesg_errors output, and Bluetooth firmware candidates. Make firmware discovery target-agnostic by scanning generic QCA firmware locations instead of hardcoding a single QCA6698 path. Use existing print_path_meta() when available to avoid fragile ls-based file listing. This improves LAVA failure triage for QCA/WCN Bluetooth controller recovery issues without adding noise to passing runs. Signed-off-by: Srikanth Muppandam --- Runner/utils/lib_bluetooth.sh | 173 ++++++++++++++++++++++++++++++++-- 1 file changed, 163 insertions(+), 10 deletions(-) diff --git a/Runner/utils/lib_bluetooth.sh b/Runner/utils/lib_bluetooth.sh index 76b3c267..d65e574e 100755 --- a/Runner/utils/lib_bluetooth.sh +++ b/Runner/utils/lib_bluetooth.sh @@ -912,41 +912,194 @@ btcontrollerpresentplain() { | grep -qi '^[[:space:]]*Controller[[:space:]]' } -# Log useful diagnostics when bluetoothctl list/controller visibility is flaky. +# Log useful Bluetooth diagnostics. +# +# Usage: +# btloghcidiag [adapter] [mode] [test_path] +# +# Modes: +# basic - lightweight diagnostics for controller visibility warnings +# failure - full diagnostics for real BT test failures +# +# test_path: +# Optional path passed to scan_dmesg_errors when mode=failure. btloghcidiag() { adapter="${1:-}" + diag_mode="${2:-basic}" + diag_test_path="${3:-}" + bt_dmesg_modules="bluetooth|hci0|qca|wcn|btqca|hci_uart|serdev|rfkill|firmware|timeout" - log_warn "Bluetooth diagnostics: controller visibility is inconsistent" + log_warn "Bluetooth diagnostics: controller visibility/state is inconsistent" if [ -n "$adapter" ]; then log_warn "Adapter: $adapter" fi if command -v hciconfig >/dev/null 2>&1; then - out="$(hciconfig 2>/dev/null || true)" - if [ -z "$out" ]; then - log_warn "hciconfig output is empty (HCI attach may be incomplete). Check bluetooth.service + journalctl." + if [ "$diag_mode" = "failure" ]; then + hci_out="$(hciconfig -a 2>/dev/null || true)" + log_warn "hciconfig -a output:" else + hci_out="$(hciconfig 2>/dev/null || true)" log_warn "hciconfig output:" - printf '%s\n' "$out" | sanitize_bt_output | sed 's/^/[BT-DIAG] /' + fi + + if [ -n "$hci_out" ]; then + printf '%s\n' "$hci_out" | + sanitize_bt_output | + sed 's/^/[BT-DIAG] /' + else + log_warn "hciconfig output is empty. HCI attach may be incomplete." fi else log_warn "hciconfig not found; cannot dump HCI state" fi + if command -v bluetoothctl >/dev/null 2>&1; then + log_warn "bluetoothctl list:" + if command -v timeout >/dev/null 2>&1; then + timeout 5 bluetoothctl list 2>&1 | + sanitize_bt_output | + sed 's/^/[BT-DIAG] /' || true + else + bluetoothctl list 2>&1 | + sanitize_bt_output | + sed 's/^/[BT-DIAG] /' || true + fi + + log_warn "bluetoothctl show:" + if command -v timeout >/dev/null 2>&1; then + timeout 5 bluetoothctl show 2>&1 | + sanitize_bt_output | + sed 's/^/[BT-DIAG] /' || true + else + bluetoothctl show 2>&1 | + sanitize_bt_output | + sed 's/^/[BT-DIAG] /' || true + fi + else + log_warn "bluetoothctl not found; cannot dump controller state" + fi + + if [ "$diag_mode" != "failure" ]; then + return 0 + fi + + if command -v rfkill >/dev/null 2>&1; then + log_warn "rfkill list:" + rfkill list 2>&1 | + sed 's/^/[BT-DIAG] /' || true + else + log_warn "rfkill not found; cannot dump rfkill state" + fi + if command -v systemctl >/dev/null 2>&1; then - log_warn "systemctl status bluetooth:" - systemctl status bluetooth --no-pager 2>/dev/null | sed 's/^/[BT-DIAG] /' || true + log_warn "systemctl status bluetooth --no-pager:" + systemctl status bluetooth --no-pager 2>&1 | + sed 's/^/[BT-DIAG] /' || true else log_warn "systemctl not found; cannot dump bluetooth.service status" fi if command -v journalctl >/dev/null 2>&1; then - log_warn "journalctl -u bluetooth -b (tail):" - journalctl -u bluetooth -b --no-pager 2>/dev/null | tail -n 60 | sed 's/^/[BT-DIAG] /' || true + log_warn "journalctl -u bluetooth -b --no-pager | tail -100:" + journalctl -u bluetooth -b --no-pager 2>&1 | + tail -n 100 | + sed 's/^/[BT-DIAG] /' || true else log_warn "journalctl not found; cannot dump bluetooth logs" fi + + if command -v dmesg >/dev/null 2>&1; then + log_warn "Bluetooth-related dmesg tail:" + dmesg 2>&1 | + grep -Ei "$bt_dmesg_modules" | + tail -n 200 | + sed 's/^/[BT-DIAG] /' || true + else + log_warn "dmesg not found; cannot dump kernel Bluetooth logs" + fi + + if [ -n "$diag_test_path" ] && command -v scan_dmesg_errors >/dev/null 2>&1; then + log_warn "Running scan_dmesg_errors for Bluetooth-related modules" + scan_dmesg_errors "$diag_test_path" "$bt_dmesg_modules" "" || true + fi + + log_warn "Bluetooth firmware candidates:" + + fw_list="${TMPDIR:-/tmp}/bt_fw_candidates_$$.txt" + : > "$fw_list" + + for qca_root in /lib/firmware/qca /usr/lib/firmware/qca; do + [ -d "$qca_root" ] || continue + + log_warn "Firmware root: $qca_root" + + if command -v find >/dev/null 2>&1; then + find "$qca_root" -maxdepth 2 -type f \ + \( -name '*btfw*' -o \ + -name '*BT*' -o \ + -name '*bt*' -o \ + -name '*nv*' -o \ + -name '*.tlv' -o \ + -name '*.bin' -o \ + -name '*.mbn' -o \ + -name '*.b*' \) \ + -print 2>/dev/null >> "$fw_list" + fi + done + + if [ -s "$fw_list" ]; then + sort -u "$fw_list" 2>/dev/null | + while IFS= read -r fw_file || [ -n "$fw_file" ]; do + [ -n "$fw_file" ] || continue + + if command -v print_path_meta >/dev/null 2>&1; then + print_path_meta "$fw_file" 2>/dev/null | + sed 's/^/[BT-DIAG] /' || true + elif command -v stat >/dev/null 2>&1; then + stat -c '%A %U %G %a %n' "$fw_file" 2>/dev/null | + sed 's/^/[BT-DIAG] /' || true + else + printf '%s\n' "$fw_file" | + sed 's/^/[BT-DIAG] /' + fi + done + else + if command -v btfwpresent >/dev/null 2>&1; then + fw_dir="$(btfwpresent 2>/dev/null || true)" + + if [ -n "$fw_dir" ]; then + log_warn "btfwpresent found Bluetooth firmware under: $fw_dir" + + if command -v find >/dev/null 2>&1; then + find "$fw_dir" -maxdepth 1 -type f -print 2>/dev/null | + while IFS= read -r fw_file || [ -n "$fw_file" ]; do + [ -n "$fw_file" ] || continue + + if command -v print_path_meta >/dev/null 2>&1; then + print_path_meta "$fw_file" 2>/dev/null | + sed 's/^/[BT-DIAG] /' || true + elif command -v stat >/dev/null 2>&1; then + stat -c '%A %U %G %a %n' "$fw_file" 2>/dev/null | + sed 's/^/[BT-DIAG] /' || true + else + printf '%s\n' "$fw_file" | + sed 's/^/[BT-DIAG] /' + fi + done + else + log_warn "find not available; firmware file listing skipped" + fi + else + log_warn "No Bluetooth firmware candidates found by btfwpresent" + fi + else + log_warn "No Bluetooth firmware candidates found under known QCA firmware roots" + fi + fi + + rm -f "$fw_list" 2>/dev/null || true } # Warn+diag once per run if "bluetoothctl list" is empty in non-interactive mode. From e33a8d00274c594435d2a1b86448788bf5d581ad Mon Sep 17 00:00:00 2001 From: Srikanth Muppandam Date: Sun, 31 May 2026 20:16:56 +0530 Subject: [PATCH 2/2] BT_ON_OFF: add power-on settle delay and retry controls Add configurable OFF-to-ON settle delay and controlled power-on retry handling to the BT_ON_OFF test. Some QCA/WCN UART Bluetooth controllers may not recover immediately after Powered=no. In LAVA this can lead to power-on failures during firmware or NVM setup, such as HCI command timeouts, while the same image may pass in a local setup. The test now: - waits before requesting Power ON after successful Power OFF - retries Power ON using configurable attempt and delay values - optionally unblocks Bluetooth through rfkill before retry - optionally restarts bluetooth.service before retry - captures full Bluetooth diagnostics on real failure paths - keeps persistent failure to restore Powered=yes as a test failure - reports retry recovery as PASS with warning Update the BT_ON_OFF LAVA YAML to expose the new runtime knobs while preserving the existing single BT_ON_OFF.res result flow. Signed-off-by: Srikanth Muppandam --- .../Bluetooth/BT_ON_OFF/BT_ON_OFF.yaml | 8 +- .../Connectivity/Bluetooth/BT_ON_OFF/run.sh | 153 +++++++++++++++--- 2 files changed, 138 insertions(+), 23 deletions(-) diff --git a/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/BT_ON_OFF.yaml b/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/BT_ON_OFF.yaml index 929ee06b..48683d5e 100644 --- a/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/BT_ON_OFF.yaml +++ b/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/BT_ON_OFF.yaml @@ -1,7 +1,7 @@ metadata: name: bt-on-off format: "Lava-Test Test Definition 1.0" - description: "Toggle Bluetooth on/off using qcom-linux-testkit BT_ON_FF" + description: "Toggle Bluetooth on/off using qcom-linux-testkit BT_ON_OFF" os: - linux scope: @@ -9,10 +9,14 @@ metadata: params: BT_ADAPTER: "" + BT_POWER_CYCLE_DELAY: "10" + BT_POWER_ON_ATTEMPTS: "2" + BT_POWER_ON_RETRY_DELAY: "10" + BT_RESTART_SERVICE_ON_RETRY: "1" run: steps: - REPO_PATH=$PWD - cd Runner/suites/Connectivity/Bluetooth/BT_ON_OFF - - ./run.sh --adapter "${BT_ADAPTER}" || true + - ./run.sh --adapter "${BT_ADAPTER}" --power-cycle-delay "${BT_POWER_CYCLE_DELAY}" --power-on-attempts "${BT_POWER_ON_ATTEMPTS}" --power-on-retry-delay "${BT_POWER_ON_RETRY_DELAY}" --restart-service-on-retry "${BT_RESTART_SERVICE_ON_RETRY}" || true - $REPO_PATH/Runner/utils/send-to-lava.sh BT_ON_OFF.res diff --git a/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/run.sh b/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/run.sh index 0e23bc64..fc7db62e 100755 --- a/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/run.sh +++ b/Runner/suites/Connectivity/Bluetooth/BT_ON_OFF/run.sh @@ -1,11 +1,13 @@ #!/bin/sh + # Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause# BT_ON_OFF - Basic Bluetooth power toggle validation (non-expect version) +# SPDX-License-Identifier: BSD-3-Clause +# BT_ON_OFF - Basic Bluetooth power toggle validation (non-expect version) # Robustly find and source init_env SCRIPT_DIR="$( - cd "$(dirname "$0")" || exit 1 - pwd + cd "$(dirname "$0")" || exit 1 + pwd )" INIT_ENV="" SEARCH="$SCRIPT_DIR" @@ -39,12 +41,35 @@ fi # BT_ADAPTER can be set from CLI via --adapter or from environment. BT_ADAPTER="${BT_ADAPTER-}" +# QCA/WCN UART controllers may need a short settle window after Powered=no +# before a new Powered=yes request. Defaults are CI-safe but overridable. +BT_POWER_CYCLE_DELAY="${BT_POWER_CYCLE_DELAY:-10}" +BT_POWER_ON_ATTEMPTS="${BT_POWER_ON_ATTEMPTS:-2}" +BT_POWER_ON_RETRY_DELAY="${BT_POWER_ON_RETRY_DELAY:-10}" +BT_RESTART_SERVICE_ON_RETRY="${BT_RESTART_SERVICE_ON_RETRY:-1}" + while [ "$#" -gt 0 ]; do case "$1" in --adapter) BT_ADAPTER="$2" shift 2 ;; + --power-cycle-delay) + BT_POWER_CYCLE_DELAY="$2" + shift 2 + ;; + --power-on-attempts) + BT_POWER_ON_ATTEMPTS="$2" + shift 2 + ;; + --power-on-retry-delay) + BT_POWER_ON_RETRY_DELAY="$2" + shift 2 + ;; + --restart-service-on-retry) + BT_RESTART_SERVICE_ON_RETRY="$2" + shift 2 + ;; *) log_warn "Unknown argument ignored: $1" shift 1 @@ -52,6 +77,36 @@ while [ "$#" -gt 0 ]; do esac done +case "$BT_POWER_CYCLE_DELAY" in + ''|*[!0-9]*) + BT_POWER_CYCLE_DELAY=10 + ;; +esac + +case "$BT_POWER_ON_ATTEMPTS" in + ''|*[!0-9]*) + BT_POWER_ON_ATTEMPTS=2 + ;; +esac + +case "$BT_POWER_ON_RETRY_DELAY" in + ''|*[!0-9]*) + BT_POWER_ON_RETRY_DELAY=10 + ;; +esac + +case "$BT_RESTART_SERVICE_ON_RETRY" in + 0|1) + ;; + *) + BT_RESTART_SERVICE_ON_RETRY=1 + ;; +esac + +if [ "$BT_POWER_ON_ATTEMPTS" -lt 1 ] 2>/dev/null; then + BT_POWER_ON_ATTEMPTS=1 +fi + TESTNAME="BT_ON_OFF" testpath="$(find_test_case_by_name "$TESTNAME")" || { log_fail "$TESTNAME : Test directory not found." @@ -65,9 +120,10 @@ rm -f "$res_file" log_info "------------------------------------------------------------" log_info "Starting $TESTNAME Testcase" +log_info "Config: BT_POWER_CYCLE_DELAY=${BT_POWER_CYCLE_DELAY}s BT_POWER_ON_ATTEMPTS=$BT_POWER_ON_ATTEMPTS BT_POWER_ON_RETRY_DELAY=${BT_POWER_ON_RETRY_DELAY}s BT_RESTART_SERVICE_ON_RETRY=$BT_RESTART_SERVICE_ON_RETRY" log_info "Checking dependency: bluetoothctl" -# verify that all necessary dependencies +# Verify that all necessary dependencies are available. check_dependencies bluetoothctl pgrep log_info "Checking if bluetoothd is running..." @@ -80,6 +136,7 @@ while [ "$retry" -lt "$MAX_RETRIES" ]; do log_info "bluetoothd is running" break fi + log_warn "bluetoothd not running, retrying in ${RETRY_DELAY}s..." sleep "$RETRY_DELAY" retry=$((retry + 1)) @@ -97,7 +154,7 @@ fi if [ -n "$BT_ADAPTER" ]; then ADAPTER="$BT_ADAPTER" log_info "Using adapter from BT_ADAPTER/CLI: $ADAPTER" - + if command -v bt_adapter_is_usable >/dev/null 2>&1; then if ! bt_adapter_is_usable "$ADAPTER"; then log_warn "Requested adapter '$ADAPTER' is not currently UP/RUNNING with a valid BD address." @@ -106,7 +163,7 @@ if [ -n "$BT_ADAPTER" ]; then fi else bt_log_hci_candidates || true - + if command -v bt_select_usable_adapter >/dev/null 2>&1; then ADAPTER="$(bt_select_usable_adapter 2>/dev/null || true)" elif findhcisysfs >/dev/null 2>&1; then @@ -115,19 +172,19 @@ else ADAPTER="" fi fi -# --- NEW: warn/diag if non-interactive "bluetoothctl list" is empty (non-fatal) --- + +# Warn/diag if non-interactive "bluetoothctl list" is empty. This is non-fatal. btwarniflistempty "$ADAPTER" || true -# Ensure controller is visible to bluetoothctl (try public-addr if needed) +# Ensure controller is visible to bluetoothctl, trying public-addr if needed. if ! bt_ensure_controller_visible "$ADAPTER"; then - # --- NEW: print diagnostics before skipping --- - btloghcidiag "$ADAPTER" + btloghcidiag "$ADAPTER" failure "$testpath" || true log_warn "SKIP — no controller visible to bluetoothctl (HCI RAW/DOWN or attach incomplete)." echo "$TESTNAME SKIP" > "$res_file" exit 0 fi -# Read initial power state +# Read initial power state. initial_power="$(btgetpower "$ADAPTER" 2>/dev/null || true)" [ -z "$initial_power" ] && initial_power="unknown" log_info "Initial Powered = $initial_power" @@ -136,6 +193,7 @@ log_info "Initial Powered = $initial_power" log_info "Powering OFF..." if ! btpower "$ADAPTER" off; then log_fail "btpower($ADAPTER, off) failed (command-level error)." + btloghcidiag "$ADAPTER" failure "$testpath" || true echo "$TESTNAME FAIL" > "$res_file" exit 0 fi @@ -147,23 +205,72 @@ if [ "$after_off" = "no" ]; then log_pass "Post-OFF verification: Powered=no (as expected)." else log_fail "Post-OFF verification failed (Powered=$after_off)." + btloghcidiag "$ADAPTER" failure "$testpath" || true echo "$TESTNAME FAIL" > "$res_file" exit 0 fi # ---- Power ON test ---- +log_info "Waiting ${BT_POWER_CYCLE_DELAY}s before Powering ON..." +sleep "$BT_POWER_CYCLE_DELAY" + log_info "Powering ON..." -if ! btpower "$ADAPTER" on; then - log_fail "btpower($ADAPTER, on) failed (command-level error)." - echo "$TESTNAME FAIL" > "$res_file" - exit 0 -fi +on_attempt=1 +on_success=0 -after_on="$(btgetpower "$ADAPTER" 2>/dev/null || true)" -[ -z "$after_on" ] && after_on="unknown" +while [ "$on_attempt" -le "$BT_POWER_ON_ATTEMPTS" ]; do + log_info "Power ON attempt $on_attempt/$BT_POWER_ON_ATTEMPTS" + + if btpower "$ADAPTER" on; then + after_on="$(btgetpower "$ADAPTER" 2>/dev/null || true)" + [ -z "$after_on" ] && after_on="unknown" + + if [ "$after_on" = "yes" ]; then + on_success=1 + break + fi + + log_warn "Power ON command returned success, but post-check Powered=$after_on" + else + log_warn "btpower($ADAPTER, on) failed on attempt $on_attempt" + fi + + log_warn "Collecting Bluetooth diagnostics after failed Power ON attempt $on_attempt" + btloghcidiag "$ADAPTER" failure "$testpath" || true + + if [ "$on_attempt" -lt "$BT_POWER_ON_ATTEMPTS" ]; then + log_warn "Preparing controlled Power ON retry after ${BT_POWER_ON_RETRY_DELAY}s" + sleep "$BT_POWER_ON_RETRY_DELAY" + + if command -v rfkill >/dev/null 2>&1; then + log_info "Running rfkill unblock bluetooth before retry" + rfkill unblock bluetooth >/dev/null 2>&1 || true + elif command -v rfkillunblocksysfs >/dev/null 2>&1; then + log_info "Running rfkillunblocksysfs before retry" + rfkillunblocksysfs >/dev/null 2>&1 || true + else + log_warn "No rfkill unblock helper available before retry" + fi + + if [ "$BT_RESTART_SERVICE_ON_RETRY" -eq 1 ] 2>/dev/null && command -v systemctl >/dev/null 2>&1; then + log_info "Restarting bluetooth.service before retry" + systemctl restart bluetooth >/dev/null 2>&1 || log_warn "systemctl restart bluetooth failed" + sleep 3 + fi + + if command -v bt_ensure_controller_visible >/dev/null 2>&1; then + bt_ensure_controller_visible "$ADAPTER" || log_warn "Controller visibility check failed before retry" + fi + fi + + on_attempt=$((on_attempt + 1)) +done + +if [ "$on_success" -eq 1 ]; then + if [ "$on_attempt" -gt 1 ]; then + log_warn "Power ON recovered on attempt $on_attempt/$BT_POWER_ON_ATTEMPTS" + fi -if [ "$after_on" = "yes" ]; then - # --- NEW: post-check (covers your "list is empty after run" symptom) --- btwarniflistempty "$ADAPTER" || true log_pass "Post-ON verification: Powered=yes (as expected)." @@ -171,6 +278,10 @@ if [ "$after_on" = "yes" ]; then exit 0 fi -log_fail "Post-ON verification failed (Powered=$after_on)." +after_on="$(btgetpower "$ADAPTER" 2>/dev/null || true)" +[ -z "$after_on" ] && after_on="unknown" + +log_fail "Post-ON verification failed after $BT_POWER_ON_ATTEMPTS attempt(s) (Powered=$after_on)." +btloghcidiag "$ADAPTER" failure "$testpath" || true echo "$TESTNAME FAIL" > "$res_file" exit 0