ROCm-Research-Archive/_TestScripts/Danis ROCm Kernel Patch Research/post_reboot_v3_test.sh

#!/bin/bash
# BC-250 v3 Post-Reboot Verification Script
# Run this IMMEDIATELY after booting with v3 module
#
# Tests in order of escalation:
#   1. Module verification (no GPU access)
#   2. Boot parameter verification
#   3. sysfs GPU state check
#   4. rocminfo (light GPU access)
#   5. Single HIP test (compute)
#   6. Sequential HIP test (the crash scenario from v2)
#
# Usage: bash post_reboot_v3_test.sh [--full]

set -euo pipefail

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

pass() { echo -e "${GREEN}[PASS]${NC} $1"; }
fail() { echo -e "${RED}[FAIL]${NC} $1"; }
warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
info() { echo -e "      $1"; }

FULL_TEST=false
[[ "${1:-}" == "--full" ]] && FULL_TEST=true

echo "============================================"
echo " BC-250 v3 Post-Reboot Verification"
echo " $(date)"
echo "============================================"
echo ""

# Test 1: Kernel version
echo "--- Test 1: Kernel & Module ---"
KVER=$(uname -r)
if [[ "$KVER" == "6.18.8-3-cachyos" ]]; then
    pass "Kernel: $KVER"
else
    warn "Unexpected kernel: $KVER"
fi

# Test 2: v3 strings in loaded module
V3_STRINGS=$(sudo dmesg | grep -c "BC-250" 2>/dev/null || echo "0")
if [[ "$V3_STRINGS" -ge 1 ]]; then
    pass "v3 BC-250 messages in dmesg ($V3_STRINGS occurrences)"
else
    warn "No BC-250 messages in dmesg yet (may appear on first GPU use)"
fi

# Check GFXOFF disable message
if sudo dmesg | grep -q "GFXOFF disabled"; then
    pass "GFXOFF disabled message confirmed in dmesg"
else
    fail "GFXOFF disable message NOT found — v3 patch may not be loaded"
fi

# Test 3: ppfeaturemask
echo ""
echo "--- Test 2: Boot Parameters ---"
CMDLINE=$(cat /proc/cmdline)
if echo "$CMDLINE" | grep -q "ppfeaturemask=0xfff73ef7"; then
    pass "ppfeaturemask=0xfff73ef7 in boot cmdline"
else
    warn "ppfeaturemask not in boot cmdline (may be set via modprobe)"
fi

# Check actual ppfeaturemask from module
ACTUAL_PP=$(cat /sys/module/amdgpu/parameters/ppfeaturemask 2>/dev/null || echo "unknown")
if [[ "$ACTUAL_PP" == "0xfff73ef7" ]] || [[ "$ACTUAL_PP" == "4293869303" ]]; then
    pass "Active ppfeaturemask: $ACTUAL_PP (GFXOFF+DeepSleep+ULV disabled)"
else
    warn "Active ppfeaturemask: $ACTUAL_PP (expected 0xfff73ef7 / 4293869303)"
fi

# Test 4: KIQ errors check
echo ""
echo "--- Test 3: KIQ Error Check ---"
KIQ_ERRORS=$(sudo dmesg | grep -c "timeout waiting for kiq fence" 2>/dev/null || echo "0")
if [[ "$KIQ_ERRORS" -eq 0 ]]; then
    pass "Zero KIQ timeout errors"
else
    fail "Found $KIQ_ERRORS KIQ timeout errors!"
fi

# Check for GPU unreachable messages
GPU_DEAD=$(sudo dmesg | grep -c "GPU unreachable\|GPU died" 2>/dev/null || echo "0")
if [[ "$GPU_DEAD" -eq 0 ]]; then
    pass "Zero GPU-unreachable events"
else
    fail "Found $GPU_DEAD GPU health-check failures!"
fi

# Test 5: sysfs GPU state
echo ""
echo "--- Test 4: GPU State ---"
if [[ -f /sys/class/drm/card0/device/pp_dpm_sclk ]]; then
    SCLK=$(cat /sys/class/drm/card0/device/pp_dpm_sclk)
    pass "GPU clock levels accessible:"
    echo "$SCLK" | sed 's/^/      /'
else
    fail "Cannot read GPU clock levels"
fi

if [[ -f /sys/class/drm/card0/device/pp_od_clk_voltage ]]; then
    OD=$(cat /sys/class/drm/card0/device/pp_od_clk_voltage)
    pass "OD voltage table accessible:"
    echo "$OD" | sed 's/^/      /'
else
    warn "Cannot read OD voltage table"
fi

# Test 6: cyan-skillfish-governor
echo ""
echo "--- Test 5: Governor Status ---"
if systemctl is-active --quiet cyan-skillfish-governor.service 2>/dev/null; then
    pass "cyan-skillfish-governor is running"
else
    warn "cyan-skillfish-governor is NOT running"
fi

# Test 7: rocminfo
echo ""
echo "--- Test 6: ROCm Runtime ---"
if command -v rocminfo &>/dev/null; then
    ROCM_OUT=$(timeout 30 rocminfo 2>&1)
    if echo "$ROCM_OUT" | grep -q "gfx1013\|gfx10"; then
        pass "rocminfo detects GPU (gfx1013)"
    elif echo "$ROCM_OUT" | grep -q "Agent"; then
        pass "rocminfo detects agents"
        info "$(echo "$ROCM_OUT" | grep -i "name" | head -3)"
    else
        warn "rocminfo ran but output unexpected"
    fi
else
    warn "rocminfo not found"
fi

# Test 8: HIP test (only with --full)
echo ""
if $FULL_TEST; then
    echo "--- Test 7: HIP Compute (FULL MODE) ---"

    HIP_TEST="/home/dars/VibeROCm/hip_vector_add/hip_vector_add"
    if [[ -x "$HIP_TEST" ]]; then
        echo "  Running first HIP test..."
        if timeout 60 "$HIP_TEST" 2>&1; then
            pass "First HIP vector_add completed"
            sleep 3

            echo ""
            echo "  Running SECOND HIP test (this is the crash scenario)..."
            echo "  Monitoring dmesg for GPU errors during test..."
            sudo dmesg -C  # Clear dmesg
            if timeout 60 "$HIP_TEST" 2>&1; then
                pass "Second HIP vector_add completed — CRASH BUG RESOLVED!"
                # Check if any GPU errors occurred during the test
                POST_ERRORS=$(sudo dmesg | grep -c "BC-250.*GPU\|timeout\|error" 2>/dev/null || echo "0")
                if [[ "$POST_ERRORS" -eq 0 ]]; then
                    pass "No GPU errors during sequential HIP tests"
                else
                    warn "GPU events during test ($POST_ERRORS), check dmesg"
                fi
            else
                fail "Second HIP test failed or timed out"
                sudo dmesg | grep -i "BC-250\|error\|timeout\|GPU" | tail -10
            fi
        else
            fail "First HIP test failed"
            sudo dmesg | grep -i "BC-250\|error\|timeout\|GPU" | tail -10
        fi
    else
        warn "HIP test binary not found: $HIP_TEST"
        info "Build with: cd /home/dars/VibeROCm/hip_vector_add && hipcc hip_vector_add.cpp -o hip_vector_add"
    fi
else
    echo "--- Test 7: HIP Compute (SKIPPED — use --full to enable) ---"
    info "Run: bash post_reboot_v3_test.sh --full"
fi

echo ""
echo "============================================"
echo " Verification Complete"
echo "============================================"