#!/bin/bash
# =============================================================================
# Step 6: Verify Installation — AMD BC-250 ROCm
# =============================================================================
# Post-reboot verification. Checks all critical components:
#   - amdgpu module loaded (out-of-tree, GFXOFF disabled)
#   - No KIQ timeouts or GPU crashes
#   - Device nodes present
#   - ppfeaturemask correct
#   - GPU clock states available
#   - Governor active (if installed)
#   - rocminfo detects GPU
#
# Run as: bash 06_verify_installation.sh
# =============================================================================

set -euo pipefail

PASS=0
FAIL=0
WARN=0

check() {
    local name="$1"
    local result="$2"
    local expected="${3:-}"
    
    if [[ "$result" == "PASS" ]]; then
        echo "  [PASS] $name"
        ((PASS++))
    elif [[ "$result" == "WARN" ]]; then
        echo "  [WARN] $name${expected:+ — $expected}"
        ((WARN++))
    else
        echo "  [FAIL] $name${expected:+ — expected: $expected}"
        ((FAIL++))
    fi
}

echo "============================================"
echo " BC-250 ROCm Installation Verification"
echo "============================================"
echo ""

# ─────────────────────────────────────────────
# 1. amdgpu Module
# ─────────────────────────────────────────────
echo "--- Module Status ---"

if lsmod | grep -q "^amdgpu"; then
    SIZE=$(lsmod | grep "^amdgpu" | awk '{print $2}')
    check "amdgpu module loaded (${SIZE} bytes)" "PASS"
else
    check "amdgpu module loaded" "FAIL" "module not loaded"
fi

# Check out-of-tree (patched module indicator)
if dmesg 2>/dev/null | grep -q "loading out-of-tree module"; then
    check "Out-of-tree module (patched)" "PASS"
else
    check "Out-of-tree module (patched)" "WARN" "may be using stock module"
fi

# Check GFXOFF disabled
if dmesg 2>/dev/null | grep -qi "gfxoff.*not supported\|gfxoff.*disabled\|does not support GFXOFF"; then
    check "GFXOFF disabled" "PASS"
else
    check "GFXOFF disabled" "WARN" "no GFXOFF message in dmesg"
fi

# ─────────────────────────────────────────────
# 2. Stability Checks
# ─────────────────────────────────────────────
echo ""
echo "--- Stability ---"

KIQ_COUNT=$(dmesg 2>/dev/null | grep -c "timeout waiting for kiq fence" || echo 0)
if [[ "$KIQ_COUNT" -eq 0 ]]; then
    check "KIQ timeouts: 0" "PASS"
else
    check "KIQ timeouts: $KIQ_COUNT" "FAIL" "should be 0"
fi

DEAD_COUNT=$(dmesg 2>/dev/null | grep -c "GPU unreachable\|GPU died" || echo 0)
if [[ "$DEAD_COUNT" -eq 0 ]]; then
    check "Dead-GPU events: 0" "PASS"
else
    check "Dead-GPU events: $DEAD_COUNT" "WARN" "dead-GPU bypass triggered"
fi

# ─────────────────────────────────────────────
# 3. Device Nodes
# ─────────────────────────────────────────────
echo ""
echo "--- Device Nodes ---"

for DEV in /dev/dri/card0 /dev/dri/renderD128 /dev/kfd; do
    if [[ -e "$DEV" ]]; then
        check "$DEV exists" "PASS"
    else
        check "$DEV exists" "FAIL" "not found"
    fi
done

# ─────────────────────────────────────────────
# 4. Boot Parameters
# ─────────────────────────────────────────────
echo ""
echo "--- Boot Parameters ---"

PPF=$(cat /sys/module/amdgpu/parameters/ppfeaturemask 2>/dev/null || echo "N/A")
if [[ "$PPF" == "0xfff73ef7" || "$PPF" == "4293918455" ]]; then
    check "ppfeaturemask: $PPF" "PASS"
else
    check "ppfeaturemask: $PPF" "FAIL" "expected 0xfff73ef7"
fi

NORETRY=$(cat /sys/module/amdgpu/parameters/noretry 2>/dev/null || echo "N/A")
check "noretry: $NORETRY" "$([ "$NORETRY" = "0" ] && echo PASS || echo WARN)"

# ─────────────────────────────────────────────
# 5. GPU Status
# ─────────────────────────────────────────────
echo ""
echo "--- GPU Status ---"

if [[ -f /sys/class/drm/card0/device/pp_dpm_sclk ]]; then
    CLOCKS=$(cat /sys/class/drm/card0/device/pp_dpm_sclk)
    echo "  GPU Clocks:"
    echo "$CLOCKS" | sed 's/^/    /'
    check "GPU clock states available" "PASS"
else
    check "GPU clock states" "FAIL" "pp_dpm_sclk not found"
fi

# Governor check
if systemctl --user is-active cyan-skillfish-governor 2>/dev/null | grep -q "active"; then
    check "Governor service active" "PASS"
elif pgrep -f "cyan-skillfish-governor" &>/dev/null; then
    check "Governor process running" "PASS"
else
    check "Governor" "WARN" "not running (optional)"
fi

# ─────────────────────────────────────────────
# 6. ROCm Runtime
# ─────────────────────────────────────────────
echo ""
echo "--- ROCm Runtime ---"

if command -v rocminfo &>/dev/null; then
    ROCM_OUT=$(HSA_OVERRIDE_GFX_VERSION=10.1.0 rocminfo 2>&1 || true)
    
    if echo "$ROCM_OUT" | grep -q "gfx1010"; then
        check "rocminfo detects gfx1010" "PASS"
    else
        check "rocminfo detects gfx1010" "FAIL" "GPU not found"
    fi
    
    AGENT_NAME=$(echo "$ROCM_OUT" | grep "Marketing Name" | head -1 | sed 's/.*: *//')
    if [[ -n "$AGENT_NAME" ]]; then
        check "GPU Name: $AGENT_NAME" "PASS"
    fi
    
    CU_COUNT=$(echo "$ROCM_OUT" | grep "Compute Unit:" | tail -1 | awk '{print $NF}')
    if [[ "$CU_COUNT" == "24" ]]; then
        check "Compute Units: 24" "PASS"
    elif [[ -n "$CU_COUNT" ]]; then
        check "Compute Units: $CU_COUNT" "WARN" "expected 24"
    fi
    
    if echo "$ROCM_OUT" | grep -q "Done"; then
        check "rocminfo completed successfully" "PASS"
    fi
else
    check "rocminfo" "FAIL" "not installed"
fi

# ─────────────────────────────────────────────
# Summary
# ─────────────────────────────────────────────
echo ""
echo "============================================"
echo " Results: $PASS passed, $FAIL failed, $WARN warnings"
echo "============================================"

if [[ "$FAIL" -eq 0 ]]; then
    echo " STATUS: READY — Run 07_run_hip_tests.sh for GPU compute tests"
else
    echo " STATUS: ISSUES DETECTED — Review failures above"
fi

exit "$FAIL"