#!/bin/bash
# GPU deep state check
echo "=== dmesg (sudo) - GPU related ==="
sudo dmesg | grep -iE "amdgpu|gfx|gpu|drm|kfd|reset|fault|error|hang|gfxoff|power" | tail -60

echo ""
echo "=== GPU power state ==="
cat /sys/class/drm/card*/device/power_dpm_force_performance_level 2>/dev/null
echo ""
cat /sys/class/drm/card*/device/pp_dpm_sclk 2>/dev/null
echo ""
cat /sys/class/drm/card*/device/pp_dpm_mclk 2>/dev/null

echo ""
echo "=== GPU runtime status ==="
cat /sys/class/drm/card*/device/power/runtime_status 2>/dev/null

echo ""
echo "=== GFXOFF ==="
# Check if GFXOFF is disabled
cat /sys/module/amdgpu/parameters/noretry 2>/dev/null
cat /sys/kernel/debug/dri/*/amdgpu_gfxoff_status 2>/dev/null 2>&1

echo ""
echo "=== Force GPU wake ==="
sudo bash -c 'echo high > /sys/class/drm/card1/device/power_dpm_force_performance_level 2>/dev/null || echo "card1 failed"'
sudo bash -c 'echo high > /sys/class/drm/card0/device/power_dpm_force_performance_level 2>/dev/null || echo "card0 failed"'
sleep 1
echo "After wake:"
cat /sys/class/drm/card*/device/pp_dpm_sclk 2>/dev/null
cat /sys/class/drm/card*/device/power/runtime_status 2>/dev/null

echo ""
echo "=== rocm-smi after wake ==="
rocm-smi 2>&1 | grep -E "SCLK|MCLK|GPU%|Perf|Power"