Fabian
47 lines
1.5 KiB
Python
47 lines
1.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Diagnostic: check full log and GPU state."""
|
|
import paramiko
|
|
ssh = paramiko.SSHClient()
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
|
|
|
def run(cmd, timeout=30):
|
|
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
|
out = stdout.read().decode()
|
|
err = stderr.read().decode()
|
|
return out, err
|
|
|
|
# Full log (last 80 lines)
|
|
out, _ = run("tail -80 /home/fabian/comfyui.log 2>/dev/null")
|
|
print("=== FULL LOG (last 80 lines) ===")
|
|
print(out)
|
|
|
|
# Process state
|
|
out, _ = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
|
"if [ -n \"$PID\" ]; then "
|
|
" echo \"PID: $PID\"; "
|
|
" echo \"=== PROCESS STATE ===\"; "
|
|
" cat /proc/$PID/status | grep -E \"State|Threads|VmRSS|VmSize\"; "
|
|
" echo \"=== WCHAN (what syscall is process blocked on) ===\"; "
|
|
" cat /proc/$PID/wchan 2>/dev/null; echo; "
|
|
" echo \"=== STACK TRACE (kernel) ===\"; "
|
|
" sudo cat /proc/$PID/stack 2>/dev/null || echo \"no permission\"; "
|
|
" echo \"=== TOP THREADS ===\"; "
|
|
" ps -L -p $PID -o tid,%cpu,comm --sort=-%cpu | head -15; "
|
|
"fi'")
|
|
print(out)
|
|
|
|
# GPU info
|
|
out, _ = run("bash -c 'rocm-smi 2>/dev/null || echo no rocm-smi; "
|
|
"echo \"=== dmesg GPU ===\"; "
|
|
"dmesg 2>/dev/null | grep -i -E \"amdgpu|error|fault\" | tail -15 || echo no-dmesg'")
|
|
print("=== GPU ===")
|
|
print(out)
|
|
|
|
# Memory
|
|
out, _ = run("free -h")
|
|
print("=== MEMORY ===")
|
|
print(out)
|
|
|
|
ssh.close()
|