#!/usr/bin/env python3
"""Check PyTorch build progress on BC-250."""
import paramiko

ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')

def run(cmd, timeout=30, desc=""):
    if desc:
        print(f"\n{'='*60}")
        print(f"  {desc}")
        print(f"{'='*60}")
    _, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
    out = stdout.read().decode()
    err = stderr.read().decode()
    if out.strip():
        print(out.strip())
    if err.strip():
        print(f"STDERR: {err.strip()}")

# Build process status
run("pgrep -fa 'setup.py|build_pytorch|cmake|ninja|hipcc|cc1plus' | head -20",
    desc="Build processes")

# How long has it been running
run("ps -p 350823 -o etime=,cmd= 2>/dev/null || echo 'Process no longer running'",
    desc="Build process uptime")

# Build log tail
run("tail -60 /home/fabian/pytorch_build.log 2>/dev/null || echo 'No log file'",
    desc="Build log (last 60 lines)")

# Memory usage
run("free -h", desc="Memory status")

# Check for build completion marker
run("grep 'BUILD_COMPLETE' /home/fabian/pytorch_build.log 2>/dev/null || echo 'Build still in progress'",
    desc="Build completion check")

# Check for any errors in log
run("grep -i 'error:\\|fatal:\\|failed' /home/fabian/pytorch_build.log 2>/dev/null | tail -10 || echo 'No errors found'",
    desc="Error check")

# Disk space
run("df -h / | tail -1", desc="Disk space")

ssh.close()