Fabian
49 lines
1.5 KiB
Python
49 lines
1.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Check PyTorch build progress on BC-250."""
|
|
import paramiko
|
|
|
|
ssh = paramiko.SSHClient()
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
|
|
|
def run(cmd, timeout=30, desc=""):
|
|
if desc:
|
|
print(f"\n{'='*60}")
|
|
print(f" {desc}")
|
|
print(f"{'='*60}")
|
|
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
|
out = stdout.read().decode()
|
|
err = stderr.read().decode()
|
|
if out.strip():
|
|
print(out.strip())
|
|
if err.strip():
|
|
print(f"STDERR: {err.strip()}")
|
|
|
|
# Build process status
|
|
run("pgrep -fa 'setup.py|build_pytorch|cmake|ninja|hipcc|cc1plus' | head -20",
|
|
desc="Build processes")
|
|
|
|
# How long has it been running
|
|
run("ps -p 350823 -o etime=,cmd= 2>/dev/null || echo 'Process no longer running'",
|
|
desc="Build process uptime")
|
|
|
|
# Build log tail
|
|
run("tail -60 /home/fabian/pytorch_build.log 2>/dev/null || echo 'No log file'",
|
|
desc="Build log (last 60 lines)")
|
|
|
|
# Memory usage
|
|
run("free -h", desc="Memory status")
|
|
|
|
# Check for build completion marker
|
|
run("grep 'BUILD_COMPLETE' /home/fabian/pytorch_build.log 2>/dev/null || echo 'Build still in progress'",
|
|
desc="Build completion check")
|
|
|
|
# Check for any errors in log
|
|
run("grep -i 'error:\\|fatal:\\|failed' /home/fabian/pytorch_build.log 2>/dev/null | tail -10 || echo 'No errors found'",
|
|
desc="Error check")
|
|
|
|
# Disk space
|
|
run("df -h / | tail -1", desc="Disk space")
|
|
|
|
ssh.close()
|