Uploaded sanitized BC250/ROCm Repository.
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check PyTorch build progress on BC-250."""
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
print(f"STDERR: {err.strip()}")
|
||||
|
||||
# Build process status
|
||||
run("pgrep -fa 'setup.py|build_pytorch|cmake|ninja|hipcc|cc1plus' | head -20",
|
||||
desc="Build processes")
|
||||
|
||||
# How long has it been running
|
||||
run("ps -p 350823 -o etime=,cmd= 2>/dev/null || echo 'Process no longer running'",
|
||||
desc="Build process uptime")
|
||||
|
||||
# Build log tail
|
||||
run("tail -60 /home/fabian/pytorch_build.log 2>/dev/null || echo 'No log file'",
|
||||
desc="Build log (last 60 lines)")
|
||||
|
||||
# Memory usage
|
||||
run("free -h", desc="Memory status")
|
||||
|
||||
# Check for build completion marker
|
||||
run("grep 'BUILD_COMPLETE' /home/fabian/pytorch_build.log 2>/dev/null || echo 'Build still in progress'",
|
||||
desc="Build completion check")
|
||||
|
||||
# Check for any errors in log
|
||||
run("grep -i 'error:\\|fatal:\\|failed' /home/fabian/pytorch_build.log 2>/dev/null | tail -10 || echo 'No errors found'",
|
||||
desc="Error check")
|
||||
|
||||
# Disk space
|
||||
run("df -h / | tail -1", desc="Disk space")
|
||||
|
||||
ssh.close()
|
||||
Reference in New Issue
Block a user