Fabian
60 lines
2.1 KiB
Python
60 lines
2.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Check PyTorch state and start fresh build on BC-250."""
|
|
import paramiko
|
|
import time
|
|
|
|
ssh = paramiko.SSHClient()
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
|
|
|
def run(cmd, timeout=120, desc=""):
|
|
if desc:
|
|
print(f"\n{'='*60}")
|
|
print(f" {desc}")
|
|
print(f"{'='*60}")
|
|
print(f"$ {cmd}")
|
|
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
|
out = stdout.read().decode()
|
|
err = stderr.read().decode()
|
|
rc = stdout.channel.recv_exit_status()
|
|
if out.strip():
|
|
lines = out.strip().split('\n')
|
|
if len(lines) > 60:
|
|
print(f" ... ({len(lines)} lines, showing last 60)")
|
|
print('\n'.join(lines[-60:]))
|
|
else:
|
|
print(out.strip())
|
|
if err.strip():
|
|
lines = err.strip().split('\n')
|
|
show = lines[-20:] if len(lines) > 20 else lines
|
|
print(f"STDERR: {chr(10).join(show)}")
|
|
print(f" Exit code: {rc}")
|
|
return rc, out, err
|
|
|
|
# Check if there's a wheel already built
|
|
run("ls -lh ~/pytorch/dist/*.whl 2>/dev/null || echo 'No wheels found'",
|
|
desc="Check for existing PyTorch wheels")
|
|
|
|
# Check for any previous build directory
|
|
run("ls -la ~/pytorch/build/ 2>/dev/null | head -10 || echo 'No build dir'",
|
|
desc="Check build directory")
|
|
|
|
# Check if pytorch is already installed in venv
|
|
run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \"import torch; print(torch.__version__); print(torch.version.hip); print(torch.cuda.is_available())\" 2>&1'",
|
|
desc="Check if PyTorch is already installed")
|
|
|
|
# Check pytorch source integrity
|
|
run("ls ~/pytorch/setup.py ~/pytorch/CMakeLists.txt 2>&1",
|
|
desc="Verify PyTorch source files")
|
|
|
|
# Verify ROCm works before build
|
|
run("bash -c 'export HSA_OVERRIDE_GFX_VERSION=10.1.0 && /opt/rocm/bin/rocminfo 2>&1 | grep -E \"gfx|Marketing\" | head -5'",
|
|
desc="Verify ROCm is working")
|
|
|
|
# Check venv
|
|
run("bash -c 'source ~/comfyui-env/bin/activate && which python3 && python3 --version'",
|
|
desc="Verify venv")
|
|
|
|
ssh.close()
|
|
print("\nDone checking state.")
|