Fabian
186 lines
5.1 KiB
Python
186 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Start PyTorch build on BC-250 properly using SFTP for the script."""
|
|
import paramiko
|
|
import time
|
|
|
|
ssh = paramiko.SSHClient()
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
|
|
|
def run(cmd, timeout=120, desc=""):
|
|
if desc:
|
|
print(f"\n{'='*60}")
|
|
print(f" {desc}")
|
|
print(f"{'='*60}")
|
|
print(f"$ {cmd}")
|
|
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
|
out = stdout.read().decode()
|
|
err = stderr.read().decode()
|
|
rc = stdout.channel.recv_exit_status()
|
|
if out.strip():
|
|
lines = out.strip().split('\n')
|
|
if len(lines) > 60:
|
|
print(f" ... ({len(lines)} lines, showing last 60)")
|
|
print('\n'.join(lines[-60:]))
|
|
else:
|
|
print(out.strip())
|
|
if err.strip():
|
|
lines = err.strip().split('\n')
|
|
show = lines[-20:] if len(lines) > 20 else lines
|
|
print(f"STDERR: {chr(10).join(show)}")
|
|
print(f" Exit code: {rc}")
|
|
return rc, out, err
|
|
|
|
# Upload the build script via SFTP
|
|
build_script = '''#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
LOG="/home/fabian/pytorch_build.log"
|
|
exec > >(tee -a "$LOG") 2>&1
|
|
|
|
echo "=========================================="
|
|
echo " PyTorch Build for ROCm gfx1010 (BC-250)"
|
|
echo " Started: $(date)"
|
|
echo "=========================================="
|
|
|
|
# Activate venv
|
|
source /home/fabian/comfyui-env/bin/activate
|
|
|
|
cd /home/fabian/pytorch
|
|
|
|
# ROCm build configuration
|
|
export USE_ROCM=1
|
|
export USE_CUDA=0
|
|
export PYTORCH_ROCM_ARCH="gfx1010"
|
|
export HIP_VISIBLE_DEVICES=0
|
|
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
|
export HSA_ENABLE_SDMA=0
|
|
export ROCM_PATH=/opt/rocm
|
|
export HIP_PATH=/opt/rocm
|
|
export CMAKE_PREFIX_PATH="/opt/rocm;$(python3 -c 'import sys; print(sys.prefix)')"
|
|
export PATH=/opt/rocm/bin:$PATH
|
|
|
|
# Build settings
|
|
export USE_NINJA=1
|
|
export CMAKE_GENERATOR=Ninja
|
|
export MAX_JOBS=6
|
|
export USE_CCACHE=1
|
|
export CCACHE_DIR=/home/fabian/.ccache
|
|
|
|
# Disable unnecessary components for faster build
|
|
export USE_FBGEMM=0
|
|
export USE_KINETO=0
|
|
export USE_CUPTI_SO=0
|
|
export USE_NCCL=0
|
|
export USE_DISTRIBUTED=0
|
|
export USE_TENSORPIPE=0
|
|
export USE_GLOO=0
|
|
export USE_MPI=0
|
|
export USE_OPENMP=1
|
|
export USE_MKLDNN=1
|
|
export BUILD_TEST=0
|
|
export USE_CUDNN=0
|
|
|
|
echo ""
|
|
echo "Build config:"
|
|
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
|
|
echo " USE_ROCM=$USE_ROCM"
|
|
echo " MAX_JOBS=$MAX_JOBS"
|
|
echo " Python: $(python3 --version)"
|
|
echo " hipcc: $(hipcc --version 2>&1 | head -1)"
|
|
echo " ROCm: $(cat /opt/rocm/.info/version)"
|
|
echo ""
|
|
|
|
# Install requirements
|
|
echo "Installing PyTorch requirements..."
|
|
pip install -r requirements.txt 2>&1 | tail -10
|
|
echo ""
|
|
|
|
# Clean any partial build
|
|
echo "Cleaning previous build artifacts..."
|
|
python3 setup.py clean 2>&1 || true
|
|
echo ""
|
|
|
|
# Build the wheel
|
|
echo "Starting PyTorch build..."
|
|
echo "=========================================="
|
|
python3 setup.py bdist_wheel 2>&1
|
|
|
|
BUILD_RC=$?
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " Build exit code: $BUILD_RC"
|
|
echo " Finished: $(date)"
|
|
echo "=========================================="
|
|
|
|
if [ $BUILD_RC -eq 0 ]; then
|
|
echo ""
|
|
echo "Wheel files:"
|
|
ls -lh dist/*.whl 2>/dev/null
|
|
|
|
echo ""
|
|
echo "Installing wheel..."
|
|
pip install dist/*.whl 2>&1
|
|
|
|
echo ""
|
|
echo "=== VERIFICATION ==="
|
|
python3 -c "
|
|
import torch
|
|
print(f'PyTorch version: {torch.__version__}')
|
|
print(f'HIP version: {torch.version.hip}')
|
|
print(f'CUDA available (HIP): {torch.cuda.is_available()}')
|
|
if torch.cuda.is_available():
|
|
print(f'Device name: {torch.cuda.get_device_name(0)}')
|
|
print(f'Device count: {torch.cuda.device_count()}')
|
|
t = torch.randn(4, 4, device=\"cuda\")
|
|
print(f'Tensor on GPU: {t.device}')
|
|
print(f'Tensor sum: {t.sum().item():.4f}')
|
|
print('GPU COMPUTE: WORKING')
|
|
else:
|
|
print('WARNING: CUDA/HIP not available')
|
|
" 2>&1
|
|
fi
|
|
|
|
echo ""
|
|
echo "BUILD_COMPLETE_RC=$BUILD_RC"
|
|
'''
|
|
|
|
print("Uploading build script via SFTP...")
|
|
sftp = ssh.open_sftp()
|
|
with sftp.open('/home/fabian/build_pytorch.sh', 'w') as f:
|
|
f.write(build_script)
|
|
sftp.close()
|
|
|
|
run("chmod +x /home/fabian/build_pytorch.sh", desc="Make executable")
|
|
|
|
# Remove old log if it exists
|
|
run("rm -f /home/fabian/pytorch_build.log", desc="Clean old log")
|
|
|
|
# Start the build using nohup inside bash (not fish)
|
|
# Using bash explicitly to avoid fish issues with nohup
|
|
run("bash -c 'nohup bash /home/fabian/build_pytorch.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
|
|
desc="Start build in background")
|
|
|
|
# Wait for it to actually start
|
|
time.sleep(10)
|
|
|
|
# Verify it's running
|
|
run("pgrep -fa 'build_pytorch\\|setup.py' | head -10",
|
|
desc="Verify build is running")
|
|
|
|
# Check initial log
|
|
time.sleep(5)
|
|
run("cat /home/fabian/pytorch_build.log 2>/dev/null | head -30 || echo 'Log not yet available'",
|
|
desc="Initial build log")
|
|
|
|
# Monitor for first compile steps
|
|
time.sleep(30)
|
|
run("tail -30 /home/fabian/pytorch_build.log 2>/dev/null || echo 'Waiting for log...'",
|
|
desc="Build progress after 30 seconds")
|
|
|
|
ssh.close()
|
|
print("\n" + "="*60)
|
|
print(" PyTorch build running on BC-250!")
|
|
print(" Monitor: tail -f ~/pytorch_build.log")
|
|
print("="*60)
|