#!/usr/bin/env python3
"""Step 3: Build PyTorch from source for ROCm gfx1010 on BC-250.

This build will take a long time (1-3 hours on 12 cores).
We run it non-interactively via nohup so it survives SSH disconnects.
"""
import paramiko
import sys
import time

ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')

def run(cmd, timeout=600, desc=""):
    if desc:
        print(f"\n{'='*60}")
        print(f"  {desc}")
        print(f"{'='*60}")
    print(f"$ {cmd}")
    _, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
    out = stdout.read().decode()
    err = stderr.read().decode()
    rc = stdout.channel.recv_exit_status()
    if out.strip():
        lines = out.strip().split('\n')
        if len(lines) > 50:
            print(f"  ... ({len(lines)} lines, showing last 50)")
            print('\n'.join(lines[-50:]))
        else:
            print(out.strip())
    if err.strip():
        lines = [l for l in err.strip().split('\n')]
        if lines:
            show = lines[-30:] if len(lines) > 30 else lines
            print(f"STDERR: {chr(10).join(show)}")
    print(f"  Exit code: {rc}")
    return rc, out, err

# First, create the build script on the BC-250
build_script = r'''#!/bin/bash
set -euo pipefail

LOG="/home/fabian/pytorch_build.log"
exec > >(tee -a "$LOG") 2>&1

echo "=========================================="
echo " PyTorch Build for ROCm gfx1010 (BC-250)"
echo " Started: $(date)"
echo "=========================================="

# Activate venv
source /home/fabian/comfyui-env/bin/activate

# Go to PyTorch source
cd /home/fabian/pytorch

# Set environment for ROCm build
export USE_ROCM=1
export USE_CUDA=0
export PYTORCH_ROCM_ARCH="gfx1010"
export HIP_VISIBLE_DEVICES=0
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export ROCM_PATH=/opt/rocm
export HIP_PATH=/opt/rocm
export CMAKE_PREFIX_PATH=/opt/rocm
export PATH=/opt/rocm/bin:$PATH

# Use ninja for faster builds
export USE_NINJA=1
export CMAKE_GENERATOR=Ninja

# Limit parallel jobs to avoid OOM (14GB RAM + 14GB swap)
# Each compilation unit can use ~1-2GB during link, so limit to 6 jobs
export MAX_JOBS=6

# Use ccache to speed up rebuilds
export USE_CCACHE=1
export CCACHE_DIR=/home/fabian/.ccache

# Disable unnecessary components to speed up build
export USE_FBGEMM=0
export USE_KINETO=0
export USE_CUPTI_SO=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_TENSORPIPE=0
export USE_GLOO=0
export USE_MPI=0
export USE_OPENMP=1
export USE_MKLDNN=1
export BUILD_TEST=0

# Disable CUDA-specific stuff
export USE_CUDNN=0

echo ""
echo "Build config:"
echo "  PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo "  USE_ROCM=$USE_ROCM"
echo "  MAX_JOBS=$MAX_JOBS"
echo "  USE_CCACHE=$USE_CCACHE"
echo "  Python: $(python3 --version)"
echo "  hipcc: $(hipcc --version 2>&1 | head -1)"
echo ""

# Install requirements
echo "Installing PyTorch requirements..."
pip install -r requirements.txt 2>&1 | tail -5

# Run the build
echo ""
echo "Starting PyTorch build... (this will take 1-3 hours)"
echo "=========================================="
python3 setup.py bdist_wheel 2>&1

BUILD_RC=$?
echo ""
echo "=========================================="
echo " Build finished with exit code: $BUILD_RC"
echo " Time: $(date)"
echo "=========================================="

if [ $BUILD_RC -eq 0 ]; then
    echo "Wheel file:"
    ls -lh dist/*.whl 2>/dev/null || echo "No wheel found, trying develop install..."
    
    # Install the wheel
    echo "Installing PyTorch wheel..."
    pip install dist/*.whl 2>&1 | tail -5
    
    # Verify
    echo ""
    echo "Verification:"
    python3 -c "import torch; print(f'PyTorch {torch.__version__}'); print(f'ROCm: {torch.version.hip}'); print(f'CUDA available: {torch.cuda.is_available()}'); print(f'Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"N/A\"}')"
fi

echo "BUILD_COMPLETE_RC=$BUILD_RC" >> "$LOG"
'''

# Write the build script to BC-250
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/build_pytorch.sh', 'w') as f:
    f.write(build_script)
sftp.close()

run("chmod +x /home/fabian/build_pytorch.sh", desc="Make build script executable")

# Check if a build is already running
rc, out, _ = run("pgrep -f 'setup.py bdist_wheel' || echo 'NOT RUNNING'",
    desc="Check if build is already running")

if 'NOT RUNNING' not in out:
    print("\n  BUILD IS ALREADY RUNNING — not starting a new one.")
    print("  Monitor with: tail -f ~/pytorch_build.log")
else:
    # Start the build in background using nohup
    # This way it survives SSH disconnects
    run("nohup bash /home/fabian/build_pytorch.sh > /dev/null 2>&1 &",
        desc="Starting PyTorch build in background (nohup)")
    
    # Give it a moment to start
    time.sleep(5)
    
    # Verify it started
    run("pgrep -fa 'build_pytorch.sh' || pgrep -fa 'setup.py' || echo 'WARNING: Build may have failed to start'",
        desc="Verify build process started")
    
    # Check initial log output
    time.sleep(10)
    run("tail -30 /home/fabian/pytorch_build.log 2>/dev/null || echo 'Log not yet created'",
        desc="Initial build log output")

ssh.close()
print("\n" + "="*60)
print("  PyTorch build started in background on BC-250!")
print("  Monitor: ssh fabian@BC-250 'tail -f ~/pytorch_build.log'")
print("  Check status: ssh fabian@BC-250 'pgrep -fa setup.py'")
print("="*60)