#!/usr/bin/env python3
"""Start PyTorch build on BC-250 properly using SFTP for the script."""
import paramiko
import time

ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')

def run(cmd, timeout=120, desc=""):
    if desc:
        print(f"\n{'='*60}")
        print(f"  {desc}")
        print(f"{'='*60}")
    print(f"$ {cmd}")
    _, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
    out = stdout.read().decode()
    err = stderr.read().decode()
    rc = stdout.channel.recv_exit_status()
    if out.strip():
        lines = out.strip().split('\n')
        if len(lines) > 60:
            print(f"  ... ({len(lines)} lines, showing last 60)")
            print('\n'.join(lines[-60:]))
        else:
            print(out.strip())
    if err.strip():
        lines = err.strip().split('\n')
        show = lines[-20:] if len(lines) > 20 else lines
        print(f"STDERR: {chr(10).join(show)}")
    print(f"  Exit code: {rc}")
    return rc, out, err

# Upload the build script via SFTP
build_script = '''#!/bin/bash
set -euo pipefail

LOG="/home/fabian/pytorch_build.log"
exec > >(tee -a "$LOG") 2>&1

echo "=========================================="
echo " PyTorch Build for ROCm gfx1010 (BC-250)"
echo " Started: $(date)"
echo "=========================================="

# Activate venv
source /home/fabian/comfyui-env/bin/activate

cd /home/fabian/pytorch

# ROCm build configuration
export USE_ROCM=1
export USE_CUDA=0
export PYTORCH_ROCM_ARCH="gfx1010"
export HIP_VISIBLE_DEVICES=0
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export ROCM_PATH=/opt/rocm
export HIP_PATH=/opt/rocm
export CMAKE_PREFIX_PATH="/opt/rocm;$(python3 -c 'import sys; print(sys.prefix)')"
export PATH=/opt/rocm/bin:$PATH

# Build settings
export USE_NINJA=1
export CMAKE_GENERATOR=Ninja
export MAX_JOBS=6
export USE_CCACHE=1
export CCACHE_DIR=/home/fabian/.ccache

# Disable unnecessary components for faster build
export USE_FBGEMM=0
export USE_KINETO=0
export USE_CUPTI_SO=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_TENSORPIPE=0
export USE_GLOO=0
export USE_MPI=0
export USE_OPENMP=1
export USE_MKLDNN=1
export BUILD_TEST=0
export USE_CUDNN=0

echo ""
echo "Build config:"
echo "  PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo "  USE_ROCM=$USE_ROCM"
echo "  MAX_JOBS=$MAX_JOBS"
echo "  Python: $(python3 --version)"
echo "  hipcc: $(hipcc --version 2>&1 | head -1)"
echo "  ROCm: $(cat /opt/rocm/.info/version)"
echo ""

# Install requirements
echo "Installing PyTorch requirements..."
pip install -r requirements.txt 2>&1 | tail -10
echo ""

# Clean any partial build
echo "Cleaning previous build artifacts..."
python3 setup.py clean 2>&1 || true
echo ""

# Build the wheel
echo "Starting PyTorch build..."
echo "=========================================="
python3 setup.py bdist_wheel 2>&1

BUILD_RC=$?
echo ""
echo "=========================================="
echo " Build exit code: $BUILD_RC"
echo " Finished: $(date)"
echo "=========================================="

if [ $BUILD_RC -eq 0 ]; then
    echo ""
    echo "Wheel files:"
    ls -lh dist/*.whl 2>/dev/null

    echo ""
    echo "Installing wheel..."
    pip install dist/*.whl 2>&1

    echo ""
    echo "=== VERIFICATION ==="
    python3 -c "
import torch
print(f'PyTorch version: {torch.__version__}')
print(f'HIP version: {torch.version.hip}')
print(f'CUDA available (HIP): {torch.cuda.is_available()}')
if torch.cuda.is_available():
    print(f'Device name: {torch.cuda.get_device_name(0)}')
    print(f'Device count: {torch.cuda.device_count()}')
    t = torch.randn(4, 4, device=\"cuda\")
    print(f'Tensor on GPU: {t.device}')
    print(f'Tensor sum: {t.sum().item():.4f}')
    print('GPU COMPUTE: WORKING')
else:
    print('WARNING: CUDA/HIP not available')
" 2>&1
fi

echo ""
echo "BUILD_COMPLETE_RC=$BUILD_RC"
'''

print("Uploading build script via SFTP...")
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/build_pytorch.sh', 'w') as f:
    f.write(build_script)
sftp.close()

run("chmod +x /home/fabian/build_pytorch.sh", desc="Make executable")

# Remove old log if it exists
run("rm -f /home/fabian/pytorch_build.log", desc="Clean old log")

# Start the build using nohup inside bash (not fish)
# Using bash explicitly to avoid fish issues with nohup
run("bash -c 'nohup bash /home/fabian/build_pytorch.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
    desc="Start build in background")

# Wait for it to actually start
time.sleep(10)

# Verify it's running
run("pgrep -fa 'build_pytorch\\|setup.py' | head -10",
    desc="Verify build is running")

# Check initial log
time.sleep(5)
run("cat /home/fabian/pytorch_build.log 2>/dev/null | head -30 || echo 'Log not yet available'",
    desc="Initial build log")

# Monitor for first compile steps
time.sleep(30)
run("tail -30 /home/fabian/pytorch_build.log 2>/dev/null || echo 'Waiting for log...'",
    desc="Build progress after 30 seconds")

ssh.close()
print("\n" + "="*60)
print("  PyTorch build running on BC-250!")
print("  Monitor: tail -f ~/pytorch_build.log")
print("="*60)