#!/usr/bin/env python3
"""
Single SSH connection: kill stuck, check flags, update startup, launch, submit, monitor.
Properly closes connection when done.
"""
import paramiko
import json
import time
import sys

KEY = r'C:\Users\fabia\.ssh\id_ed25519'
HOST = '192.168.178.150'
USER = 'fabian'

def connect():
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    for attempt in range(5):
        try:
            ssh.connect(HOST, username=USER, key_filename=KEY, timeout=10)
            return ssh
        except Exception as e:
            print(f"  SSH attempt {attempt+1}/5 failed: {e}")
            time.sleep(10)
    print("FATAL: Cannot connect to BC-250")
    sys.exit(1)

def run(ssh, cmd, timeout=120):
    _, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
    out = stdout.read().decode()
    err = stderr.read().decode()
    return out, err

def main():
    ssh = connect()
    print("Connected to BC-250.\n")

    try:
        # ── 1. Kill stuck ComfyUI ──
        print("=== STEP 1: Kill stuck ComfyUI ===")
        out, _ = run(ssh, "pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; "
                          "pgrep -f 'python3 main.py' || echo 'all_dead'")
        print(f"  {out.strip()}")

        # ── 2. Check available flags ──
        print("\n=== STEP 2: ComfyUI flags ===")
        out, err = run(ssh, "bash -c 'source /home/fabian/comfyui-env/bin/activate && "
                            "cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'")
        combined = out + err
        for line in combined.split('\n'):
            low = line.lower()
            if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram',
                                       'memory', 'offload', 'precision', 'novram', 'lowvram']):
                print(f"  {line.strip()}")

        # ── 3. Write startup script ──
        print("\n=== STEP 3: Update startup script ===")
        script = r"""#!/bin/bash
# BC-250 ComfyUI Launcher - GPU inference with CPU VAE decode

# GPU identity
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0

# Use all 12 CPU cores
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12

# Activate venv
source /home/fabian/comfyui-env/bin/activate
cd /home/fabian/ComfyUI

# --novram: aggressive offload, one layer at a time to GPU
# --force-fp16: halve VRAM for diffusion
# --cpu-vae: VAE decode on CPU (GPU hangs on float32 VAE conv2d)
# --disable-smart-memory: no memory heuristics
exec python3 main.py \
    --listen 0.0.0.0 --port 8188 \
    --novram \
    --force-fp16 \
    --cpu-vae \
    --disable-smart-memory
"""
        # Write via heredoc to avoid SFTP
        escaped = script.replace("'", "'\\''")
        out, _ = run(ssh, f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n"
                          f"chmod +x /home/fabian/start_comfyui.sh && echo 'written'")
        print(f"  {out.strip()}")

        # ── 4. Launch ComfyUI ──
        print("\n=== STEP 4: Launch ComfyUI ===")
        run(ssh, "nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &")
        time.sleep(2)

        print("  Waiting for server...")
        for i in range(40):
            time.sleep(3)
            out, _ = run(ssh, "curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0")
            code = out.strip()
            if code == '200':
                print(f"  Server ready ({(i+1)*3}s)")
                break
            if i % 5 == 4:
                log, _ = run(ssh, "tail -2 /home/fabian/comfyui.log 2>/dev/null")
                last = [l.strip() for l in log.strip().split('\n') if l.strip()][-1:]
                print(f"  [{(i+1)*3}s] HTTP {code} ... {last[0][:80] if last else ''}")
        else:
            print("  Server didn't start in 120s!")
            out, _ = run(ssh, "tail -40 /home/fabian/comfyui.log 2>/dev/null")
            print(out)
            return

        # Show startup log
        out, _ = run(ssh, "tail -20 /home/fabian/comfyui.log 2>/dev/null")
        for line in out.strip().split('\n'):
            l = line.strip()
            if l and not l.startswith('FETCH'):
                print(f"  LOG: {l[:120]}")

        # ── 5. Submit workflow ──
        print("\n=== STEP 5: Submit workflow (512x512, 8 steps) ===")
        workflow = {
            "prompt": {
                "1": {"class_type": "UnetLoaderGGUF",
                      "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
                "2": {"class_type": "CLIPLoaderGGUF",
                      "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
                "3": {"class_type": "VAELoader",
                      "inputs": {"vae_name": "ae.safetensors"}},
                "4": {"class_type": "CLIPTextEncode",
                      "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
                "5": {"class_type": "CLIPTextEncode",
                      "inputs": {"text": "", "clip": ["2", 0]}},
                "6": {"class_type": "EmptyLatentImage",
                      "inputs": {"width": 512, "height": 512, "batch_size": 1}},
                "7": {"class_type": "KSampler",
                      "inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
                                 "sampler_name": "euler", "scheduler": "simple",
                                 "positive": ["4", 0], "negative": ["5", 0],
                                 "latent_image": ["6", 0], "denoise": 1.0}},
                "8": {"class_type": "VAEDecode",
                      "inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
                "9": {"class_type": "SaveImage",
                      "inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
            }
        }

        wf_json = json.dumps(workflow).replace("'", "'\\''")
        out, _ = run(ssh, f"curl -s -X POST http://localhost:8188/prompt "
                          f"-H 'Content-Type: application/json' "
                          f"-d '{wf_json}'")
        try:
            resp = json.loads(out.strip())
            if 'error' in resp:
                print(f"  API ERROR: {resp['error']}")
                if 'node_errors' in resp:
                    for nid, e in resp['node_errors'].items():
                        print(f"  Node {nid}: {e}")
                return
            print(f"  Prompt ID: {resp.get('prompt_id')}")
        except:
            print(f"  Response: {out.strip()[:500]}")

        # ── 6. Monitor generation ──
        print("\n=== STEP 6: Monitoring generation ===")
        last_log = ""
        for i in range(120):  # up to 30 minutes
            time.sleep(15)

            stats, _ = run(ssh, "bash -c '"
                "PID=$(pgrep -f \"python3 main.py\" | head -1); "
                "if [ -n \"$PID\" ]; then "
                "  CPU=$(ps -p $PID -o %cpu --no-headers); "
                "  MEM=$(ps -p $PID -o rss --no-headers); "
                "  LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
                "  echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:${LOAD}\"; "
                "else echo DEAD; fi'")

            log, _ = run(ssh, "tail -10 /home/fabian/comfyui.log 2>/dev/null")
            log_s = log.strip()

            elapsed = (i+1) * 15
            m, s = divmod(elapsed, 60)
            print(f"  [{m}m{s:02d}s] {stats.strip()}")

            if log_s != last_log:
                for line in reversed(log_s.split('\n')):
                    l = line.strip()
                    if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
                        print(f"    LOG: {l[:120]}")
                        break
                last_log = log_s

            if 'DEAD' in stats:
                print("\n  PROCESS DIED!")
                out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
                print(out)
                break

            if 'Prompt executed in' in log_s:
                print(f"\n  SUCCESS! Image generated!")
                out, _ = run(ssh, "tail -30 /home/fabian/comfyui.log 2>/dev/null")
                print(out)
                break

            if 'Traceback' in log_s or 'RuntimeError' in log_s:
                print("\n  ERROR detected!")
                out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
                print(out)
                break

        # ── 7. Check output ──
        print("\n=== Output files ===")
        out, _ = run(ssh, "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null")
        print(out.strip())

    finally:
        ssh.close()
        print("\nSSH connection closed.")

if __name__ == '__main__':
    main()
