#!/usr/bin/env python3 """ Single SSH connection: kill stuck, check flags, update startup, launch, submit, monitor. Properly closes connection when done. """ import paramiko import json import time import sys KEY = r'C:\Users\fabia\.ssh\id_ed25519' HOST = '192.168.178.150' USER = 'fabian' def connect(): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) for attempt in range(5): try: ssh.connect(HOST, username=USER, key_filename=KEY, timeout=10) return ssh except Exception as e: print(f" SSH attempt {attempt+1}/5 failed: {e}") time.sleep(10) print("FATAL: Cannot connect to BC-250") sys.exit(1) def run(ssh, cmd, timeout=120): _, stdout, stderr = ssh.exec_command(cmd, timeout=timeout) out = stdout.read().decode() err = stderr.read().decode() return out, err def main(): ssh = connect() print("Connected to BC-250.\n") try: # ── 1. Kill stuck ComfyUI ── print("=== STEP 1: Kill stuck ComfyUI ===") out, _ = run(ssh, "pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; " "pgrep -f 'python3 main.py' || echo 'all_dead'") print(f" {out.strip()}") # ── 2. Check available flags ── print("\n=== STEP 2: ComfyUI flags ===") out, err = run(ssh, "bash -c 'source /home/fabian/comfyui-env/bin/activate && " "cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'") combined = out + err for line in combined.split('\n'): low = line.lower() if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram', 'memory', 'offload', 'precision', 'novram', 'lowvram']): print(f" {line.strip()}") # ── 3. Write startup script ── print("\n=== STEP 3: Update startup script ===") script = r"""#!/bin/bash # BC-250 ComfyUI Launcher - GPU inference with CPU VAE decode # GPU identity export HSA_OVERRIDE_GFX_VERSION=10.1.0 export HIP_VISIBLE_DEVICES=0 export HSA_ENABLE_SDMA=0 export HSA_TOOLS_LIB="" export HSA_TOOLS_REPORT_LOAD_FAILURE=0 # Use all 12 CPU cores export OMP_NUM_THREADS=12 export MKL_NUM_THREADS=12 export OPENBLAS_NUM_THREADS=12 # Activate venv source /home/fabian/comfyui-env/bin/activate cd /home/fabian/ComfyUI # --novram: aggressive offload, one layer at a time to GPU # --force-fp16: halve VRAM for diffusion # --cpu-vae: VAE decode on CPU (GPU hangs on float32 VAE conv2d) # --disable-smart-memory: no memory heuristics exec python3 main.py \ --listen 0.0.0.0 --port 8188 \ --novram \ --force-fp16 \ --cpu-vae \ --disable-smart-memory """ # Write via heredoc to avoid SFTP escaped = script.replace("'", "'\\''") out, _ = run(ssh, f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n" f"chmod +x /home/fabian/start_comfyui.sh && echo 'written'") print(f" {out.strip()}") # ── 4. Launch ComfyUI ── print("\n=== STEP 4: Launch ComfyUI ===") run(ssh, "nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &") time.sleep(2) print(" Waiting for server...") for i in range(40): time.sleep(3) out, _ = run(ssh, "curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0") code = out.strip() if code == '200': print(f" Server ready ({(i+1)*3}s)") break if i % 5 == 4: log, _ = run(ssh, "tail -2 /home/fabian/comfyui.log 2>/dev/null") last = [l.strip() for l in log.strip().split('\n') if l.strip()][-1:] print(f" [{(i+1)*3}s] HTTP {code} ... {last[0][:80] if last else ''}") else: print(" Server didn't start in 120s!") out, _ = run(ssh, "tail -40 /home/fabian/comfyui.log 2>/dev/null") print(out) return # Show startup log out, _ = run(ssh, "tail -20 /home/fabian/comfyui.log 2>/dev/null") for line in out.strip().split('\n'): l = line.strip() if l and not l.startswith('FETCH'): print(f" LOG: {l[:120]}") # ── 5. Submit workflow ── print("\n=== STEP 5: Submit workflow (512x512, 8 steps) ===") workflow = { "prompt": { "1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}}, "2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}}, "3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}}, "4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}}, "5": {"class_type": "CLIPTextEncode", "inputs": {"text": "", "clip": ["2", 0]}}, "6": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}}, "7": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0, "sampler_name": "euler", "scheduler": "simple", "positive": ["4", 0], "negative": ["5", 0], "latent_image": ["6", 0], "denoise": 1.0}}, "8": {"class_type": "VAEDecode", "inputs": {"samples": ["7", 0], "vae": ["3", 0]}}, "9": {"class_type": "SaveImage", "inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}} } } wf_json = json.dumps(workflow).replace("'", "'\\''") out, _ = run(ssh, f"curl -s -X POST http://localhost:8188/prompt " f"-H 'Content-Type: application/json' " f"-d '{wf_json}'") try: resp = json.loads(out.strip()) if 'error' in resp: print(f" API ERROR: {resp['error']}") if 'node_errors' in resp: for nid, e in resp['node_errors'].items(): print(f" Node {nid}: {e}") return print(f" Prompt ID: {resp.get('prompt_id')}") except: print(f" Response: {out.strip()[:500]}") # ── 6. Monitor generation ── print("\n=== STEP 6: Monitoring generation ===") last_log = "" for i in range(120): # up to 30 minutes time.sleep(15) stats, _ = run(ssh, "bash -c '" "PID=$(pgrep -f \"python3 main.py\" | head -1); " "if [ -n \"$PID\" ]; then " " CPU=$(ps -p $PID -o %cpu --no-headers); " " MEM=$(ps -p $PID -o rss --no-headers); " " LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); " " echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:${LOAD}\"; " "else echo DEAD; fi'") log, _ = run(ssh, "tail -10 /home/fabian/comfyui.log 2>/dev/null") log_s = log.strip() elapsed = (i+1) * 15 m, s = divmod(elapsed, 60) print(f" [{m}m{s:02d}s] {stats.strip()}") if log_s != last_log: for line in reversed(log_s.split('\n')): l = line.strip() if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'): print(f" LOG: {l[:120]}") break last_log = log_s if 'DEAD' in stats: print("\n PROCESS DIED!") out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null") print(out) break if 'Prompt executed in' in log_s: print(f"\n SUCCESS! Image generated!") out, _ = run(ssh, "tail -30 /home/fabian/comfyui.log 2>/dev/null") print(out) break if 'Traceback' in log_s or 'RuntimeError' in log_s: print("\n ERROR detected!") out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null") print(out) break # ── 7. Check output ── print("\n=== Output files ===") out, _ = run(ssh, "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null") print(out.strip()) finally: ssh.close() print("\nSSH connection closed.") if __name__ == '__main__': main()