Fabian
227 lines
8.8 KiB
Python
227 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Single SSH connection: kill stuck, check flags, update startup, launch, submit, monitor.
|
|
Properly closes connection when done.
|
|
"""
|
|
import paramiko
|
|
import json
|
|
import time
|
|
import sys
|
|
|
|
KEY = r'C:\Users\fabia\.ssh\id_ed25519'
|
|
HOST = '192.168.178.150'
|
|
USER = 'fabian'
|
|
|
|
def connect():
|
|
ssh = paramiko.SSHClient()
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
for attempt in range(5):
|
|
try:
|
|
ssh.connect(HOST, username=USER, key_filename=KEY, timeout=10)
|
|
return ssh
|
|
except Exception as e:
|
|
print(f" SSH attempt {attempt+1}/5 failed: {e}")
|
|
time.sleep(10)
|
|
print("FATAL: Cannot connect to BC-250")
|
|
sys.exit(1)
|
|
|
|
def run(ssh, cmd, timeout=120):
|
|
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
|
out = stdout.read().decode()
|
|
err = stderr.read().decode()
|
|
return out, err
|
|
|
|
def main():
|
|
ssh = connect()
|
|
print("Connected to BC-250.\n")
|
|
|
|
try:
|
|
# ── 1. Kill stuck ComfyUI ──
|
|
print("=== STEP 1: Kill stuck ComfyUI ===")
|
|
out, _ = run(ssh, "pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; "
|
|
"pgrep -f 'python3 main.py' || echo 'all_dead'")
|
|
print(f" {out.strip()}")
|
|
|
|
# ── 2. Check available flags ──
|
|
print("\n=== STEP 2: ComfyUI flags ===")
|
|
out, err = run(ssh, "bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
|
"cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'")
|
|
combined = out + err
|
|
for line in combined.split('\n'):
|
|
low = line.lower()
|
|
if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram',
|
|
'memory', 'offload', 'precision', 'novram', 'lowvram']):
|
|
print(f" {line.strip()}")
|
|
|
|
# ── 3. Write startup script ──
|
|
print("\n=== STEP 3: Update startup script ===")
|
|
script = r"""#!/bin/bash
|
|
# BC-250 ComfyUI Launcher - GPU inference with CPU VAE decode
|
|
|
|
# GPU identity
|
|
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
|
export HIP_VISIBLE_DEVICES=0
|
|
export HSA_ENABLE_SDMA=0
|
|
export HSA_TOOLS_LIB=""
|
|
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
|
|
|
# Use all 12 CPU cores
|
|
export OMP_NUM_THREADS=12
|
|
export MKL_NUM_THREADS=12
|
|
export OPENBLAS_NUM_THREADS=12
|
|
|
|
# Activate venv
|
|
source /home/fabian/comfyui-env/bin/activate
|
|
cd /home/fabian/ComfyUI
|
|
|
|
# --novram: aggressive offload, one layer at a time to GPU
|
|
# --force-fp16: halve VRAM for diffusion
|
|
# --cpu-vae: VAE decode on CPU (GPU hangs on float32 VAE conv2d)
|
|
# --disable-smart-memory: no memory heuristics
|
|
exec python3 main.py \
|
|
--listen 0.0.0.0 --port 8188 \
|
|
--novram \
|
|
--force-fp16 \
|
|
--cpu-vae \
|
|
--disable-smart-memory
|
|
"""
|
|
# Write via heredoc to avoid SFTP
|
|
escaped = script.replace("'", "'\\''")
|
|
out, _ = run(ssh, f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n"
|
|
f"chmod +x /home/fabian/start_comfyui.sh && echo 'written'")
|
|
print(f" {out.strip()}")
|
|
|
|
# ── 4. Launch ComfyUI ──
|
|
print("\n=== STEP 4: Launch ComfyUI ===")
|
|
run(ssh, "nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &")
|
|
time.sleep(2)
|
|
|
|
print(" Waiting for server...")
|
|
for i in range(40):
|
|
time.sleep(3)
|
|
out, _ = run(ssh, "curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0")
|
|
code = out.strip()
|
|
if code == '200':
|
|
print(f" Server ready ({(i+1)*3}s)")
|
|
break
|
|
if i % 5 == 4:
|
|
log, _ = run(ssh, "tail -2 /home/fabian/comfyui.log 2>/dev/null")
|
|
last = [l.strip() for l in log.strip().split('\n') if l.strip()][-1:]
|
|
print(f" [{(i+1)*3}s] HTTP {code} ... {last[0][:80] if last else ''}")
|
|
else:
|
|
print(" Server didn't start in 120s!")
|
|
out, _ = run(ssh, "tail -40 /home/fabian/comfyui.log 2>/dev/null")
|
|
print(out)
|
|
return
|
|
|
|
# Show startup log
|
|
out, _ = run(ssh, "tail -20 /home/fabian/comfyui.log 2>/dev/null")
|
|
for line in out.strip().split('\n'):
|
|
l = line.strip()
|
|
if l and not l.startswith('FETCH'):
|
|
print(f" LOG: {l[:120]}")
|
|
|
|
# ── 5. Submit workflow ──
|
|
print("\n=== STEP 5: Submit workflow (512x512, 8 steps) ===")
|
|
workflow = {
|
|
"prompt": {
|
|
"1": {"class_type": "UnetLoaderGGUF",
|
|
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
|
"2": {"class_type": "CLIPLoaderGGUF",
|
|
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
|
"3": {"class_type": "VAELoader",
|
|
"inputs": {"vae_name": "ae.safetensors"}},
|
|
"4": {"class_type": "CLIPTextEncode",
|
|
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
|
"5": {"class_type": "CLIPTextEncode",
|
|
"inputs": {"text": "", "clip": ["2", 0]}},
|
|
"6": {"class_type": "EmptyLatentImage",
|
|
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
|
"7": {"class_type": "KSampler",
|
|
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
|
"sampler_name": "euler", "scheduler": "simple",
|
|
"positive": ["4", 0], "negative": ["5", 0],
|
|
"latent_image": ["6", 0], "denoise": 1.0}},
|
|
"8": {"class_type": "VAEDecode",
|
|
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
|
|
"9": {"class_type": "SaveImage",
|
|
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
|
|
}
|
|
}
|
|
|
|
wf_json = json.dumps(workflow).replace("'", "'\\''")
|
|
out, _ = run(ssh, f"curl -s -X POST http://localhost:8188/prompt "
|
|
f"-H 'Content-Type: application/json' "
|
|
f"-d '{wf_json}'")
|
|
try:
|
|
resp = json.loads(out.strip())
|
|
if 'error' in resp:
|
|
print(f" API ERROR: {resp['error']}")
|
|
if 'node_errors' in resp:
|
|
for nid, e in resp['node_errors'].items():
|
|
print(f" Node {nid}: {e}")
|
|
return
|
|
print(f" Prompt ID: {resp.get('prompt_id')}")
|
|
except:
|
|
print(f" Response: {out.strip()[:500]}")
|
|
|
|
# ── 6. Monitor generation ──
|
|
print("\n=== STEP 6: Monitoring generation ===")
|
|
last_log = ""
|
|
for i in range(120): # up to 30 minutes
|
|
time.sleep(15)
|
|
|
|
stats, _ = run(ssh, "bash -c '"
|
|
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
|
"if [ -n \"$PID\" ]; then "
|
|
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
|
" MEM=$(ps -p $PID -o rss --no-headers); "
|
|
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
|
|
" echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:${LOAD}\"; "
|
|
"else echo DEAD; fi'")
|
|
|
|
log, _ = run(ssh, "tail -10 /home/fabian/comfyui.log 2>/dev/null")
|
|
log_s = log.strip()
|
|
|
|
elapsed = (i+1) * 15
|
|
m, s = divmod(elapsed, 60)
|
|
print(f" [{m}m{s:02d}s] {stats.strip()}")
|
|
|
|
if log_s != last_log:
|
|
for line in reversed(log_s.split('\n')):
|
|
l = line.strip()
|
|
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
|
|
print(f" LOG: {l[:120]}")
|
|
break
|
|
last_log = log_s
|
|
|
|
if 'DEAD' in stats:
|
|
print("\n PROCESS DIED!")
|
|
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
|
|
print(out)
|
|
break
|
|
|
|
if 'Prompt executed in' in log_s:
|
|
print(f"\n SUCCESS! Image generated!")
|
|
out, _ = run(ssh, "tail -30 /home/fabian/comfyui.log 2>/dev/null")
|
|
print(out)
|
|
break
|
|
|
|
if 'Traceback' in log_s or 'RuntimeError' in log_s:
|
|
print("\n ERROR detected!")
|
|
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
|
|
print(out)
|
|
break
|
|
|
|
# ── 7. Check output ──
|
|
print("\n=== Output files ===")
|
|
out, _ = run(ssh, "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null")
|
|
print(out.strip())
|
|
|
|
finally:
|
|
ssh.close()
|
|
print("\nSSH connection closed.")
|
|
|
|
if __name__ == '__main__':
|
|
main()
|