Fabian
134 lines
4.7 KiB
Python
134 lines
4.7 KiB
Python
"""Recreate launcher (lost on reboot) and start ComfyUI."""
|
|
import paramiko, time, json, sys
|
|
|
|
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
|
c = paramiko.SSHClient()
|
|
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
|
sftp = c.open_sftp()
|
|
|
|
def sh(cmd, t=30):
|
|
stdin, stdout, stderr = c.exec_command(f"bash -lc '{cmd}'", timeout=t)
|
|
return stdout.read().decode(errors='replace').strip()
|
|
|
|
# Recreate launcher (wiped by reboot since /tmp)
|
|
launcher = """#!/bin/bash
|
|
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
|
export HIP_VISIBLE_DEVICES=0
|
|
export HSA_ENABLE_SDMA=0
|
|
export HSA_TOOLS_LIB=""
|
|
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
|
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
|
export OMP_NUM_THREADS=12
|
|
export MKL_NUM_THREADS=12
|
|
export OPENBLAS_NUM_THREADS=12
|
|
export MIOPEN_FIND_MODE=3
|
|
export COMFYUI_SHARED_MEMORY=1
|
|
|
|
cd ~/ComfyUI
|
|
source ~/comfyui-env/bin/activate
|
|
|
|
exec python3 main.py \\
|
|
--listen 0.0.0.0 --port 8188 \\
|
|
--force-fp16 \\
|
|
--fp16-vae
|
|
"""
|
|
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
|
|
f.write(launcher)
|
|
sh("chmod +x /tmp/run_comfyui.sh")
|
|
print("Launcher recreated")
|
|
|
|
# Verify patches
|
|
p = sh("grep -c COMFYUI_SHARED_MEMORY ~/ComfyUI/comfy/model_management.py")
|
|
print(f"SHARED patch refs: {p}")
|
|
p2 = sh("grep 'def unet_offload_device' -A2 ~/ComfyUI/comfy/model_management.py | head -3")
|
|
print(f"unet_offload: {p2}")
|
|
p3 = sh("grep 'def vae_offload_device' -A2 ~/ComfyUI/comfy/model_management.py | head -3")
|
|
print(f"vae_offload: {p3}")
|
|
|
|
# Start
|
|
sh("rm -f /tmp/comfyui.log")
|
|
sh("nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &")
|
|
time.sleep(4)
|
|
pid = sh("pgrep -f 'python3.*main.py'")
|
|
print(f"PID: {pid}")
|
|
|
|
if not pid:
|
|
print("FAILED! Log:")
|
|
print(sh("cat /tmp/comfyui.log 2>/dev/null"))
|
|
c.close()
|
|
sys.exit(1)
|
|
|
|
# Wait for HTTP
|
|
print("Waiting for HTTP...", end='', flush=True)
|
|
for i in range(90):
|
|
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null')
|
|
if '200' in code:
|
|
print(f" ready ({i*2}s)")
|
|
break
|
|
print('.', end='', flush=True)
|
|
time.sleep(2)
|
|
else:
|
|
print(" TIMEOUT")
|
|
print(sh("tail -30 /tmp/comfyui.log"))
|
|
c.close()
|
|
sys.exit(1)
|
|
|
|
# Log state
|
|
log = sh("cat /tmp/comfyui.log")
|
|
for l in log.split('\n'):
|
|
s = l.strip()
|
|
if any(x in s for x in ['vram state', 'SHARED', 'Device:', 'Total VRAM']):
|
|
print(f" {s}")
|
|
|
|
# Submit
|
|
wf = {"prompt": {
|
|
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
|
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
|
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
|
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
|
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
|
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
|
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
|
|
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
|
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
|
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
|
}}
|
|
with sftp.open('/tmp/wf.json', 'w') as f:
|
|
f.write(json.dumps(wf))
|
|
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
|
print(f"Submitted: {resp[:100]}")
|
|
|
|
# Monitor
|
|
t0 = time.time()
|
|
shown = set()
|
|
for _ in range(200):
|
|
el = int(time.time() - t0)
|
|
log = sh("cat /tmp/comfyui.log 2>/dev/null")
|
|
|
|
for l in log.split('\n'):
|
|
s = l.strip()
|
|
if s not in shown and any(x in s for x in ['/8', 'loaded completely', 'load device', 'offload device', 'Requested to load', 'Prompt executed', 'Error', 'OOM']):
|
|
if 'FETCH' not in s and 'audio' not in s and 'split attention' not in s:
|
|
gpu = sh("cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null")
|
|
print(f" [{el:>3}s] GPU:{gpu}% | {s[-110:]}")
|
|
shown.add(s)
|
|
|
|
if 'Prompt executed' in log:
|
|
print(f"\n*** DONE in {el}s! ***")
|
|
imgs = sh("ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null")
|
|
print(f" Images: {imgs}")
|
|
break
|
|
|
|
alive = sh("pgrep -c -f 'python3.*main.py' 2>/dev/null")
|
|
if alive == '0':
|
|
print(f"\nCRASHED at {el}s!")
|
|
for l in log.split('\n')[-20:]:
|
|
if l.strip(): print(f" {l.strip()}")
|
|
break
|
|
|
|
time.sleep(5)
|
|
|
|
sftp.close()
|
|
c.close()
|