Uploaded sanitized BC250/ROCm Repository.
This commit is contained in:
Fabian
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Single SSH connection: kill stuck, check flags, update startup, launch, submit, monitor.
|
||||
Properly closes connection when done.
|
||||
"""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
import sys
|
||||
|
||||
KEY = r'C:\Users\fabia\.ssh\id_ed25519'
|
||||
HOST = '192.168.178.150'
|
||||
USER = 'fabian'
|
||||
|
||||
def connect():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
for attempt in range(5):
|
||||
try:
|
||||
ssh.connect(HOST, username=USER, key_filename=KEY, timeout=10)
|
||||
return ssh
|
||||
except Exception as e:
|
||||
print(f" SSH attempt {attempt+1}/5 failed: {e}")
|
||||
time.sleep(10)
|
||||
print("FATAL: Cannot connect to BC-250")
|
||||
sys.exit(1)
|
||||
|
||||
def run(ssh, cmd, timeout=120):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
return out, err
|
||||
|
||||
def main():
|
||||
ssh = connect()
|
||||
print("Connected to BC-250.\n")
|
||||
|
||||
try:
|
||||
# ── 1. Kill stuck ComfyUI ──
|
||||
print("=== STEP 1: Kill stuck ComfyUI ===")
|
||||
out, _ = run(ssh, "pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; "
|
||||
"pgrep -f 'python3 main.py' || echo 'all_dead'")
|
||||
print(f" {out.strip()}")
|
||||
|
||||
# ── 2. Check available flags ──
|
||||
print("\n=== STEP 2: ComfyUI flags ===")
|
||||
out, err = run(ssh, "bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
||||
"cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'")
|
||||
combined = out + err
|
||||
for line in combined.split('\n'):
|
||||
low = line.lower()
|
||||
if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram',
|
||||
'memory', 'offload', 'precision', 'novram', 'lowvram']):
|
||||
print(f" {line.strip()}")
|
||||
|
||||
# ── 3. Write startup script ──
|
||||
print("\n=== STEP 3: Update startup script ===")
|
||||
script = r"""#!/bin/bash
|
||||
# BC-250 ComfyUI Launcher - GPU inference with CPU VAE decode
|
||||
|
||||
# GPU identity
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# Use all 12 CPU cores
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
# --novram: aggressive offload, one layer at a time to GPU
|
||||
# --force-fp16: halve VRAM for diffusion
|
||||
# --cpu-vae: VAE decode on CPU (GPU hangs on float32 VAE conv2d)
|
||||
# --disable-smart-memory: no memory heuristics
|
||||
exec python3 main.py \
|
||||
--listen 0.0.0.0 --port 8188 \
|
||||
--novram \
|
||||
--force-fp16 \
|
||||
--cpu-vae \
|
||||
--disable-smart-memory
|
||||
"""
|
||||
# Write via heredoc to avoid SFTP
|
||||
escaped = script.replace("'", "'\\''")
|
||||
out, _ = run(ssh, f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n"
|
||||
f"chmod +x /home/fabian/start_comfyui.sh && echo 'written'")
|
||||
print(f" {out.strip()}")
|
||||
|
||||
# ── 4. Launch ComfyUI ──
|
||||
print("\n=== STEP 4: Launch ComfyUI ===")
|
||||
run(ssh, "nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &")
|
||||
time.sleep(2)
|
||||
|
||||
print(" Waiting for server...")
|
||||
for i in range(40):
|
||||
time.sleep(3)
|
||||
out, _ = run(ssh, "curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0")
|
||||
code = out.strip()
|
||||
if code == '200':
|
||||
print(f" Server ready ({(i+1)*3}s)")
|
||||
break
|
||||
if i % 5 == 4:
|
||||
log, _ = run(ssh, "tail -2 /home/fabian/comfyui.log 2>/dev/null")
|
||||
last = [l.strip() for l in log.strip().split('\n') if l.strip()][-1:]
|
||||
print(f" [{(i+1)*3}s] HTTP {code} ... {last[0][:80] if last else ''}")
|
||||
else:
|
||||
print(" Server didn't start in 120s!")
|
||||
out, _ = run(ssh, "tail -40 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
return
|
||||
|
||||
# Show startup log
|
||||
out, _ = run(ssh, "tail -20 /home/fabian/comfyui.log 2>/dev/null")
|
||||
for line in out.strip().split('\n'):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH'):
|
||||
print(f" LOG: {l[:120]}")
|
||||
|
||||
# ── 5. Submit workflow ──
|
||||
print("\n=== STEP 5: Submit workflow (512x512, 8 steps) ===")
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}},
|
||||
"6": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"7": {"class_type": "KSampler",
|
||||
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple",
|
||||
"positive": ["4", 0], "negative": ["5", 0],
|
||||
"latent_image": ["6", 0], "denoise": 1.0}},
|
||||
"8": {"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
|
||||
"9": {"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
|
||||
}
|
||||
}
|
||||
|
||||
wf_json = json.dumps(workflow).replace("'", "'\\''")
|
||||
out, _ = run(ssh, f"curl -s -X POST http://localhost:8188/prompt "
|
||||
f"-H 'Content-Type: application/json' "
|
||||
f"-d '{wf_json}'")
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f" API ERROR: {resp['error']}")
|
||||
if 'node_errors' in resp:
|
||||
for nid, e in resp['node_errors'].items():
|
||||
print(f" Node {nid}: {e}")
|
||||
return
|
||||
print(f" Prompt ID: {resp.get('prompt_id')}")
|
||||
except:
|
||||
print(f" Response: {out.strip()[:500]}")
|
||||
|
||||
# ── 6. Monitor generation ──
|
||||
print("\n=== STEP 6: Monitoring generation ===")
|
||||
last_log = ""
|
||||
for i in range(120): # up to 30 minutes
|
||||
time.sleep(15)
|
||||
|
||||
stats, _ = run(ssh, "bash -c '"
|
||||
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
||||
" MEM=$(ps -p $PID -o rss --no-headers); "
|
||||
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
|
||||
" echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:${LOAD}\"; "
|
||||
"else echo DEAD; fi'")
|
||||
|
||||
log, _ = run(ssh, "tail -10 /home/fabian/comfyui.log 2>/dev/null")
|
||||
log_s = log.strip()
|
||||
|
||||
elapsed = (i+1) * 15
|
||||
m, s = divmod(elapsed, 60)
|
||||
print(f" [{m}m{s:02d}s] {stats.strip()}")
|
||||
|
||||
if log_s != last_log:
|
||||
for line in reversed(log_s.split('\n')):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
|
||||
print(f" LOG: {l[:120]}")
|
||||
break
|
||||
last_log = log_s
|
||||
|
||||
if 'DEAD' in stats:
|
||||
print("\n PROCESS DIED!")
|
||||
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
break
|
||||
|
||||
if 'Prompt executed in' in log_s:
|
||||
print(f"\n SUCCESS! Image generated!")
|
||||
out, _ = run(ssh, "tail -30 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
break
|
||||
|
||||
if 'Traceback' in log_s or 'RuntimeError' in log_s:
|
||||
print("\n ERROR detected!")
|
||||
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
break
|
||||
|
||||
# ── 7. Check output ──
|
||||
print("\n=== Output files ===")
|
||||
out, _ = run(ssh, "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null")
|
||||
print(out.strip())
|
||||
|
||||
finally:
|
||||
ssh.close()
|
||||
print("\nSSH connection closed.")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user