Fabian
284 lines
10 KiB
Python
284 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Fix GPU inference: kill stuck, diagnose, restart with --novram, test."""
|
|
import paramiko, json, time, textwrap
|
|
|
|
ssh = paramiko.SSHClient()
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
|
|
|
def run(cmd, timeout=60, desc=""):
|
|
if desc:
|
|
print(f"\n{'='*60}\n {desc}\n{'='*60}")
|
|
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
|
out = stdout.read().decode()
|
|
err = stderr.read().decode()
|
|
rc = stdout.channel.recv_exit_status()
|
|
combined = out.strip()
|
|
if combined:
|
|
lines = combined.split('\n')
|
|
if len(lines) > 50:
|
|
print(f" ... ({len(lines)} lines, showing last 50)")
|
|
print('\n'.join(lines[-50:]))
|
|
else:
|
|
print(combined)
|
|
if err.strip():
|
|
for line in err.strip().split('\n')[-10:]:
|
|
print(f" STDERR: {line}")
|
|
return rc, out, err
|
|
|
|
# ============================================================
|
|
# STEP 1: Kill stuck ComfyUI
|
|
# ============================================================
|
|
run("pkill -f 'python3 main.py' 2>/dev/null; sleep 2; "
|
|
"pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1; "
|
|
"echo 'Killed.'", desc="Kill stuck ComfyUI")
|
|
|
|
# ============================================================
|
|
# STEP 2: Check dmesg for GPU errors
|
|
# ============================================================
|
|
run("dmesg | grep -i -E 'amdgpu|error|fault|gpu|kiq|gfx' | tail -30",
|
|
desc="Check dmesg for GPU errors")
|
|
|
|
# ============================================================
|
|
# STEP 3: Quick GPU sanity test
|
|
# ============================================================
|
|
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
|
"HSA_OVERRIDE_GFX_VERSION=10.1.0 HSA_ENABLE_SDMA=0 "
|
|
"python3 -c \""
|
|
"import torch; "
|
|
"print(f\\\"CUDA available: {torch.cuda.is_available()}\\\"); "
|
|
"print(f\\\"Device: {torch.cuda.get_device_name(0)}\\\"); "
|
|
"a = torch.randn(1024, 1024, device=\\\"cuda\\\"); "
|
|
"b = torch.randn(1024, 1024, device=\\\"cuda\\\"); "
|
|
"c = a @ b; "
|
|
"print(f\\\"Matmul result shape: {c.shape}, sum: {c.sum().item():.2f}\\\"); "
|
|
"# Test fp16 "
|
|
"a16 = a.half(); b16 = b.half(); c16 = a16 @ b16; "
|
|
"print(f\\\"FP16 matmul OK: {c16.shape}\\\"); "
|
|
"print(f\\\"Free VRAM: {torch.cuda.mem_get_info()[0]/1024**2:.0f} MB\\\"); "
|
|
"print(f\\\"Total VRAM: {torch.cuda.mem_get_info()[1]/1024**2:.0f} MB\\\"); "
|
|
"print(\\\"GPU SANITY: PASS\\\")\"'",
|
|
desc="Quick GPU sanity test")
|
|
|
|
# ============================================================
|
|
# STEP 4: Check what ComfyUI flags are available
|
|
# ============================================================
|
|
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && "
|
|
"python3 main.py --help 2>&1 | grep -E \"novram|lowvram|cpu|fp16|force|vram|disable-smart|channels\"'",
|
|
desc="ComfyUI VRAM-related flags")
|
|
|
|
# ============================================================
|
|
# STEP 5: Write new startup script with --novram
|
|
# ============================================================
|
|
startup_script = textwrap.dedent("""\
|
|
#!/bin/bash
|
|
# BC-250 ComfyUI Launcher - GPU mode with aggressive offloading
|
|
|
|
# GPU identity
|
|
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
|
export HIP_VISIBLE_DEVICES=0
|
|
|
|
# Disable SDMA (known issue on Cyan Skillfish)
|
|
export HSA_ENABLE_SDMA=0
|
|
|
|
# Suppress HSA tool warnings
|
|
export HSA_TOOLS_LIB=""
|
|
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
|
|
|
# Threading: use all 12 cores for CPU-side work
|
|
export OMP_NUM_THREADS=12
|
|
export MKL_NUM_THREADS=12
|
|
export OPENBLAS_NUM_THREADS=12
|
|
|
|
# HIP memory: allow expandable segments to reduce fragmentation
|
|
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:True
|
|
|
|
# Activate venv
|
|
source /home/fabian/comfyui-env/bin/activate
|
|
cd /home/fabian/ComfyUI
|
|
|
|
# --novram: most aggressive offloading - keeps almost nothing on GPU,
|
|
# sends individual layers to GPU one at a time during forward pass.
|
|
# This is needed because BC-250 has only ~7.6GB shared VRAM.
|
|
# --disable-smart-memory: prevents ComfyUI from trying to be clever about memory
|
|
# --force-fp16: force fp16 to halve VRAM usage
|
|
exec python3 main.py --listen 0.0.0.0 --port 8188 --novram --force-fp16 --disable-smart-memory
|
|
""")
|
|
|
|
sftp = ssh.open_sftp()
|
|
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
|
f.write(startup_script)
|
|
sftp.close()
|
|
run("chmod +x /home/fabian/start_comfyui.sh", desc="Make script executable")
|
|
print("\n Startup script updated with --novram --force-fp16 --disable-smart-memory")
|
|
|
|
# ============================================================
|
|
# STEP 6: Launch ComfyUI with new settings
|
|
# ============================================================
|
|
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'; sleep 1; echo 'Launched'",
|
|
desc="Launch ComfyUI with --novram")
|
|
|
|
# Wait for server to be ready
|
|
print("\n Waiting for server to start...")
|
|
for i in range(30):
|
|
time.sleep(3)
|
|
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null || echo 0'")
|
|
code = out.strip()
|
|
if code == '200':
|
|
print(f" Server ready after {(i+1)*3}s!")
|
|
break
|
|
# Check log for errors
|
|
rc2, log, _ = run("tail -3 /home/fabian/comfyui.log 2>/dev/null")
|
|
if 'Error' in log or 'error' in log.lower():
|
|
print(f" Log: {log.strip()}")
|
|
print(f" [{(i+1)*3}s] HTTP {code}...")
|
|
else:
|
|
print(" Server didn't start in 90s!")
|
|
run("tail -40 /home/fabian/comfyui.log", desc="Startup log")
|
|
ssh.close()
|
|
exit(1)
|
|
|
|
# Confirm server info
|
|
run("tail -30 /home/fabian/comfyui.log", desc="Startup log")
|
|
|
|
# ============================================================
|
|
# STEP 7: Submit test workflow (smaller 512x512 image first)
|
|
# ============================================================
|
|
workflow = {
|
|
"prompt": {
|
|
"1": {
|
|
"class_type": "UnetLoaderGGUF",
|
|
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
|
|
},
|
|
"2": {
|
|
"class_type": "CLIPLoaderGGUF",
|
|
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
|
|
},
|
|
"3": {
|
|
"class_type": "VAELoader",
|
|
"inputs": {"vae_name": "ae.safetensors"}
|
|
},
|
|
"4": {
|
|
"class_type": "CLIPTextEncode",
|
|
"inputs": {
|
|
"text": "A red fox in a snowy forest, photorealistic",
|
|
"clip": ["2", 0]
|
|
}
|
|
},
|
|
"5": {
|
|
"class_type": "CLIPTextEncode",
|
|
"inputs": {"text": "", "clip": ["2", 0]}
|
|
},
|
|
"6": {
|
|
"class_type": "EmptyLatentImage",
|
|
"inputs": {"width": 512, "height": 512, "batch_size": 1}
|
|
},
|
|
"7": {
|
|
"class_type": "KSampler",
|
|
"inputs": {
|
|
"model": ["1", 0],
|
|
"seed": 12345,
|
|
"steps": 8,
|
|
"cfg": 1.0,
|
|
"sampler_name": "euler",
|
|
"scheduler": "simple",
|
|
"positive": ["4", 0],
|
|
"negative": ["5", 0],
|
|
"latent_image": ["6", 0],
|
|
"denoise": 1.0
|
|
}
|
|
},
|
|
"8": {
|
|
"class_type": "VAEDecode",
|
|
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
|
|
},
|
|
"9": {
|
|
"class_type": "SaveImage",
|
|
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}
|
|
}
|
|
}
|
|
}
|
|
|
|
sftp2 = ssh.open_sftp()
|
|
with sftp2.open('/tmp/zimage_workflow.json', 'w') as f:
|
|
f.write(json.dumps(workflow))
|
|
sftp2.close()
|
|
|
|
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
|
"-H \"Content-Type: application/json\" "
|
|
"-d @/tmp/zimage_workflow.json'",
|
|
desc="Submit 512x512 test workflow")
|
|
|
|
prompt_id = None
|
|
try:
|
|
resp = json.loads(out.strip())
|
|
if 'error' in resp:
|
|
print(f"\n API ERROR: {resp['error']}")
|
|
if 'node_errors' in resp:
|
|
for nid, e in resp['node_errors'].items():
|
|
print(f" Node {nid}: {e}")
|
|
ssh.close()
|
|
exit(1)
|
|
prompt_id = resp.get('prompt_id', 'unknown')
|
|
print(f"\n Prompt ID: {prompt_id}")
|
|
except:
|
|
print(f" Raw response: {out.strip()[:500]}")
|
|
|
|
# ============================================================
|
|
# STEP 8: Monitor generation
|
|
# ============================================================
|
|
print("\n Monitoring GPU generation...")
|
|
last_log = ""
|
|
for i in range(120): # up to 30 minutes
|
|
time.sleep(15)
|
|
|
|
# CPU + GPU status
|
|
rc, status, _ = run("bash -c '"
|
|
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
|
"if [ -n \"$PID\" ]; then "
|
|
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
|
" MEM=$(ps -p $PID -o %mem --no-headers); "
|
|
" THREADS=$(ps -p $PID -o nlwp --no-headers); "
|
|
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
|
|
" GPU_USE=$(cat /sys/class/drm/card1/device/gpu_busy_percent 2>/dev/null || echo N/A); "
|
|
" VRAM_USED=$(cat /sys/class/drm/card1/device/mem_info_vram_used 2>/dev/null || echo 0); "
|
|
" VRAM_TOTAL=$(cat /sys/class/drm/card1/device/mem_info_vram_total 2>/dev/null || echo 1); "
|
|
" echo \"CPU:${CPU}% MEM:${MEM}% THR:${THREADS} LOAD:${LOAD} GPU:${GPU_USE}% VRAM:$((VRAM_USED/1048576))/$((VRAM_TOTAL/1048576))MB\"; "
|
|
"else echo DEAD; fi'")
|
|
|
|
rc, log, _ = run("bash -c 'tail -8 /home/fabian/comfyui.log 2>/dev/null'")
|
|
log_lines = log.strip()
|
|
|
|
# Show status
|
|
status_line = status.strip()
|
|
print(f" [{i+1}] {(i+1)*15}s | {status_line}")
|
|
|
|
# Show new log lines
|
|
if log_lines != last_log:
|
|
new_part = log_lines
|
|
for line in new_part.split('\n')[-5:]:
|
|
if line.strip():
|
|
print(f" LOG: {line.strip()}")
|
|
last_log = log_lines
|
|
|
|
if 'DEAD' in status_line:
|
|
print("\n ComfyUI DIED!")
|
|
run("tail -60 /home/fabian/comfyui.log", desc="Death log")
|
|
break
|
|
|
|
if 'Prompt executed in' in log_lines:
|
|
print(f"\n SUCCESS! Image generated at check {i+1} (~{(i+1)*15}s)")
|
|
break
|
|
|
|
if 'Error' in log_lines or 'Traceback' in log_lines:
|
|
print("\n ERROR detected!")
|
|
run("tail -60 /home/fabian/comfyui.log", desc="Error log")
|
|
break
|
|
|
|
# Final check
|
|
run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null", desc="Output files")
|
|
run("tail -25 /home/fabian/comfyui.log", desc="Final log")
|
|
|
|
ssh.close()
|
|
print("\nDone.")
|