Uploaded sanitized BC250/ROCm Repository.
This commit is contained in:
Fabian
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Switch ComfyUI to --cpu mode so ALL 12 cores are used.
|
||||
The GPU (Cyan Skillfish gfx1013) hangs during HIP inference ops,
|
||||
causing the single-core stall. CPU mode with MKL+OpenMP will use all cores.
|
||||
"""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
import textwrap
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 30:
|
||||
print(f" ({len(lines)} lines, showing last 30)")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
for line in err.strip().split('\n')[-5:]:
|
||||
print(f" STDERR: {line}")
|
||||
return rc, out, err
|
||||
|
||||
# ── Step 1: Kill stuck ComfyUI ──
|
||||
run("bash -c 'pkill -f \"python3 main.py\" 2>/dev/null; sleep 2; "
|
||||
"pkill -9 -f \"python3 main.py\" 2>/dev/null; sleep 1; "
|
||||
"echo \"Killed. Remaining:\"; pgrep -af \"main.py\" || echo none'",
|
||||
desc="Kill stuck ComfyUI")
|
||||
|
||||
# ── Step 2: Write new startup script with --cpu ──
|
||||
# Key: OMP_NUM_THREADS=12 + MKL_NUM_THREADS=12 + --cpu
|
||||
# This uses Intel MKL (built into this PyTorch) for matrix ops across all cores
|
||||
startup_script = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
# ComfyUI CPU-mode launcher for BC-250
|
||||
# Forces ALL computation on CPU using 12 cores via MKL + OpenMP
|
||||
|
||||
# Threading: use ALL 12 cores
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export OMP_PROC_BIND=spread
|
||||
export OMP_PLACES=cores
|
||||
export GOMP_CPU_AFFINITY="0-11"
|
||||
|
||||
# No GPU needed in CPU mode, but keep env for potential future use
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# MKL tuning for multi-core
|
||||
export MKL_DYNAMIC=FALSE
|
||||
export MKL_ENABLE_INSTRUCTIONS=AVX2
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
echo "=== BC-250 ComfyUI CPU Mode ==="
|
||||
echo "Cores: 12, OMP_NUM_THREADS=$OMP_NUM_THREADS, MKL_NUM_THREADS=$MKL_NUM_THREADS"
|
||||
echo "OMP_PROC_BIND=$OMP_PROC_BIND, OMP_PLACES=$OMP_PLACES"
|
||||
|
||||
# --cpu: force ALL ops on CPU (no GPU)
|
||||
# --disable-auto-launch: don't open browser
|
||||
exec python3 main.py --listen 0.0.0.0 --port 8188 --cpu --disable-auto-launch
|
||||
""")
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
||||
f.write(startup_script)
|
||||
sftp.close()
|
||||
run("chmod +x /home/fabian/start_comfyui.sh")
|
||||
print("\n Updated start_comfyui.sh -> --cpu mode, 12 cores, MKL tuning")
|
||||
|
||||
# ── Step 3: Update sitecustomize.py to also set MKL_DYNAMIC=FALSE ──
|
||||
sitecustomize = textwrap.dedent("""\
|
||||
import os
|
||||
os.environ.setdefault('OMP_NUM_THREADS', '12')
|
||||
os.environ.setdefault('MKL_NUM_THREADS', '12')
|
||||
os.environ.setdefault('MKL_DYNAMIC', 'FALSE')
|
||||
os.environ.setdefault('OMP_PROC_BIND', 'spread')
|
||||
os.environ.setdefault('OMP_PLACES', 'cores')
|
||||
|
||||
try:
|
||||
import torch
|
||||
torch.set_num_threads(12)
|
||||
torch.set_num_interop_threads(12)
|
||||
print(f"Threads: intra={torch.get_num_threads()} inter={torch.get_num_interop_threads()}")
|
||||
except Exception:
|
||||
pass
|
||||
""")
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/comfyui-env/lib/python3.14/site-packages/sitecustomize.py', 'w') as f:
|
||||
f.write(sitecustomize)
|
||||
sftp.close()
|
||||
print(" Updated sitecustomize.py with MKL_DYNAMIC=FALSE")
|
||||
|
||||
# ── Step 4: Launch ComfyUI in CPU mode ──
|
||||
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'",
|
||||
desc="Launch ComfyUI in CPU mode")
|
||||
time.sleep(8)
|
||||
|
||||
rc, out, _ = run("bash -c 'tail -30 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc="Startup log")
|
||||
|
||||
# Verify server started
|
||||
for attempt in range(10):
|
||||
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null'")
|
||||
if '200' in out:
|
||||
print(f"\n Server is UP on port 8188 (attempt {attempt+1})")
|
||||
break
|
||||
time.sleep(5)
|
||||
else:
|
||||
print("\n WARNING: Server didn't respond after 50s")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Full log")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
|
||||
# ── Step 5: Submit workflow with slightly smaller image for faster CPU gen ──
|
||||
# 768x432 instead of 1024x576 to speed up first test
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A majestic mountain landscape at sunset, golden light on snow peaks, crystal lake reflection, photorealistic, 8k",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 768, "height": 432, "batch_size": 1}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": ["4", 0],
|
||||
"negative": ["5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_BC250"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/zimage_workflow.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
sftp.close()
|
||||
|
||||
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H \"Content-Type: application/json\" "
|
||||
"-d @/tmp/zimage_workflow.json'",
|
||||
desc="Submit 768x432 workflow")
|
||||
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f"\n ERROR: {resp['error']}")
|
||||
if 'node_errors' in resp:
|
||||
for nid, e in resp['node_errors'].items():
|
||||
print(f" Node {nid}: {e}")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
prompt_id = resp.get('prompt_id', 'unknown')
|
||||
print(f"\n Prompt ID: {prompt_id}")
|
||||
except Exception as e:
|
||||
print(f" Parse error: {e}\n Raw: {out[:500]}")
|
||||
|
||||
# ── Step 6: Monitor CPU/progress ──
|
||||
print("\n Monitoring generation (CPU mode, 12 cores)...")
|
||||
print(" This is a 6B model on CPU — expect several minutes per step")
|
||||
|
||||
start_time = time.time()
|
||||
last_log = ""
|
||||
for i in range(240): # up to 60 min
|
||||
time.sleep(15)
|
||||
elapsed = time.time() - start_time
|
||||
minutes = int(elapsed // 60)
|
||||
seconds = int(elapsed % 60)
|
||||
|
||||
# CPU usage - check if ALL cores are active
|
||||
rc, cpu_out, _ = run("bash -c '"
|
||||
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" echo \"CPU_PCT=$(ps -p $PID -o %cpu= 2>/dev/null)\"; "
|
||||
" echo \"MEM_PCT=$(ps -p $PID -o %mem= 2>/dev/null)\"; "
|
||||
" echo \"THREADS=$(ps -p $PID -o nlwp= 2>/dev/null)\"; "
|
||||
" echo \"LOADAVG=$(cat /proc/loadavg)\"; "
|
||||
"else echo PROCESS_DEAD; fi'")
|
||||
|
||||
# Parse CPU metrics
|
||||
cpu_pct = "?"
|
||||
load_avg = "?"
|
||||
for line in (cpu_out or '').split('\n'):
|
||||
if line.startswith('CPU_PCT='):
|
||||
cpu_pct = line.split('=')[1].strip()
|
||||
if line.startswith('LOADAVG='):
|
||||
load_avg = line.split('=')[1].strip().split()[0]
|
||||
|
||||
# Log tail
|
||||
rc, log_out, _ = run("bash -c 'tail -3 /home/fabian/comfyui.log 2>/dev/null'")
|
||||
log_tail = (log_out or '').strip().split('\n')[-1] if log_out else ""
|
||||
|
||||
if 'PROCESS_DEAD' in (cpu_out or ''):
|
||||
print(f"\n [{minutes}m{seconds}s] PROCESS DIED!")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Death log")
|
||||
break
|
||||
|
||||
# Show progress
|
||||
print(f" [{minutes}m{seconds}s] CPU={cpu_pct}% Load={load_avg} | {log_tail[:80]}")
|
||||
|
||||
if 'Prompt executed in' in (log_out or ''):
|
||||
print(f"\n IMAGE GENERATED! Total time: {minutes}m{seconds}s")
|
||||
run("bash -c 'tail -20 /home/fabian/comfyui.log'", desc="Completion log")
|
||||
run("bash -c 'ls -lah ~/ComfyUI/output/'", desc="Output files")
|
||||
break
|
||||
|
||||
if 'Error' in log_tail or 'Traceback' in (log_out or ''):
|
||||
print(f"\n ERROR DETECTED!")
|
||||
run("bash -c 'tail -60 /home/fabian/comfyui.log'", desc="Error log")
|
||||
break
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
Reference in New Issue
Block a user