Uploaded sanitized BC250/ROCm Repository.
This commit is contained in:
Fabian
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,123 @@
|
||||
"""Clean old output, re-submit, get REAL GPU timing."""
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# Delete old output
|
||||
print("Cleaning old output...")
|
||||
sh('rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
|
||||
|
||||
# Truncate log to see fresh output only
|
||||
sh('truncate -s 0 /tmp/comfyui.log; sleep 1')
|
||||
|
||||
# Submit fresh workflow
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 123, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
|
||||
}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}
|
||||
}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
|
||||
print("Submitting fresh workflow (seed=123)...")
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f" {resp[:120]}")
|
||||
|
||||
t0 = time.time()
|
||||
print("\nMonitoring (NORMAL_VRAM = real GPU compute)...")
|
||||
|
||||
for i in range(200):
|
||||
elapsed = int(time.time() - t0)
|
||||
|
||||
gpu_temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
|
||||
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
sampling = ''
|
||||
last = ''
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if '/8' in s and ('it/s' in s or 's/it' in s):
|
||||
sampling = s
|
||||
if s and 'FETCH' not in s and 'startup tasks' not in s and 'DEPRECATION' not in s:
|
||||
last = s
|
||||
|
||||
display = sampling if sampling else last[-100:]
|
||||
print(f" [{elapsed:>4}s] {temp_c}C | {display}")
|
||||
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
exec_time = ''
|
||||
for line in log.split('\n'):
|
||||
if 'Prompt executed in' in line:
|
||||
exec_time = line.strip()
|
||||
print(f"\n *** IMAGE GENERATED! ***")
|
||||
print(f" File: {imgs}")
|
||||
print(f" {exec_time}")
|
||||
print(f" Wall time: {elapsed}s")
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['loaded completely', 'loaded partially', '/8', 'Prompt executed']):
|
||||
print(f" {s}")
|
||||
break
|
||||
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 20:
|
||||
time.sleep(2)
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** IMAGE: {imgs} ***")
|
||||
else:
|
||||
print(f"\n Queue empty, no image:")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
except: pass
|
||||
|
||||
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
|
||||
if alive == 'N':
|
||||
print(f"\n CRASHED!")
|
||||
for line in log.split('\n')[-25:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Quick check: what's ACTUALLY happening in the ComfyUI log right now?"""
|
||||
import paramiko
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
|
||||
def sh(cmd):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(30)
|
||||
chan.exec_command(f"/bin/bash -c '{cmd}'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
print("=== FULL LOG (minus ComfyUI-Manager spam) ===")
|
||||
# Show all lines EXCEPT the registry fetch / manager spam
|
||||
log = sh("grep -v 'FETCH ComfyRegistry\\|All startup tasks\\|ComfyUI-Manager' /tmp/comfyui.log | tail -50")
|
||||
print(log)
|
||||
|
||||
print("\n=== PROCESS ===")
|
||||
print(sh("ps aux | grep python3 | grep -v grep"))
|
||||
|
||||
print("\n=== GPU sysfs ===")
|
||||
# Find the actual gpu_busy path
|
||||
print(sh("find /sys/class/drm/ -name 'gpu_busy_percent' 2>/dev/null"))
|
||||
print(sh("cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null; cat /sys/class/drm/card1/device/gpu_busy_percent 2>/dev/null"))
|
||||
|
||||
print("\n=== rocm-smi ===")
|
||||
print(sh("rocm-smi 2>/dev/null | head -15"))
|
||||
|
||||
print("\n=== OUTPUT DIR ===")
|
||||
print(sh("ls -la ~/ComfyUI/output/ 2>/dev/null"))
|
||||
|
||||
print("\n=== QUEUE ===")
|
||||
print(sh("curl -s http://127.0.0.1:8188/queue 2>/dev/null"))
|
||||
|
||||
print("\n=== Log lines with 'load' or 'sample' or 'error' or '%' ===")
|
||||
print(sh("grep -iE 'load|sample|error|%|step|Traceback|OOM|killed' /tmp/comfyui.log | tail -30"))
|
||||
|
||||
c.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check full log after sampling for VAE decode status."""
|
||||
import paramiko
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
def run(cmd):
|
||||
_, so, se = ssh.exec_command(cmd, timeout=15)
|
||||
return so.read().decode()
|
||||
|
||||
# Get more log lines - look for everything after the sampling
|
||||
print("=== FULL LOG (last 60 lines) ===")
|
||||
print(run("tail -60 /home/fabian/comfyui.log 2>/dev/null"))
|
||||
|
||||
# Check output directory
|
||||
print("\n=== OUTPUT FILES ===")
|
||||
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null"))
|
||||
|
||||
# Queue status
|
||||
print("=== QUEUE ===")
|
||||
print(run("curl -s http://localhost:8188/queue 2>/dev/null"))
|
||||
|
||||
# History
|
||||
print("\n=== HISTORY ===")
|
||||
print(run("curl -s http://localhost:8188/history 2>/dev/null")[:2000])
|
||||
|
||||
# Process count and wchan
|
||||
print("\n=== PROCESS STATE ===")
|
||||
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"cat /proc/$PID/wchan 2>/dev/null; echo; "
|
||||
"ps -L -p $PID -o tid,%cpu,comm --sort=-%cpu 2>/dev/null | head -20'"))
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check PyTorch build progress on BC-250."""
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
print(f"STDERR: {err.strip()}")
|
||||
|
||||
# Build process status
|
||||
run("pgrep -fa 'setup.py|build_pytorch|cmake|ninja|hipcc|cc1plus' | head -20",
|
||||
desc="Build processes")
|
||||
|
||||
# How long has it been running
|
||||
run("ps -p 350823 -o etime=,cmd= 2>/dev/null || echo 'Process no longer running'",
|
||||
desc="Build process uptime")
|
||||
|
||||
# Build log tail
|
||||
run("tail -60 /home/fabian/pytorch_build.log 2>/dev/null || echo 'No log file'",
|
||||
desc="Build log (last 60 lines)")
|
||||
|
||||
# Memory usage
|
||||
run("free -h", desc="Memory status")
|
||||
|
||||
# Check for build completion marker
|
||||
run("grep 'BUILD_COMPLETE' /home/fabian/pytorch_build.log 2>/dev/null || echo 'Build still in progress'",
|
||||
desc="Build completion check")
|
||||
|
||||
# Check for any errors in log
|
||||
run("grep -i 'error:\\|fatal:\\|failed' /home/fabian/pytorch_build.log 2>/dev/null | tail -10 || echo 'No errors found'",
|
||||
desc="Error check")
|
||||
|
||||
# Disk space
|
||||
run("df -h / | tail -1", desc="Disk space")
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check the build error from PyTorch CMake on BC-250."""
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
|
||||
# Get the full cmake error
|
||||
run("grep -A5 -B2 'Error\\|error\\|FATAL\\|fatal\\|Could not find' /home/fabian/pytorch_build.log | head -60",
|
||||
desc="CMake errors in build log")
|
||||
|
||||
# Also check the cmake output file if it exists
|
||||
run("cat /home/fabian/pytorch/build/CMakeFiles/CMakeOutput.log 2>/dev/null | tail -30 || echo 'no output log'",
|
||||
desc="CMake output log")
|
||||
|
||||
run("cat /home/fabian/pytorch/build/CMakeFiles/CMakeError.log 2>/dev/null | tail -50 || echo 'no error log'",
|
||||
desc="CMake error log")
|
||||
|
||||
# Check specifically what's missing
|
||||
run("grep -i 'not found\\|could not find\\|missing' /home/fabian/pytorch_build.log | head -20",
|
||||
desc="Missing packages")
|
||||
|
||||
# Also check if the process is still running
|
||||
run("pgrep -fa 'setup.py\\|build_pytorch' || echo 'Build process not running'",
|
||||
desc="Build process status")
|
||||
|
||||
# Check roctracer
|
||||
run("find /opt/rocm -name 'roctracer*' 2>/dev/null | head -10",
|
||||
desc="roctracer files")
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check ComfyUI flags and update startup."""
|
||||
import paramiko
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=60):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
return out + err
|
||||
|
||||
# Kill any leftover
|
||||
print(run("pkill -9 -f 'python3 main.py' 2>/dev/null; echo killed"))
|
||||
|
||||
# Full help output
|
||||
print("=== ComfyUI --help ===")
|
||||
help_text = run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'")
|
||||
# Filter for interesting lines
|
||||
for line in help_text.split('\n'):
|
||||
low = line.lower()
|
||||
if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram', 'memory', 'offload', 'precision']):
|
||||
print(f" {line.strip()}")
|
||||
|
||||
# Also just dump the full thing to see everything
|
||||
print("\n=== FULL HELP ===")
|
||||
print(help_text)
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check PyTorch state and start fresh build on BC-250."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 60:
|
||||
print(f" ... ({len(lines)} lines, showing last 60)")
|
||||
print('\n'.join(lines[-60:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Check if there's a wheel already built
|
||||
run("ls -lh ~/pytorch/dist/*.whl 2>/dev/null || echo 'No wheels found'",
|
||||
desc="Check for existing PyTorch wheels")
|
||||
|
||||
# Check for any previous build directory
|
||||
run("ls -la ~/pytorch/build/ 2>/dev/null | head -10 || echo 'No build dir'",
|
||||
desc="Check build directory")
|
||||
|
||||
# Check if pytorch is already installed in venv
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \"import torch; print(torch.__version__); print(torch.version.hip); print(torch.cuda.is_available())\" 2>&1'",
|
||||
desc="Check if PyTorch is already installed")
|
||||
|
||||
# Check pytorch source integrity
|
||||
run("ls ~/pytorch/setup.py ~/pytorch/CMakeLists.txt 2>&1",
|
||||
desc="Verify PyTorch source files")
|
||||
|
||||
# Verify ROCm works before build
|
||||
run("bash -c 'export HSA_OVERRIDE_GFX_VERSION=10.1.0 && /opt/rocm/bin/rocminfo 2>&1 | grep -E \"gfx|Marketing\" | head -5'",
|
||||
desc="Verify ROCm is working")
|
||||
|
||||
# Check venv
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && which python3 && python3 --version'",
|
||||
desc="Verify venv")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone checking state.")
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quick check on ComfyUI status."""
|
||||
import paramiko
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
return stdout.read().decode()
|
||||
|
||||
# Log tail
|
||||
print("=== LOG (last 40 lines) ===")
|
||||
print(run("tail -40 /home/fabian/comfyui.log 2>/dev/null"))
|
||||
|
||||
# Process status
|
||||
print("=== PROCESS ===")
|
||||
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" ps -p $PID -o pid,%cpu,%mem,nlwp,stat --no-headers; "
|
||||
" echo \"LOAD: $(cat /proc/loadavg)\"; "
|
||||
"else echo DEAD; fi'"))
|
||||
|
||||
# rocm-smi
|
||||
print("=== GPU ===")
|
||||
print(run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null || echo 'no rocm-smi'"))
|
||||
|
||||
# Queue
|
||||
print("=== QUEUE ===")
|
||||
print(run("curl -s http://localhost:8188/queue 2>/dev/null || echo 'no connection'"))
|
||||
|
||||
# History
|
||||
print("=== HISTORY ===")
|
||||
hist = run("curl -s http://localhost:8188/history 2>/dev/null || echo 'no connection'")
|
||||
import json
|
||||
try:
|
||||
h = json.loads(hist)
|
||||
for pid, info in h.items():
|
||||
print(f" Prompt: {pid}")
|
||||
print(f" Status: {info.get('status', {})}")
|
||||
outputs = info.get('outputs', {})
|
||||
if outputs:
|
||||
for nid, nout in outputs.items():
|
||||
if isinstance(nout, dict):
|
||||
for key, val in nout.items():
|
||||
print(f" Output node {nid}/{key}: {str(val)[:200]}")
|
||||
else:
|
||||
print(" No outputs")
|
||||
except:
|
||||
print(hist[:1000])
|
||||
|
||||
# Output directory
|
||||
print("\n=== OUTPUT FILES ===")
|
||||
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null"))
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read full ops.py and check torch thread defaults, then fix threading."""
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')[-10:]
|
||||
print(f"STDERR: {chr(10).join(lines)}")
|
||||
print(f" Exit: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Check current torch thread defaults
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && "
|
||||
"export HSA_OVERRIDE_GFX_VERSION=10.1.0 && "
|
||||
"python -c \""
|
||||
"import torch; "
|
||||
"print(f\\\"num_threads={torch.get_num_threads()}\\\"); "
|
||||
"print(f\\\"num_interop_threads={torch.get_num_interop_threads()}\\\"); "
|
||||
"import os; "
|
||||
"print(f\\\"OMP_NUM_THREADS={os.environ.get(\\\\\\\"OMP_NUM_THREADS\\\\\\\", \\\\\\\"not set\\\\\\\")}\\\"); "
|
||||
"print(f\\\"MKL_NUM_THREADS={os.environ.get(\\\\\\\"MKL_NUM_THREADS\\\\\\\", \\\\\\\"not set\\\\\\\")}\\\"); "
|
||||
"\"'",
|
||||
desc="Check default torch thread settings")
|
||||
|
||||
# Read forward_ggml_cast_weights (where dequant happens during inference)
|
||||
run("bash -c 'sed -n \"200,281p\" ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py'",
|
||||
desc="ops.py lines 200-281 (forward functions)")
|
||||
|
||||
# Check __init__.py for any loading/patching
|
||||
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/__init__.py 2>/dev/null | head -40'",
|
||||
desc="ComfyUI-GGUF __init__.py")
|
||||
|
||||
# Check nodes.py for model loading
|
||||
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/nodes.py 2>/dev/null'",
|
||||
desc="ComfyUI-GGUF nodes.py (model loader)")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Switch ComfyUI to --cpu mode so ALL 12 cores are used.
|
||||
The GPU (Cyan Skillfish gfx1013) hangs during HIP inference ops,
|
||||
causing the single-core stall. CPU mode with MKL+OpenMP will use all cores.
|
||||
"""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
import textwrap
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 30:
|
||||
print(f" ({len(lines)} lines, showing last 30)")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
for line in err.strip().split('\n')[-5:]:
|
||||
print(f" STDERR: {line}")
|
||||
return rc, out, err
|
||||
|
||||
# ── Step 1: Kill stuck ComfyUI ──
|
||||
run("bash -c 'pkill -f \"python3 main.py\" 2>/dev/null; sleep 2; "
|
||||
"pkill -9 -f \"python3 main.py\" 2>/dev/null; sleep 1; "
|
||||
"echo \"Killed. Remaining:\"; pgrep -af \"main.py\" || echo none'",
|
||||
desc="Kill stuck ComfyUI")
|
||||
|
||||
# ── Step 2: Write new startup script with --cpu ──
|
||||
# Key: OMP_NUM_THREADS=12 + MKL_NUM_THREADS=12 + --cpu
|
||||
# This uses Intel MKL (built into this PyTorch) for matrix ops across all cores
|
||||
startup_script = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
# ComfyUI CPU-mode launcher for BC-250
|
||||
# Forces ALL computation on CPU using 12 cores via MKL + OpenMP
|
||||
|
||||
# Threading: use ALL 12 cores
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export OMP_PROC_BIND=spread
|
||||
export OMP_PLACES=cores
|
||||
export GOMP_CPU_AFFINITY="0-11"
|
||||
|
||||
# No GPU needed in CPU mode, but keep env for potential future use
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# MKL tuning for multi-core
|
||||
export MKL_DYNAMIC=FALSE
|
||||
export MKL_ENABLE_INSTRUCTIONS=AVX2
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
echo "=== BC-250 ComfyUI CPU Mode ==="
|
||||
echo "Cores: 12, OMP_NUM_THREADS=$OMP_NUM_THREADS, MKL_NUM_THREADS=$MKL_NUM_THREADS"
|
||||
echo "OMP_PROC_BIND=$OMP_PROC_BIND, OMP_PLACES=$OMP_PLACES"
|
||||
|
||||
# --cpu: force ALL ops on CPU (no GPU)
|
||||
# --disable-auto-launch: don't open browser
|
||||
exec python3 main.py --listen 0.0.0.0 --port 8188 --cpu --disable-auto-launch
|
||||
""")
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
||||
f.write(startup_script)
|
||||
sftp.close()
|
||||
run("chmod +x /home/fabian/start_comfyui.sh")
|
||||
print("\n Updated start_comfyui.sh -> --cpu mode, 12 cores, MKL tuning")
|
||||
|
||||
# ── Step 3: Update sitecustomize.py to also set MKL_DYNAMIC=FALSE ──
|
||||
sitecustomize = textwrap.dedent("""\
|
||||
import os
|
||||
os.environ.setdefault('OMP_NUM_THREADS', '12')
|
||||
os.environ.setdefault('MKL_NUM_THREADS', '12')
|
||||
os.environ.setdefault('MKL_DYNAMIC', 'FALSE')
|
||||
os.environ.setdefault('OMP_PROC_BIND', 'spread')
|
||||
os.environ.setdefault('OMP_PLACES', 'cores')
|
||||
|
||||
try:
|
||||
import torch
|
||||
torch.set_num_threads(12)
|
||||
torch.set_num_interop_threads(12)
|
||||
print(f"Threads: intra={torch.get_num_threads()} inter={torch.get_num_interop_threads()}")
|
||||
except Exception:
|
||||
pass
|
||||
""")
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/comfyui-env/lib/python3.14/site-packages/sitecustomize.py', 'w') as f:
|
||||
f.write(sitecustomize)
|
||||
sftp.close()
|
||||
print(" Updated sitecustomize.py with MKL_DYNAMIC=FALSE")
|
||||
|
||||
# ── Step 4: Launch ComfyUI in CPU mode ──
|
||||
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'",
|
||||
desc="Launch ComfyUI in CPU mode")
|
||||
time.sleep(8)
|
||||
|
||||
rc, out, _ = run("bash -c 'tail -30 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc="Startup log")
|
||||
|
||||
# Verify server started
|
||||
for attempt in range(10):
|
||||
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null'")
|
||||
if '200' in out:
|
||||
print(f"\n Server is UP on port 8188 (attempt {attempt+1})")
|
||||
break
|
||||
time.sleep(5)
|
||||
else:
|
||||
print("\n WARNING: Server didn't respond after 50s")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Full log")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
|
||||
# ── Step 5: Submit workflow with slightly smaller image for faster CPU gen ──
|
||||
# 768x432 instead of 1024x576 to speed up first test
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A majestic mountain landscape at sunset, golden light on snow peaks, crystal lake reflection, photorealistic, 8k",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 768, "height": 432, "batch_size": 1}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": ["4", 0],
|
||||
"negative": ["5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_BC250"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/zimage_workflow.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
sftp.close()
|
||||
|
||||
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H \"Content-Type: application/json\" "
|
||||
"-d @/tmp/zimage_workflow.json'",
|
||||
desc="Submit 768x432 workflow")
|
||||
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f"\n ERROR: {resp['error']}")
|
||||
if 'node_errors' in resp:
|
||||
for nid, e in resp['node_errors'].items():
|
||||
print(f" Node {nid}: {e}")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
prompt_id = resp.get('prompt_id', 'unknown')
|
||||
print(f"\n Prompt ID: {prompt_id}")
|
||||
except Exception as e:
|
||||
print(f" Parse error: {e}\n Raw: {out[:500]}")
|
||||
|
||||
# ── Step 6: Monitor CPU/progress ──
|
||||
print("\n Monitoring generation (CPU mode, 12 cores)...")
|
||||
print(" This is a 6B model on CPU — expect several minutes per step")
|
||||
|
||||
start_time = time.time()
|
||||
last_log = ""
|
||||
for i in range(240): # up to 60 min
|
||||
time.sleep(15)
|
||||
elapsed = time.time() - start_time
|
||||
minutes = int(elapsed // 60)
|
||||
seconds = int(elapsed % 60)
|
||||
|
||||
# CPU usage - check if ALL cores are active
|
||||
rc, cpu_out, _ = run("bash -c '"
|
||||
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" echo \"CPU_PCT=$(ps -p $PID -o %cpu= 2>/dev/null)\"; "
|
||||
" echo \"MEM_PCT=$(ps -p $PID -o %mem= 2>/dev/null)\"; "
|
||||
" echo \"THREADS=$(ps -p $PID -o nlwp= 2>/dev/null)\"; "
|
||||
" echo \"LOADAVG=$(cat /proc/loadavg)\"; "
|
||||
"else echo PROCESS_DEAD; fi'")
|
||||
|
||||
# Parse CPU metrics
|
||||
cpu_pct = "?"
|
||||
load_avg = "?"
|
||||
for line in (cpu_out or '').split('\n'):
|
||||
if line.startswith('CPU_PCT='):
|
||||
cpu_pct = line.split('=')[1].strip()
|
||||
if line.startswith('LOADAVG='):
|
||||
load_avg = line.split('=')[1].strip().split()[0]
|
||||
|
||||
# Log tail
|
||||
rc, log_out, _ = run("bash -c 'tail -3 /home/fabian/comfyui.log 2>/dev/null'")
|
||||
log_tail = (log_out or '').strip().split('\n')[-1] if log_out else ""
|
||||
|
||||
if 'PROCESS_DEAD' in (cpu_out or ''):
|
||||
print(f"\n [{minutes}m{seconds}s] PROCESS DIED!")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Death log")
|
||||
break
|
||||
|
||||
# Show progress
|
||||
print(f" [{minutes}m{seconds}s] CPU={cpu_pct}% Load={load_avg} | {log_tail[:80]}")
|
||||
|
||||
if 'Prompt executed in' in (log_out or ''):
|
||||
print(f"\n IMAGE GENERATED! Total time: {minutes}m{seconds}s")
|
||||
run("bash -c 'tail -20 /home/fabian/comfyui.log'", desc="Completion log")
|
||||
run("bash -c 'ls -lah ~/ComfyUI/output/'", desc="Output files")
|
||||
break
|
||||
|
||||
if 'Error' in log_tail or 'Traceback' in (log_out or ''):
|
||||
print(f"\n ERROR DETECTED!")
|
||||
run("bash -c 'tail -60 /home/fabian/comfyui.log'", desc="Error log")
|
||||
break
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,31 @@
|
||||
import paramiko, time
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
|
||||
# 1) Full diagnostic
|
||||
cmds = {
|
||||
"LOG_LAST_40": "tail -40 /tmp/comfyui.log 2>/dev/null",
|
||||
"PROCESS": "ps aux | grep -E 'python|comfy' | grep -v grep",
|
||||
"GPU": "cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null",
|
||||
"GPU_TEMP": "cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null",
|
||||
"CPU_CORES": "mpstat -P ALL 1 1 2>/dev/null | tail -15 || top -bn1 | head -5",
|
||||
"MEM": "free -m",
|
||||
"OUTPUT": "ls -la ~/ComfyUI/output/ 2>/dev/null",
|
||||
"QUEUE": "curl -s http://127.0.0.1:8188/queue 2>/dev/null",
|
||||
"ROCM_CHECK": "rocm-smi --showuse --showtemp --showpower 2>/dev/null | head -20",
|
||||
}
|
||||
|
||||
for name, cmd in cmds.items():
|
||||
print(f"\n=== {name} ===")
|
||||
_, o, e = c.exec_command(cmd)
|
||||
out = o.read().decode(errors='replace').strip()
|
||||
err = e.read().decode(errors='replace').strip()
|
||||
print(out if out else "(empty)")
|
||||
if err and name not in ("ROCM_CHECK",):
|
||||
print(f" STDERR: {err}")
|
||||
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Diagnostic: check full log and GPU state."""
|
||||
import paramiko
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
return out, err
|
||||
|
||||
# Full log (last 80 lines)
|
||||
out, _ = run("tail -80 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print("=== FULL LOG (last 80 lines) ===")
|
||||
print(out)
|
||||
|
||||
# Process state
|
||||
out, _ = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" echo \"PID: $PID\"; "
|
||||
" echo \"=== PROCESS STATE ===\"; "
|
||||
" cat /proc/$PID/status | grep -E \"State|Threads|VmRSS|VmSize\"; "
|
||||
" echo \"=== WCHAN (what syscall is process blocked on) ===\"; "
|
||||
" cat /proc/$PID/wchan 2>/dev/null; echo; "
|
||||
" echo \"=== STACK TRACE (kernel) ===\"; "
|
||||
" sudo cat /proc/$PID/stack 2>/dev/null || echo \"no permission\"; "
|
||||
" echo \"=== TOP THREADS ===\"; "
|
||||
" ps -L -p $PID -o tid,%cpu,comm --sort=-%cpu | head -15; "
|
||||
"fi'")
|
||||
print(out)
|
||||
|
||||
# GPU info
|
||||
out, _ = run("bash -c 'rocm-smi 2>/dev/null || echo no rocm-smi; "
|
||||
"echo \"=== dmesg GPU ===\"; "
|
||||
"dmesg 2>/dev/null | grep -i -E \"amdgpu|error|fault\" | tail -15 || echo no-dmesg'")
|
||||
print("=== GPU ===")
|
||||
print(out)
|
||||
|
||||
# Memory
|
||||
out, _ = run("free -h")
|
||||
print("=== MEMORY ===")
|
||||
print(out)
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check ComfyUI API queue/history for errors."""
|
||||
import paramiko, json
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
return stdout.read().decode(), stderr.read().decode()
|
||||
|
||||
# Queue status
|
||||
out, _ = run("curl -s http://localhost:8188/queue")
|
||||
print("=== QUEUE ===")
|
||||
try:
|
||||
q = json.loads(out)
|
||||
print(f"Running: {len(q.get('queue_running', []))}")
|
||||
print(f"Pending: {len(q.get('queue_pending', []))}")
|
||||
except:
|
||||
print(out[:500])
|
||||
|
||||
# History
|
||||
out, _ = run("curl -s http://localhost:8188/history")
|
||||
print("\n=== HISTORY ===")
|
||||
try:
|
||||
h = json.loads(out)
|
||||
for pid, info in h.items():
|
||||
print(f"\nPrompt ID: {pid}")
|
||||
status = info.get('status', {})
|
||||
print(f" Status: {status}")
|
||||
outputs = info.get('outputs', {})
|
||||
for nid, nout in outputs.items():
|
||||
print(f" Node {nid}: {list(nout.keys()) if isinstance(nout, dict) else nout}")
|
||||
if not outputs:
|
||||
print(" NO OUTPUTS")
|
||||
except:
|
||||
print(out[:2000])
|
||||
|
||||
# Check stderr output (nohup might redirect differently)
|
||||
out, _ = run("cat /home/fabian/comfyui_err.log 2>/dev/null || echo 'no err log'")
|
||||
print(f"\n=== STDERR LOG ===\n{out[:2000]}")
|
||||
|
||||
# Check full nohup output
|
||||
out, _ = run("wc -l /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(f"\n=== LOG LINES: {out.strip()}")
|
||||
|
||||
# Check if there are processes actively computing
|
||||
out, _ = run("bash -c 'top -bn1 | head -20'")
|
||||
print(f"\n=== TOP ===\n{out}")
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deep diagnosis: WHY only 1 core? Check OpenMP, threading, GGUF code path."""
|
||||
import paramiko, json, textwrap
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=60):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
return rc, out, err
|
||||
|
||||
def show(label, cmd, timeout=60):
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {label}")
|
||||
print(f"{'='*60}")
|
||||
rc, out, err = run(cmd, timeout)
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
for line in err.strip().split('\n')[-10:]:
|
||||
print(f"STDERR: {line}")
|
||||
return out
|
||||
|
||||
# 1. Check ComfyUI queue/history - did generation succeed or fail?
|
||||
show("Queue status", "curl -s http://localhost:8188/queue")
|
||||
hist_out = show("History", "curl -s http://localhost:8188/history")
|
||||
try:
|
||||
h = json.loads(hist_out.strip())
|
||||
for pid, info in h.items():
|
||||
status = info.get('status', {})
|
||||
print(f"\n Prompt {pid}: status={status}")
|
||||
outputs = info.get('outputs', {})
|
||||
if outputs:
|
||||
print(f" Outputs: {json.dumps(outputs, indent=2)[:500]}")
|
||||
else:
|
||||
print(" NO OUTPUTS")
|
||||
except:
|
||||
pass
|
||||
|
||||
# 2. Check if PyTorch has OpenMP
|
||||
show("PyTorch OpenMP & threading",
|
||||
"bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
||||
"OMP_NUM_THREADS=12 python3 -c \""
|
||||
"import torch; "
|
||||
"print(f\\\"OpenMP available: {torch.backends.openmp.is_available()}\\\"); "
|
||||
"print(f\\\"MKL available: {torch.backends.mkl.is_available()}\\\"); "
|
||||
"print(f\\\"Num threads: {torch.get_num_threads()}\\\"); "
|
||||
"print(f\\\"Num interop threads: {torch.get_num_interop_threads()}\\\"); "
|
||||
"print(f\\\"torch.__config__.show(): \\\"); "
|
||||
"print(torch.__config__.show()); "
|
||||
"\"'")
|
||||
|
||||
# 3. Check if libomp/libgomp is available
|
||||
show("OpenMP libraries",
|
||||
"bash -c 'ldconfig -p 2>/dev/null | grep -i omp; "
|
||||
"echo ---; "
|
||||
"pacman -Qs openmp 2>/dev/null; "
|
||||
"echo ---; "
|
||||
"pacman -Qs libgomp 2>/dev/null; "
|
||||
"echo ---; "
|
||||
"ls -la /usr/lib/libomp* /usr/lib/libgomp* 2>/dev/null || echo none'")
|
||||
|
||||
# 4. Check pytorch shared lib dependencies for OpenMP
|
||||
show("PyTorch .so OpenMP deps",
|
||||
"bash -c 'ldd /usr/lib/python3.14/site-packages/torch/lib/libtorch_cpu.so 2>/dev/null | grep -i omp'")
|
||||
|
||||
# 5. Actual thread test - does a matrix multiply use multiple cores?
|
||||
show("Matrix multiply CPU benchmark (should use all cores)",
|
||||
"bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
||||
"OMP_NUM_THREADS=12 python3 -c \""
|
||||
"import torch, time, os; "
|
||||
"print(f\\\"PID: {os.getpid()}\\\"); "
|
||||
"torch.set_num_threads(12); "
|
||||
"print(f\\\"Threads set to: {torch.get_num_threads()}\\\"); "
|
||||
"a = torch.randn(4096, 4096); "
|
||||
"b = torch.randn(4096, 4096); "
|
||||
"# warmup; "
|
||||
"c = torch.mm(a, b); "
|
||||
"import subprocess; "
|
||||
"# Start monitoring in background; "
|
||||
"start = time.time(); "
|
||||
"for i in range(5): c = torch.mm(a, b); "
|
||||
"elapsed = time.time() - start; "
|
||||
"print(f\\\"5x matmul 4096x4096: {elapsed:.2f}s\\\"); "
|
||||
"\"'")
|
||||
|
||||
# 6. Check what the GGUF dequant code actually does (single-threaded python loop?)
|
||||
show("GGUF dequant code - is there a Python for-loop?",
|
||||
"bash -c 'grep -n \"for \" /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py | head -20; "
|
||||
"echo \"---\"; "
|
||||
"grep -n \"for \" /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py | head -20; "
|
||||
"echo \"---\"; "
|
||||
"grep -n \"def dequantize\" /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py'")
|
||||
|
||||
# 7. Check if lowvram is causing sequential layer-by-layer processing
|
||||
show("ComfyUI lowvram model loading code",
|
||||
"bash -c 'grep -rn \"lowvram\\|low_vram\\|offload\" /home/fabian/ComfyUI/comfy/model_management.py 2>/dev/null | head -30'")
|
||||
|
||||
# 8. Output directory
|
||||
show("Output files", "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null")
|
||||
|
||||
ssh.close()
|
||||
print("\n\nDONE.")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Diagnose and fix VAE hang. Check log, fix threading, restart."""
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=30):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# 1. What's running?
|
||||
print("=== CURRENT STATE ===")
|
||||
ps = sh('ps aux | grep main.py | grep -v grep')
|
||||
print(f"Process: {ps or 'NONE'}")
|
||||
|
||||
# Check env of running process
|
||||
env = sh(r'cat /proc/$(pgrep -f "python3.*main.py" | head -1)/environ 2>/dev/null | tr "\0" "\n" | grep -E "OMP|MKL|COMFYUI|THREAD|OPENBLAS"')
|
||||
print(f"Env:\n{env}")
|
||||
|
||||
# 2. Log tail
|
||||
print("\n=== LOG TAIL ===")
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n')[-40:]:
|
||||
s = line.strip()
|
||||
if s: print(f" {s}")
|
||||
except Exception as e:
|
||||
log = ''
|
||||
print(f" No log: {e}")
|
||||
|
||||
# 3. Check launcher
|
||||
print("\n=== LAUNCHER ===")
|
||||
try:
|
||||
with sftp.open('/tmp/run_comfyui.sh', 'r') as f:
|
||||
print(f.read().decode())
|
||||
except: print(" No launcher")
|
||||
|
||||
# 4. model_management.py patch?
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
mm = f.read().decode()
|
||||
print(f"SHARED patch: {'YES' if 'COMFYUI_SHARED_MEMORY' in mm else 'NO'}")
|
||||
|
||||
# 5. Check torch threads in same env
|
||||
print("\n=== TORCH THREADS ===")
|
||||
tcheck = sh('''source ~/comfyui-env/bin/activate
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
python3 -c "
|
||||
import torch, os
|
||||
print(f'torch.get_num_threads() = {torch.get_num_threads()}')
|
||||
print(f'OMP_NUM_THREADS = {os.environ.get(chr(34)+'OMP_NUM_THREADS'+chr(34), chr(34)+'NOT SET'+chr(34))}')
|
||||
"''', timeout=30)
|
||||
print(tcheck)
|
||||
|
||||
# 6. Check sitecustomize
|
||||
print("\n=== SITECUSTOMIZE ===")
|
||||
sc = sh('cat ~/comfyui-env/lib/python*/site-packages/sitecustomize.py 2>/dev/null || echo MISSING')
|
||||
print(sc[:500])
|
||||
|
||||
# 7. Key log lines
|
||||
print("\n=== KEY LOG ENTRIES ===")
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(x in s.lower() for x in ['vram state', 'shared', 'loaded', 'offloaded', 'device:', 'total vram', 'vae', 'thread']):
|
||||
print(f" {s}")
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDiag done.")
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Diagnose ComfyUI state on BC-250 — check if stuck or OOM."""
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')[-10:]
|
||||
print(f"STDERR: {chr(10).join(lines)}")
|
||||
print(f" Exit: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Check if ComfyUI process is alive
|
||||
run("bash -c 'ps aux | grep \"python main.py\" | grep -v grep'",
|
||||
desc="ComfyUI process status")
|
||||
|
||||
# Memory state
|
||||
run("bash -c 'free -h'", desc="RAM/Swap usage")
|
||||
|
||||
# GPU VRAM
|
||||
run("bash -c 'cat /sys/class/drm/card1/device/mem_info_vram_used 2>/dev/null; "
|
||||
"echo \"---\"; cat /sys/class/drm/card1/device/mem_info_vram_total 2>/dev/null'",
|
||||
desc="GPU VRAM usage")
|
||||
|
||||
# dmesg for OOM
|
||||
run("bash -c 'dmesg | tail -20'", desc="Recent kernel messages")
|
||||
|
||||
# Last 80 lines of comfyui log
|
||||
run("bash -c 'tail -80 /home/fabian/comfyui.log 2>/dev/null'", desc="ComfyUI log (last 80)")
|
||||
|
||||
# Check if port still listening
|
||||
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_GONE'", desc="Port 8188")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Diagnose GPU hang: kill stuck ComfyUI, run targeted HIP tests,
|
||||
check what ops hang on Cyan Skillfish gfx1013->gfx1010.
|
||||
Single SSH connection, properly closed.
|
||||
"""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
import sys
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
for attempt in range(5):
|
||||
try:
|
||||
ssh.connect('192.168.178.150', username='fabian',
|
||||
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f" SSH attempt {attempt+1}/5: {e}")
|
||||
time.sleep(10)
|
||||
else:
|
||||
print("FATAL: Cannot connect"); sys.exit(1)
|
||||
|
||||
def run(cmd, timeout=120):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
return out, err
|
||||
|
||||
try:
|
||||
# 1. Kill stuck ComfyUI
|
||||
print("=== Kill stuck ComfyUI ===")
|
||||
out, _ = run("pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; echo killed")
|
||||
print(f" {out.strip()}")
|
||||
|
||||
# 2. Check ComfyUI help for --cpu-vae flag existence
|
||||
print("\n=== Check if --cpu-vae exists ===")
|
||||
out, err = run("bash -c 'source ~/comfyui-env/bin/activate && cd ~/ComfyUI && python3 main.py --help 2>&1'")
|
||||
full_help = out + err
|
||||
has_cpu_vae = '--cpu-vae' in full_help
|
||||
print(f" --cpu-vae flag exists: {has_cpu_vae}")
|
||||
# Print all vram/gpu related flags
|
||||
for line in full_help.split('\n'):
|
||||
if any(w in line.lower() for w in ['vram', 'cpu', 'gpu', 'fp16', 'fp32', 'vae', 'force', 'precision']):
|
||||
print(f" {line.strip()}")
|
||||
|
||||
# 3. Check what the CachyOS pytorch-rocm was built for
|
||||
print("\n=== PyTorch ROCm build info ===")
|
||||
out, _ = run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \""
|
||||
"import torch; "
|
||||
"print(f\\\"PyTorch version: {torch.__version__}\\\"); "
|
||||
"print(f\\\"CUDA/HIP available: {torch.cuda.is_available()}\\\"); "
|
||||
"print(f\\\"ROCm version: {torch.version.hip}\\\"); "
|
||||
"print(f\\\"Device: {torch.cuda.get_device_name(0)}\\\"); "
|
||||
"print(f\\\"Arch: {torch.cuda.get_device_capability(0)}\\\"); "
|
||||
"print(f\\\"VRAM free/total: {torch.cuda.mem_get_info()[0]//1048576}/{torch.cuda.mem_get_info()[1]//1048576} MB\\\"); "
|
||||
"\"' 2>&1")
|
||||
print(out.strip())
|
||||
|
||||
# 4. Targeted GPU op tests - find what hangs
|
||||
print("\n=== GPU operation tests (timeout 30s each) ===")
|
||||
tests = [
|
||||
("Basic matmul fp32",
|
||||
"a=torch.randn(256,256,device='cuda'); b=a@a; print(f'fp32 matmul: {b.shape} sum={b.sum().item():.1f}')"),
|
||||
("Basic matmul fp16",
|
||||
"a=torch.randn(256,256,device='cuda').half(); b=a@a; print(f'fp16 matmul: {b.shape} sum={b.sum().item():.1f}')"),
|
||||
("Conv2d fp16 (VAE-like)",
|
||||
"import torch.nn as nn; c=nn.Conv2d(128,128,3,padding=1).half().cuda(); x=torch.randn(1,128,64,64,device='cuda').half(); y=c(x); print(f'conv2d fp16: {y.shape}')"),
|
||||
("Conv2d fp32 (VAE default)",
|
||||
"import torch.nn as nn; c=nn.Conv2d(128,128,3,padding=1).cuda(); x=torch.randn(1,128,64,64,device='cuda'); y=c(x); print(f'conv2d fp32: {y.shape}')"),
|
||||
("GroupNorm fp16",
|
||||
"import torch.nn as nn; gn=nn.GroupNorm(32,128).half().cuda(); x=torch.randn(1,128,32,32,device='cuda').half(); y=gn(x); print(f'groupnorm fp16: {y.shape}')"),
|
||||
("GroupNorm fp32",
|
||||
"import torch.nn as nn; gn=nn.GroupNorm(32,128).cuda(); x=torch.randn(1,128,32,32,device='cuda'); y=gn(x); print(f'groupnorm fp32: {y.shape}')"),
|
||||
("LayerNorm fp16",
|
||||
"import torch.nn as nn; ln=nn.LayerNorm(256).half().cuda(); x=torch.randn(1,64,256,device='cuda').half(); y=ln(x); print(f'layernorm fp16: {y.shape}')"),
|
||||
("Linear fp16 (DiT-like)",
|
||||
"import torch.nn as nn; l=nn.Linear(1024,1024).half().cuda(); x=torch.randn(1,64,1024,device='cuda').half(); y=l(x); print(f'linear fp16: {y.shape}')"),
|
||||
("Attention fp16 (scaled_dot_product)",
|
||||
"q=torch.randn(1,8,64,64,device='cuda').half(); k=q.clone(); v=q.clone(); "
|
||||
"y=torch.nn.functional.scaled_dot_product_attention(q,k,v); print(f'sdpa fp16: {y.shape}')"),
|
||||
("Attention fp32 (scaled_dot_product)",
|
||||
"q=torch.randn(1,8,64,64,device='cuda'); k=q.clone(); v=q.clone(); "
|
||||
"y=torch.nn.functional.scaled_dot_product_attention(q,k,v); print(f'sdpa fp32: {y.shape}')"),
|
||||
("Large matmul fp16 (5032x5032)",
|
||||
"a=torch.randn(2048,2048,device='cuda').half(); b=a@a; print(f'large fp16 matmul: {b.shape} sum={b.sum().item():.1f}')"),
|
||||
("RoPE-like op (complex multiply)",
|
||||
"x=torch.randn(1,8,64,64,device='cuda').half(); "
|
||||
"f=torch.randn(64,32,2,device='cuda').half(); "
|
||||
"print(f'rope input shapes: x={x.shape} f={f.shape} OK')"),
|
||||
("torch.compile basic test",
|
||||
"import torch._dynamo; f=lambda x: x*2+1; cf=torch.compile(f); "
|
||||
"x=torch.randn(100,device='cuda'); y=cf(x); print(f'compile: {y.shape}')"),
|
||||
]
|
||||
|
||||
for name, code in tests:
|
||||
print(f"\n Testing: {name}...", end=" ", flush=True)
|
||||
cmd = (f"bash -c 'timeout 30 bash -c \""
|
||||
f"source ~/comfyui-env/bin/activate && "
|
||||
f"HSA_OVERRIDE_GFX_VERSION=10.1.0 HSA_ENABLE_SDMA=0 "
|
||||
f"python3 -c \\\"import torch; {code}\\\"\" 2>&1 || echo TIMEOUT_OR_ERROR'")
|
||||
out, err = run(cmd, timeout=40)
|
||||
result = (out + err).strip()
|
||||
if 'TIMEOUT_OR_ERROR' in result:
|
||||
# Get just the error part
|
||||
lines = result.split('\n')
|
||||
for l in reversed(lines):
|
||||
if l.strip() and l.strip() != 'TIMEOUT_OR_ERROR':
|
||||
print(f"FAILED: {l.strip()[:100]}")
|
||||
break
|
||||
else:
|
||||
print("TIMEOUT (GPU HANG)")
|
||||
elif result:
|
||||
last_line = [l for l in result.split('\n') if l.strip()][-1] if result.split('\n') else result
|
||||
print(f"OK: {last_line.strip()[:100]}")
|
||||
else:
|
||||
print("NO OUTPUT (possible hang)")
|
||||
|
||||
# 5. Check dmesg for GPU errors after tests
|
||||
print("\n\n=== dmesg GPU errors (last 20) ===")
|
||||
out, _ = run("dmesg 2>/dev/null | grep -i -E 'amdgpu|gpu|gfx|error|fault' | tail -20 || echo 'no permission'")
|
||||
print(out.strip() if out.strip() else " (empty or no permission)")
|
||||
|
||||
# 6. Check rocm-smi for GPU health
|
||||
print("\n=== GPU health after tests ===")
|
||||
out, _ = run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null")
|
||||
for line in out.split('\n'):
|
||||
if any(c in line for c in ['°C', '%', 'Device', 'Node']):
|
||||
print(f" {line.strip()}")
|
||||
|
||||
finally:
|
||||
ssh.close()
|
||||
print("\n\nSSH connection closed.")
|
||||
@@ -0,0 +1,165 @@
|
||||
import paramiko, time, json, textwrap
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# 1) Kill
|
||||
print("1) Kill")
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
|
||||
# 2) New launcher: --novram streams weights (proven GPU 136W), no --cpu-vae
|
||||
print("2) Write launcher")
|
||||
launcher = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export MIOPEN_FIND_MODE=3
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
# --novram: weights in system RAM, GPU computes via streaming (337MB buffer fits in 512MB real VRAM)
|
||||
# --force-fp16: half precision
|
||||
# NO --cpu-vae: let VAE run on GPU (320MB fits in 512MB VRAM)
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--novram \\
|
||||
--force-fp16
|
||||
""")
|
||||
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
|
||||
f.write(launcher)
|
||||
sh('chmod +x /tmp/run_comfyui.sh')
|
||||
|
||||
# 3) Start
|
||||
print("3) Start")
|
||||
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
print(f" PID: {pid}")
|
||||
|
||||
# 4) Wait ready
|
||||
print("4) Wait HTTP", end='', flush=True)
|
||||
for i in range(90):
|
||||
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if '200' in code:
|
||||
print(f" OK ({i*2}s)")
|
||||
break
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
|
||||
# Show mode
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['vram state', 'Device:', 'Total VRAM', 'offloading']):
|
||||
print(f" {s}")
|
||||
|
||||
# 5) Submit
|
||||
print("5) Submit")
|
||||
wf = {"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 777, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(wf))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f" {resp[:120]}")
|
||||
|
||||
# 6) Monitor
|
||||
print("6) Monitor")
|
||||
t0 = time.time()
|
||||
for i in range(200):
|
||||
el = int(time.time() - t0)
|
||||
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
tc = int(temp)//1000 if temp.isdigit() else '?'
|
||||
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
samp = ''
|
||||
last = ''
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if '/8' in s and ('it/s' in s or 's/it' in s): samp = s
|
||||
if s and 'FETCH' not in s and 'startup' not in s and 'DEPRECATION' not in s: last = s
|
||||
|
||||
print(f" [{el:>4}s] {tc}C | {(samp or last)[-90:]}")
|
||||
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
et = ''
|
||||
for line in log.split('\n'):
|
||||
if 'Prompt executed' in line: et = line.strip()
|
||||
print(f"\n *** DONE! *** {imgs}")
|
||||
print(f" {et}")
|
||||
print(f" Wall: {el}s")
|
||||
# Show key log lines
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['loaded', '/8', 'Prompt executed', 'VAE', 'Requested']):
|
||||
if 'FETCH' not in s:
|
||||
print(f" {s}")
|
||||
break
|
||||
|
||||
# Queue empty?
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and el > 30:
|
||||
time.sleep(2)
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** DONE: {imgs} ***")
|
||||
else:
|
||||
print(f"\n Queue empty, no image:")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip() and 'FETCH' not in line: print(f" {line.strip()}")
|
||||
break
|
||||
except: pass
|
||||
|
||||
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
|
||||
print("\n CRASHED!")
|
||||
for line in log.split('\n')[-25:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,168 @@
|
||||
"""FAST FIX: patch offload devices, restart. No fluff."""
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, t=30):
|
||||
ch = c.get_transport().open_session()
|
||||
ch.settimeout(t)
|
||||
ch.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
o = b""
|
||||
while True:
|
||||
try:
|
||||
d = ch.recv(65536)
|
||||
if not d: break
|
||||
o += d
|
||||
except: break
|
||||
ch.close()
|
||||
return o.decode(errors='replace').strip()
|
||||
|
||||
# 1. KILL
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 1')
|
||||
print("Killed")
|
||||
|
||||
# 2. READ
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
code = f.read().decode()
|
||||
|
||||
# 3. PATCH: unet_offload_device - return GPU for SHARED too
|
||||
# Find the function and add SHARED check
|
||||
changed = False
|
||||
|
||||
# Patch unet_offload_device: "HIGH_VRAM" -> "HIGH_VRAM or SHARED"
|
||||
if 'def unet_offload_device' in code:
|
||||
lines = code.split('\n')
|
||||
for i, line in enumerate(lines):
|
||||
if 'def unet_offload_device' in line:
|
||||
# Look at next few lines for the HIGH_VRAM check
|
||||
for j in range(i, min(i+8, len(lines))):
|
||||
if 'HIGH_VRAM' in lines[j] and 'SHARED' not in lines[j] and 'unet_offload' not in lines[j]:
|
||||
old = lines[j]
|
||||
lines[j] = old.replace('VRAMState.HIGH_VRAM', 'VRAMState.HIGH_VRAM or vram_state == VRAMState.SHARED')
|
||||
print(f"Patched unet_offload L{j+1}: {lines[j].strip()}")
|
||||
changed = True
|
||||
break
|
||||
break
|
||||
code = '\n'.join(lines)
|
||||
|
||||
# Patch vae_offload_device: "args.gpu_only" -> "args.gpu_only or SHARED"
|
||||
if 'def vae_offload_device' in code:
|
||||
lines = code.split('\n')
|
||||
for i, line in enumerate(lines):
|
||||
if 'def vae_offload_device' in line:
|
||||
for j in range(i, min(i+8, len(lines))):
|
||||
if 'gpu_only' in lines[j] and 'SHARED' not in lines[j]:
|
||||
old = lines[j]
|
||||
lines[j] = old.replace('args.gpu_only', '(args.gpu_only or vram_state == VRAMState.SHARED)')
|
||||
print(f"Patched vae_offload L{j+1}: {lines[j].strip()}")
|
||||
changed = True
|
||||
break
|
||||
break
|
||||
code = '\n'.join(lines)
|
||||
|
||||
# Also patch text_encoder_offload_device if it offloads to CPU
|
||||
if 'def text_encoder_offload_device' in code:
|
||||
lines = code.split('\n')
|
||||
for i, line in enumerate(lines):
|
||||
if 'def text_encoder_offload_device' in line:
|
||||
for j in range(i, min(i+8, len(lines))):
|
||||
if 'gpu_only' in lines[j] and 'SHARED' not in lines[j]:
|
||||
old = lines[j]
|
||||
lines[j] = old.replace('args.gpu_only', '(args.gpu_only or vram_state == VRAMState.SHARED)')
|
||||
print(f"Patched text_enc_offload L{j+1}: {lines[j].strip()}")
|
||||
changed = True
|
||||
break
|
||||
break
|
||||
code = '\n'.join(lines)
|
||||
|
||||
if changed:
|
||||
sh('cp /home/fabian/ComfyUI/comfy/model_management.py /home/fabian/ComfyUI/comfy/model_management.py.bak3')
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
|
||||
f.write(code)
|
||||
print("Written!")
|
||||
else:
|
||||
print("Already patched or structure changed")
|
||||
|
||||
# 4. RESTART
|
||||
sh('rm -f /tmp/comfyui.log')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(4)
|
||||
print(f"PID: {sh('pgrep -f python3.*main.py')}")
|
||||
|
||||
# 5. WAIT FOR READY
|
||||
for i in range(60):
|
||||
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', t=5)
|
||||
if '200' in r: print(f"Ready ({i*2}s)"); break
|
||||
time.sleep(2)
|
||||
|
||||
# Quick check
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if any(x in s for x in ['vram state', 'SHARED', 'Device:']): print(f" {s}")
|
||||
|
||||
# 6. SUBMIT
|
||||
wf = {"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(wf))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f"Submitted: {resp[:100]}")
|
||||
|
||||
# 7. MONITOR - compact, fast checks
|
||||
print("\nWaiting for image...")
|
||||
t0 = time.time()
|
||||
last_shown = ''
|
||||
for i in range(180):
|
||||
el = int(time.time() - t0)
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
# Find latest status
|
||||
status = ''
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if any(x in s for x in ['/8', 'loaded', 'Requested', 'VAE', 'Prompt executed', 'Error']):
|
||||
if 'FETCH' not in s: status = s
|
||||
|
||||
if status != last_shown:
|
||||
gpu = sh('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null', t=5)
|
||||
print(f" [{el:>3}s] GPU:{gpu}% | {status[-100:]}")
|
||||
last_shown = status
|
||||
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', t=5)
|
||||
if imgs:
|
||||
print(f"\n*** DONE in {el}s! ***")
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if any(x in s for x in ['load device', 'offload device', 'loaded completely', 'Prompt executed']):
|
||||
print(f" {s}")
|
||||
break
|
||||
|
||||
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', t=5) == 'N':
|
||||
print(f"\nCRASHED at {el}s!")
|
||||
for l in log.split('\n')[-15:]:
|
||||
if l.strip(): print(f" {l.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix single-core bottleneck: set threading env vars and patch ComfyUI-GGUF for parallel dequant."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd[:300]}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 50:
|
||||
print(f" ... ({len(lines)} lines, showing last 50)")
|
||||
print('\n'.join(lines[-50:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-15:] if len(lines) > 15 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# 1. Kill stuck ComfyUI
|
||||
run("bash -c 'kill -9 484588 2>/dev/null; pkill -9 -f \"python main.py\" 2>/dev/null; sleep 2; echo done'",
|
||||
desc="Kill stuck ComfyUI process")
|
||||
|
||||
# 2. Check how many CPU cores
|
||||
run("bash -c 'nproc'", desc="CPU core count")
|
||||
|
||||
# 3. Check the ComfyUI-GGUF dequant code to understand the bottleneck
|
||||
run("bash -c 'grep -rn \"dequant\\|num_threads\\|torch.set_num_threads\\|ThreadPool\\|parallel\" ~/ComfyUI/custom_nodes/ComfyUI-GGUF/*.py 2>/dev/null | head -30'",
|
||||
desc="Search for threading in ComfyUI-GGUF")
|
||||
|
||||
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py 2>/dev/null | head -80'",
|
||||
desc="ComfyUI-GGUF ops.py (dequant logic)")
|
||||
|
||||
# 4. Check the dequant function
|
||||
run("bash -c 'grep -n \"def dequantize\\|class GGMLTensor\\|def forward\" ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py 2>/dev/null'",
|
||||
desc="Key functions in ops.py")
|
||||
|
||||
run("bash -c 'wc -l ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py ~/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py 2>/dev/null'",
|
||||
desc="File sizes")
|
||||
|
||||
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py 2>/dev/null | head -60'",
|
||||
desc="dequant.py start")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix threading: set all 12 cores for PyTorch ops, patch ComfyUI startup, restart."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd[:300]}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 40:
|
||||
print(f" ... ({len(lines)} lines, showing last 40)")
|
||||
print('\n'.join(lines[-40:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-15:] if len(lines) > 15 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# 1. Kill any existing ComfyUI
|
||||
run("bash -c 'pkill -9 -f \"python main.py\" 2>/dev/null; sleep 1; echo killed'",
|
||||
desc="Kill existing ComfyUI")
|
||||
|
||||
# 2. Check the loader.py for the Dequantizing message source
|
||||
run("bash -c 'grep -rn \"Dequantizing\" ~/ComfyUI/ --include=\"*.py\" 2>/dev/null | head -10'",
|
||||
desc="Find 'Dequantizing' message source")
|
||||
|
||||
# 3. Check torch thread defaults without any env vars
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \"import torch; print(torch.get_num_threads(), torch.get_num_interop_threads())\"'",
|
||||
desc="Default torch thread count")
|
||||
|
||||
# 4. Verify it works with env vars
|
||||
run("bash -c 'export OMP_NUM_THREADS=12; export MKL_NUM_THREADS=12; "
|
||||
"source ~/comfyui-env/bin/activate && python3 -c \"import torch; "
|
||||
"torch.set_num_threads(12); torch.set_num_interop_threads(4); "
|
||||
"print(torch.get_num_threads(), torch.get_num_interop_threads())\"'",
|
||||
desc="Torch threads with env vars set to 12")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix: maximize threading to 12 cores, add --lowvram for 7.6GB VRAM, restart ComfyUI."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd[:300]}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 40:
|
||||
print(f" ... ({len(lines)} lines, showing last 40)")
|
||||
print('\n'.join(lines[-40:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-15:] if len(lines) > 15 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# 1. Kill any existing ComfyUI
|
||||
run("bash -c 'pkill -9 -f \"python main.py\" 2>/dev/null; sleep 2; echo ok'",
|
||||
desc="Kill existing ComfyUI")
|
||||
|
||||
# 2. Write updated start_comfyui.sh with full threading + lowvram
|
||||
startup_script = r'''#!/bin/bash
|
||||
# ComfyUI Startup Script for AsRock BC-250 (AMD Cyan Skillfish / ROCm 7.2)
|
||||
# Optimized for all 12 CPU cores + 7.6GB shared VRAM
|
||||
set -euo pipefail
|
||||
|
||||
# ═══════════════ BC-250 GPU Environment ═══════════════
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# ═══════════════ THREADING — ALL 12 CORES ═══════════════
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export VECLIB_MAXIMUM_THREADS=12
|
||||
export NUMEXPR_NUM_THREADS=12
|
||||
# PyTorch intra-op (tensor math) and inter-op (parallel node execution) threads
|
||||
export TORCH_NUM_THREADS=12
|
||||
|
||||
# ═══════════════ Memory / ROCm tuning ═══════════════
|
||||
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
|
||||
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
|
||||
# Disable HIP memory caching to avoid fragmentation on shared VRAM
|
||||
export PYTORCH_NO_HIP_MEMORY_CACHING=0
|
||||
|
||||
# ═══════════════ Activate venv ═══════════════
|
||||
source "$HOME/comfyui-env/bin/activate"
|
||||
|
||||
cd "$HOME/ComfyUI"
|
||||
|
||||
# Force PyTorch to use all 12 cores
|
||||
python3 -c "import torch; torch.set_num_threads(12); torch.set_num_interop_threads(12); print(f'Threads: intra={torch.get_num_threads()} inter={torch.get_num_interop_threads()}')"
|
||||
|
||||
echo "=========================================="
|
||||
echo " ComfyUI on BC-250 (ROCm 7.2)"
|
||||
echo " GPU: AMD Cyan Skillfish (gfx1013→gfx1010)"
|
||||
echo " PyTorch: $(python3 -c 'import torch; print(torch.__version__)')"
|
||||
echo " HIP: $(python3 -c 'import torch; print(torch.version.hip)')"
|
||||
echo " CUDA: $(python3 -c 'import torch; print(torch.cuda.is_available())')"
|
||||
echo " CPU: $(nproc) cores (all used)"
|
||||
echo " VRAM: 7.6GB shared — using --lowvram mode"
|
||||
echo "=========================================="
|
||||
|
||||
# Default: listen on all interfaces, --lowvram for 7.6GB shared VRAM
|
||||
LISTEN_ARGS="--listen 0.0.0.0 --port 8188 --lowvram"
|
||||
|
||||
if [ $# -gt 0 ]; then
|
||||
LISTEN_ARGS="$@"
|
||||
fi
|
||||
|
||||
echo "Starting: python main.py $LISTEN_ARGS"
|
||||
echo "Access at: http://192.168.178.150:8188"
|
||||
echo ""
|
||||
|
||||
# Set threads inside the actual process too
|
||||
exec python3 -c "
|
||||
import torch, sys, os
|
||||
torch.set_num_threads(12)
|
||||
torch.set_num_interop_threads(12)
|
||||
# Now exec ComfyUI main
|
||||
sys.argv = ['main.py'] + '$LISTEN_ARGS'.split()
|
||||
exec(open('main.py').read())
|
||||
"
|
||||
'''
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
||||
f.write(startup_script)
|
||||
sftp.close()
|
||||
|
||||
run("chmod +x /home/fabian/start_comfyui.sh", desc="Make script executable")
|
||||
|
||||
# 3. Launch ComfyUI with new settings
|
||||
run("bash -c 'rm -f /home/fabian/comfyui.log'", desc="Clean old log")
|
||||
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
|
||||
desc="Launch ComfyUI with 12-core threading + lowvram")
|
||||
|
||||
# 4. Wait for startup
|
||||
time.sleep(15)
|
||||
run("bash -c 'tail -30 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc="Startup log")
|
||||
|
||||
time.sleep(10)
|
||||
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
|
||||
desc="Check port 8188")
|
||||
|
||||
# Verify threads are set
|
||||
run("bash -c 'tail -40 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc="Full startup log")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix startup: use proper subprocess instead of exec, set threads via sitecustomize."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd[:300]}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 40:
|
||||
print(f" ... ({len(lines)} lines, showing last 40)")
|
||||
print('\n'.join(lines[-40:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-15:] if len(lines) > 15 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# 1. Create a sitecustomize.py in the venv to set threads on import
|
||||
sitecustomize = '''# Auto-set PyTorch threading to use all 12 CPU cores on BC-250
|
||||
import os
|
||||
os.environ.setdefault("OMP_NUM_THREADS", "12")
|
||||
os.environ.setdefault("MKL_NUM_THREADS", "12")
|
||||
os.environ.setdefault("OPENBLAS_NUM_THREADS", "12")
|
||||
|
||||
try:
|
||||
import torch
|
||||
torch.set_num_threads(12)
|
||||
torch.set_num_interop_threads(12)
|
||||
except Exception:
|
||||
pass
|
||||
'''
|
||||
|
||||
# Find the venv site-packages path
|
||||
rc, out, _ = run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \"import site; print(site.getsitepackages()[0])\"'",
|
||||
desc="Find venv site-packages")
|
||||
site_packages = out.strip()
|
||||
print(f" Site-packages: {site_packages}")
|
||||
|
||||
# Write sitecustomize.py
|
||||
sftp = ssh.open_sftp()
|
||||
sitecust_path = f"{site_packages}/sitecustomize.py"
|
||||
# Check if it exists first
|
||||
try:
|
||||
sftp.stat(sitecust_path)
|
||||
print(f" sitecustomize.py already exists, backing up")
|
||||
sftp.rename(sitecust_path, f"{sitecust_path}.bak")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
with sftp.open(sitecust_path, 'w') as f:
|
||||
f.write(sitecustomize)
|
||||
sftp.close()
|
||||
print(f" Written: {sitecust_path}")
|
||||
|
||||
# 2. Update startup script — simple, using exec python main.py directly
|
||||
startup_script = r'''#!/bin/bash
|
||||
# ComfyUI Startup Script for AsRock BC-250 (AMD Cyan Skillfish / ROCm 7.2)
|
||||
# All 12 CPU cores + lowvram for 7.6GB shared VRAM
|
||||
set -euo pipefail
|
||||
|
||||
# ═══════════════ BC-250 GPU ═══════════════
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# ═══════════════ ALL 12 CORES ═══════════════
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export VECLIB_MAXIMUM_THREADS=12
|
||||
export NUMEXPR_NUM_THREADS=12
|
||||
|
||||
# ═══════════════ Memory tuning ═══════════════
|
||||
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
|
||||
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
|
||||
|
||||
# ═══════════════ Activate venv ═══════════════
|
||||
source "$HOME/comfyui-env/bin/activate"
|
||||
cd "$HOME/ComfyUI"
|
||||
|
||||
echo "=========================================="
|
||||
echo " ComfyUI on BC-250 (ROCm 7.2)"
|
||||
echo " GPU: AMD Cyan Skillfish (gfx1010)"
|
||||
echo " PyTorch: $(python3 -c 'import torch; print(torch.__version__)')"
|
||||
echo " Threads: $(python3 -c 'import torch; print(f"intra={torch.get_num_threads()} inter={torch.get_num_interop_threads()}")')"
|
||||
echo " CPU: $(nproc) cores"
|
||||
echo " VRAM: 7.6GB shared — lowvram mode"
|
||||
echo "=========================================="
|
||||
|
||||
# Default args: listen on all, lowvram for tight VRAM
|
||||
ARGS="--listen 0.0.0.0 --port 8188 --lowvram"
|
||||
if [ $# -gt 0 ]; then
|
||||
ARGS="$@"
|
||||
fi
|
||||
|
||||
echo "Starting: python main.py $ARGS"
|
||||
echo "Access: http://192.168.178.150:8188"
|
||||
echo ""
|
||||
|
||||
exec python3 main.py $ARGS
|
||||
'''
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
||||
f.write(startup_script)
|
||||
sftp.close()
|
||||
|
||||
run("chmod +x /home/fabian/start_comfyui.sh", desc="Make executable")
|
||||
|
||||
# 3. Launch
|
||||
run("bash -c 'rm -f /home/fabian/comfyui.log'", desc="Clean log")
|
||||
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
|
||||
desc="Launch ComfyUI (12 cores + lowvram)")
|
||||
|
||||
time.sleep(20)
|
||||
run("bash -c 'tail -40 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc="Startup log")
|
||||
|
||||
time.sleep(10)
|
||||
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
|
||||
desc="Check port 8188")
|
||||
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc="Full log")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix torchvision compatibility on BC-250.
|
||||
|
||||
The pip-installed torchvision conflicts with the system python-pytorch-rocm.
|
||||
Need to use system torchvision-rocm or fix the version.
|
||||
"""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 40:
|
||||
print(f" ... ({len(lines)} lines, showing last 40)")
|
||||
print('\n'.join(lines[-40:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Check what torchvision packages exist in repos
|
||||
run("bash -c 'pacman -Ss torchvision 2>/dev/null'",
|
||||
desc="Search for torchvision packages in repos")
|
||||
|
||||
run("bash -c 'pacman -Ss torchaudio 2>/dev/null'",
|
||||
desc="Search for torchaudio packages")
|
||||
|
||||
# Check what's currently installed
|
||||
run("bash -c 'pacman -Qs torch 2>/dev/null'",
|
||||
desc="Currently installed torch packages (system)")
|
||||
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && pip list 2>/dev/null | grep -i torch'",
|
||||
desc="torch packages in venv")
|
||||
|
||||
# Check the versions
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && python -c \""
|
||||
"import torch; print(f\\\"torch: {torch.__version__} from {torch.__file__}\\\"); "
|
||||
"\"'",
|
||||
desc="Check torch location")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix torchvision — use system package instead of pip version."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 40:
|
||||
print(f" ... ({len(lines)} lines, showing last 40)")
|
||||
print('\n'.join(lines[-40:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# 1. Uninstall pip torchvision and torchaudio from venv
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && pip uninstall -y torchvision torchaudio 2>&1'",
|
||||
desc="Uninstall pip torchvision and torchaudio from venv")
|
||||
|
||||
# 2. Install system python-torchvision via pacman
|
||||
run("sudo pacman -S --noconfirm python-torchvision",
|
||||
desc="Install system python-torchvision (matches system pytorch)")
|
||||
|
||||
# 3. Verify torchvision now works
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
||||
"export HSA_OVERRIDE_GFX_VERSION=10.1.0 && "
|
||||
"export HIP_VISIBLE_DEVICES=0 && "
|
||||
"export HSA_ENABLE_SDMA=0 && "
|
||||
"python -c \""
|
||||
"import torch; print(f\\\"torch {torch.__version__} from {torch.__file__}\\\"); "
|
||||
"import torchvision; print(f\\\"torchvision {torchvision.__version__} from {torchvision.__file__}\\\"); "
|
||||
"print(\\\"torchvision ops OK\\\"); "
|
||||
"\"'",
|
||||
desc="Verify torchvision import works")
|
||||
|
||||
# 4. Kill old ComfyUI and relaunch
|
||||
run("bash -c 'pkill -f \"python main.py\" 2>/dev/null; sleep 2; echo done'",
|
||||
desc="Kill old ComfyUI process")
|
||||
|
||||
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
|
||||
desc="Relaunch ComfyUI")
|
||||
|
||||
time.sleep(20)
|
||||
run("tail -40 /home/fabian/comfyui.log 2>/dev/null",
|
||||
desc="ComfyUI startup log")
|
||||
|
||||
time.sleep(10)
|
||||
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
|
||||
desc="Check if port 8188 is listening")
|
||||
|
||||
run("tail -60 /home/fabian/comfyui.log 2>/dev/null",
|
||||
desc="Full ComfyUI log")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix VAE decode hang: kill stuck, check available flags, restart with --cpu-vae."""
|
||||
import paramiko, json, time, textwrap
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=60, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}\n {desc}\n{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 50:
|
||||
print(f" ... ({len(lines)} lines, showing last 50)")
|
||||
print('\n'.join(lines[-50:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
for l in err.strip().split('\n')[-5:]:
|
||||
print(f" STDERR: {l}")
|
||||
return rc, out, err
|
||||
|
||||
# Kill stuck
|
||||
run("pkill -f 'python3 main.py' 2>/dev/null; sleep 2; pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1; echo killed",
|
||||
desc="Kill stuck ComfyUI")
|
||||
|
||||
# Check available VAE flags
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && "
|
||||
"python3 main.py --help 2>&1 | grep -i -E \"vae|fp16|fp32|force|cpu|novram|lowvram\"'",
|
||||
desc="ComfyUI VAE/VRAM flags")
|
||||
|
||||
# Update startup script: add --cpu-vae to keep diffusion on GPU but VAE on CPU
|
||||
startup_script = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
# BC-250 ComfyUI Launcher — GPU inference with CPU VAE decode
|
||||
# Diffusion sampling: GPU (~6s/step, 8 steps = 51s total)
|
||||
# VAE decode: CPU (GPU hangs on float32 VAE ops on Cyan Skillfish)
|
||||
# Text encoding: CPU (GGUF model, dequant on CPU)
|
||||
|
||||
# GPU identity
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# Use all 12 CPU cores
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
# --novram: send one layer at a time to GPU (needed for 7.6GB shared VRAM)
|
||||
# --force-fp16: halve VRAM usage for diffusion model
|
||||
# --cpu-vae: decode VAE on CPU (GPU hangs on VAE float32 conv2d ops)
|
||||
# --disable-smart-memory: prevent memory heuristics from interfering
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--novram \\
|
||||
--force-fp16 \\
|
||||
--cpu-vae \\
|
||||
--disable-smart-memory
|
||||
""")
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
||||
f.write(startup_script)
|
||||
sftp.close()
|
||||
run("chmod +x /home/fabian/start_comfyui.sh")
|
||||
print("\n Updated: added --cpu-vae (GPU sampler + CPU VAE)")
|
||||
|
||||
# Launch
|
||||
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'; sleep 1; echo launched",
|
||||
desc="Launch ComfyUI")
|
||||
|
||||
print("\n Waiting for server...")
|
||||
for i in range(40):
|
||||
time.sleep(3)
|
||||
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null || echo 0'")
|
||||
if out.strip() == '200':
|
||||
print(f" Server ready! ({(i+1)*3}s)")
|
||||
break
|
||||
if i % 5 == 4:
|
||||
rc2, log, _ = run("tail -3 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(f" [{(i+1)*3}s] waiting... {log.strip().split(chr(10))[-1][:80]}")
|
||||
else:
|
||||
print(" Timeout!")
|
||||
run("tail -40 /home/fabian/comfyui.log", desc="Log")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
|
||||
# Submit workflow
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A red fox in a snowy forest, photorealistic",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": ["4", 0],
|
||||
"negative": ["5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sftp2 = ssh.open_sftp()
|
||||
with sftp2.open('/tmp/zimage_workflow.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
sftp2.close()
|
||||
|
||||
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H \"Content-Type: application/json\" "
|
||||
"-d @/tmp/zimage_workflow.json'",
|
||||
desc="Submit workflow (GPU sampling + CPU VAE)")
|
||||
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f" ERROR: {resp['error']}")
|
||||
if 'node_errors' in resp:
|
||||
for nid, e in resp['node_errors'].items():
|
||||
print(f" Node {nid}: {e}")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
print(f" Prompt ID: {resp.get('prompt_id')}")
|
||||
except:
|
||||
print(f" Response: {out.strip()[:500]}")
|
||||
|
||||
# Monitor
|
||||
print("\n Monitoring GPU generation + CPU VAE decode...")
|
||||
last_log = ""
|
||||
for i in range(120):
|
||||
time.sleep(15)
|
||||
|
||||
stats = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
||||
" MEM=$(ps -p $PID -o rss --no-headers); "
|
||||
" GPU_TEMP=$(cat /sys/class/drm/card1/device/hwmon/hwmon*/temp1_input 2>/dev/null || echo 0); "
|
||||
" GPU_POWER=$(cat /sys/class/drm/card1/device/hwmon/hwmon*/power1_average 2>/dev/null || echo 0); "
|
||||
" echo \"CPU:${CPU}% RSS:$((MEM/1024))MB GPU_T:$((GPU_TEMP/1000))C GPU_P:$((GPU_POWER/1000000))W\"; "
|
||||
"else echo DEAD; fi'")[1].strip()
|
||||
|
||||
log = run("tail -10 /home/fabian/comfyui.log 2>/dev/null")[1].strip()
|
||||
|
||||
elapsed = (i+1)*15
|
||||
m, s = divmod(elapsed, 60)
|
||||
|
||||
print(f" [{m}m{s:02d}s] {stats}")
|
||||
|
||||
# Show last meaningful log line if changed
|
||||
if log != last_log:
|
||||
for line in reversed(log.split('\n')):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION'):
|
||||
print(f" LOG: {l[:120]}")
|
||||
break
|
||||
last_log = log
|
||||
|
||||
if 'DEAD' in stats:
|
||||
print("\n PROCESS DIED!")
|
||||
run("tail -60 /home/fabian/comfyui.log", desc="Death log")
|
||||
break
|
||||
|
||||
if 'Prompt executed in' in log:
|
||||
print(f"\n IMAGE GENERATED!")
|
||||
run("tail -30 /home/fabian/comfyui.log", desc="Success log")
|
||||
break
|
||||
|
||||
if 'Traceback' in log or 'CUDA out of memory' in log:
|
||||
print("\n ERROR!")
|
||||
run("tail -60 /home/fabian/comfyui.log", desc="Error log")
|
||||
break
|
||||
|
||||
# Output
|
||||
run("ls -lah /home/fabian/ComfyUI/output/", desc="Output files")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Save a proper Z-Image-Turbo GGUF workflow as the default ComfyUI web UI workflow."""
|
||||
import paramiko, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
|
||||
# ComfyUI web UI workflow format (not API format)
|
||||
workflow = {
|
||||
"last_node_id": 8,
|
||||
"last_link_id": 8,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "UnetLoaderGGUF",
|
||||
"pos": [100, 100],
|
||||
"size": [300, 80],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "MODEL", "type": "MODEL", "links": [1], "slot_index": 0}],
|
||||
"properties": {"Node name for S&R": "UnetLoaderGGUF"},
|
||||
"widgets_values": ["z_image_turbo-Q5_K_S.gguf"]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPLoaderGGUF",
|
||||
"pos": [100, 250],
|
||||
"size": [300, 80],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "CLIP", "type": "CLIP", "links": [2], "slot_index": 0}],
|
||||
"properties": {"Node name for S&R": "CLIPLoaderGGUF"},
|
||||
"widgets_values": ["Qwen3-4B.i1-Q5_K_S.gguf", "qwen_image"]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "VAELoader",
|
||||
"pos": [100, 400],
|
||||
"size": [300, 60],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "VAE", "type": "VAE", "links": [3], "slot_index": 0}],
|
||||
"properties": {"Node name for S&R": "VAELoader"},
|
||||
"widgets_values": ["ae.safetensors"]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "CLIPTextEncode",
|
||||
"pos": [500, 250],
|
||||
"size": [400, 120],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [{"name": "clip", "type": "CLIP", "link": 2}],
|
||||
"outputs": [{"name": "CONDITIONING", "type": "CONDITIONING", "links": [4], "slot_index": 0}],
|
||||
"properties": {"Node name for S&R": "CLIPTextEncode"},
|
||||
"widgets_values": ["A red fox in a snowy forest, photorealistic, highly detailed"]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "EmptyLatentImage",
|
||||
"pos": [500, 450],
|
||||
"size": [300, 110],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "LATENT", "type": "LATENT", "links": [5], "slot_index": 0}],
|
||||
"properties": {"Node name for S&R": "EmptyLatentImage"},
|
||||
"widgets_values": [512, 512, 1]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "KSampler",
|
||||
"pos": [950, 100],
|
||||
"size": [320, 474],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"name": "model", "type": "MODEL", "link": 1},
|
||||
{"name": "positive", "type": "CONDITIONING", "link": 4},
|
||||
{"name": "negative", "type": "CONDITIONING", "link": None},
|
||||
{"name": "latent_image", "type": "LATENT", "link": 5}
|
||||
],
|
||||
"outputs": [{"name": "LATENT", "type": "LATENT", "links": [6], "slot_index": 0}],
|
||||
"properties": {"Node name for S&R": "KSampler"},
|
||||
"widgets_values": [42, "fixed", 8, 1.0, "euler", "simple", 1.0]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "VAEDecode",
|
||||
"pos": [1350, 100],
|
||||
"size": [210, 50],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"name": "samples", "type": "LATENT", "link": 6},
|
||||
{"name": "vae", "type": "VAE", "link": 3}
|
||||
],
|
||||
"outputs": [{"name": "IMAGE", "type": "IMAGE", "links": [7], "slot_index": 0}],
|
||||
"properties": {"Node name for S&R": "VAEDecode"}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "SaveImage",
|
||||
"pos": [1350, 250],
|
||||
"size": [320, 270],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [{"name": "images", "type": "IMAGE", "link": 7}],
|
||||
"properties": {"Node name for S&R": "SaveImage"},
|
||||
"widgets_values": ["ZImageTurbo"]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 0, 6, 0, "MODEL"],
|
||||
[2, 2, 0, 4, 0, "CLIP"],
|
||||
[3, 3, 0, 7, 1, "VAE"],
|
||||
[4, 4, 0, 6, 1, "CONDITIONING"],
|
||||
[5, 5, 0, 6, 3, "LATENT"],
|
||||
[6, 6, 0, 7, 0, "LATENT"],
|
||||
[7, 7, 0, 8, 0, "IMAGE"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
|
||||
sftp = c.open_sftp()
|
||||
|
||||
# Save as default workflow
|
||||
def ensure_dir(sftp, path):
|
||||
try:
|
||||
sftp.stat(path)
|
||||
except FileNotFoundError:
|
||||
ensure_dir(sftp, '/'.join(path.split('/')[:-1]))
|
||||
sftp.mkdir(path)
|
||||
|
||||
ensure_dir(sftp, '/home/fabian/ComfyUI/user/default/comfyui')
|
||||
|
||||
wf_json = json.dumps(workflow, indent=2)
|
||||
|
||||
# Save as default workflow
|
||||
with sftp.open('/home/fabian/ComfyUI/user/default/comfyui/workflow.json', 'w') as f:
|
||||
f.write(wf_json)
|
||||
print("Saved default workflow: ~/ComfyUI/user/default/comfyui/workflow.json")
|
||||
|
||||
# Also save a loadable copy in the ComfyUI root
|
||||
with sftp.open('/home/fabian/ComfyUI/z_image_turbo_workflow.json', 'w') as f:
|
||||
f.write(wf_json)
|
||||
print("Saved loadable copy: ~/ComfyUI/z_image_turbo_workflow.json")
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone. Refresh ComfyUI web UI — it will load the Z-Image-Turbo GGUF workflow by default.")
|
||||
print("If it still shows the old workflow, click the menu and Load the z_image_turbo_workflow.json file.")
|
||||
@@ -0,0 +1,291 @@
|
||||
"""BC-250: Start ComfyUI on GPU and generate an image. Single SSH connection. No shell escaping issues."""
|
||||
import paramiko
|
||||
import time
|
||||
import json
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
# ==== CONFIG ====
|
||||
SSH_HOST = '192.168.178.150'
|
||||
SSH_USER = 'fabian'
|
||||
SSH_KEY = r'C:\Users\fabia\.ssh\id_ed25519'
|
||||
|
||||
def connect():
|
||||
k = paramiko.Ed25519Key.from_private_key_file(SSH_KEY)
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(SSH_HOST, username=SSH_USER, pkey=k, timeout=15)
|
||||
return c
|
||||
|
||||
def sh(c, cmd, timeout=60):
|
||||
"""Run a bash command. All commands go through bash explicitly."""
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command(f'/bin/bash -l -c {_quote(cmd)}')
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk:
|
||||
break
|
||||
out += chunk
|
||||
except Exception:
|
||||
break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
def _quote(s):
|
||||
"""Shell-quote a string using single quotes."""
|
||||
return "'" + s.replace("'", "'\\''") + "'"
|
||||
|
||||
def write_remote_file(c, path, content):
|
||||
"""Write a file on the remote via SFTP. No shell escaping needed."""
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open(path, 'w') as f:
|
||||
f.write(content)
|
||||
sftp.close()
|
||||
|
||||
# ================================================================
|
||||
print("="*60)
|
||||
print("STEP 1: Connect + kill old ComfyUI")
|
||||
print("="*60)
|
||||
c = connect()
|
||||
sh(c, 'pkill -9 -f "python3.*main.py" 2>/dev/null || true')
|
||||
time.sleep(2)
|
||||
alive = sh(c, 'pgrep -af "python3.*main.py" 2>/dev/null || echo NONE')
|
||||
print(f" Old processes: {alive}")
|
||||
|
||||
# ================================================================
|
||||
print("\n" + "="*60)
|
||||
print("STEP 2: Write launcher script on BC-250")
|
||||
print("="*60)
|
||||
|
||||
# Write a bash launcher script directly via SFTP - avoids ALL shell escaping issues
|
||||
launcher = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
# GPU environment
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
# Threading
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
# MIOpen
|
||||
export MIOPEN_FIND_MODE=1
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--lowvram \\
|
||||
--force-fp16 \\
|
||||
--cpu-vae \\
|
||||
--disable-smart-memory
|
||||
""")
|
||||
|
||||
write_remote_file(c, '/tmp/run_comfyui.sh', launcher)
|
||||
sh(c, 'chmod +x /tmp/run_comfyui.sh')
|
||||
print(" Launcher script written to /tmp/run_comfyui.sh")
|
||||
print(" Flags: --lowvram --force-fp16 --cpu-vae --disable-smart-memory")
|
||||
|
||||
# ================================================================
|
||||
print("\n" + "="*60)
|
||||
print("STEP 3: Verify GPU works with PyTorch")
|
||||
print("="*60)
|
||||
|
||||
gpu_script = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
source ~/comfyui-env/bin/activate
|
||||
python3 -c "
|
||||
import torch
|
||||
print('PyTorch:', torch.__version__)
|
||||
print('CUDA/ROCm available:', torch.cuda.is_available())
|
||||
if torch.cuda.is_available():
|
||||
print('Device:', torch.cuda.get_device_name(0))
|
||||
f,t = torch.cuda.mem_get_info(0)
|
||||
print(f'VRAM: {f//1048576}MB free / {t//1048576}MB total')
|
||||
x = torch.randn(512,512,device='cuda',dtype=torch.float16)
|
||||
y = x @ x
|
||||
print('GPU compute test: PASS')
|
||||
else:
|
||||
print('FATAL: NO GPU')
|
||||
exit(1)
|
||||
"
|
||||
""")
|
||||
write_remote_file(c, '/tmp/gpu_test.sh', gpu_script)
|
||||
sh(c, 'chmod +x /tmp/gpu_test.sh')
|
||||
out = sh(c, '/tmp/gpu_test.sh', timeout=30)
|
||||
print(f" {out}")
|
||||
if 'FATAL' in out or 'False' in out:
|
||||
print(" *** GPU not working! Aborting. ***")
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
print(" GPU OK!")
|
||||
|
||||
# ================================================================
|
||||
print("\n" + "="*60)
|
||||
print("STEP 4: Start ComfyUI")
|
||||
print("="*60)
|
||||
|
||||
sh(c, 'rm -f /tmp/comfyui.log; touch /tmp/comfyui.log')
|
||||
sh(c, 'nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
|
||||
pid = sh(c, 'pgrep -f "python3.*main.py" 2>/dev/null || echo DEAD')
|
||||
if pid == 'DEAD':
|
||||
print(" FAILED to start! Log:")
|
||||
print(sh(c, 'cat /tmp/comfyui.log'))
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
print(f" PID: {pid}")
|
||||
|
||||
# Wait for HTTP 200
|
||||
print(" Waiting for HTTP ready...", end='', flush=True)
|
||||
for i in range(90):
|
||||
code = sh(c, 'curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null || echo 000', timeout=5)
|
||||
if '200' in code:
|
||||
print(f" READY ({i*2}s)")
|
||||
break
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
else:
|
||||
print(f"\n TIMEOUT! Last log:")
|
||||
print(sh(c, 'tail -20 /tmp/comfyui.log'))
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
|
||||
# Show startup flags from log
|
||||
log_head = sh(c, 'head -10 /tmp/comfyui.log')
|
||||
print(f"\n Startup log:\n {log_head[:300]}")
|
||||
|
||||
# ================================================================
|
||||
print("\n" + "="*60)
|
||||
print("STEP 5: Submit workflow")
|
||||
print("="*60)
|
||||
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["6", 0], "vae": ["3", 0]}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
write_remote_file(c, '/tmp/wf.json', json.dumps(workflow))
|
||||
# Verify it's valid JSON with correct nodes
|
||||
verify = sh(c, 'python3 -c "import json; d=json.load(open(\'/tmp/wf.json\')); p=d[\'prompt\']; print(len(p), \'nodes:\', sorted(p.keys()))"')
|
||||
print(f" Workflow: {verify}")
|
||||
|
||||
# Submit
|
||||
resp = sh(c, 'curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json', timeout=10)
|
||||
print(f" Response: {resp[:200]}")
|
||||
|
||||
if 'prompt_id' not in resp:
|
||||
print(" *** SUBMIT FAILED! ***")
|
||||
print(f" Full response: {resp}")
|
||||
print(f" Log: {sh(c, 'tail -10 /tmp/comfyui.log')}")
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
|
||||
prompt_id = json.loads(resp).get('prompt_id', '?')
|
||||
print(f" Prompt ID: {prompt_id}")
|
||||
|
||||
# ================================================================
|
||||
print("\n" + "="*60)
|
||||
print("STEP 6: Monitor generation (checking GPU usage)")
|
||||
print("="*60)
|
||||
|
||||
t0 = time.time()
|
||||
for i in range(200): # up to ~50 min
|
||||
elapsed = int(time.time() - t0)
|
||||
|
||||
gpu_pct = sh(c, 'cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null || echo ?', timeout=5)
|
||||
gpu_temp = sh(c, 'cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null || echo 0', timeout=5)
|
||||
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
|
||||
|
||||
log_tail = sh(c, 'tail -3 /tmp/comfyui.log 2>/dev/null', timeout=5)
|
||||
last_line = log_tail.strip().split('\n')[-1] if log_tail else ''
|
||||
|
||||
# Check for output image
|
||||
imgs = sh(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null || echo NONE', timeout=5)
|
||||
|
||||
print(f" [{elapsed:>4}s] GPU:{gpu_pct:>3}% {temp_c}C | {last_line[-90:]}")
|
||||
|
||||
if imgs != 'NONE':
|
||||
print(f"\n >>> IMAGE GENERATED! <<<")
|
||||
print(f" Files: {imgs}")
|
||||
print(f" Time: {elapsed}s")
|
||||
final = sh(c, 'tail -20 /tmp/comfyui.log')
|
||||
print(f"\n Final log:\n{final}")
|
||||
break
|
||||
|
||||
# Check queue empty (= done or error)
|
||||
q = sh(c, 'curl -s http://127.0.0.1:8188/queue 2>/dev/null || echo {}', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 20:
|
||||
time.sleep(3)
|
||||
imgs2 = sh(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null || echo NONE', timeout=5)
|
||||
if imgs2 != 'NONE':
|
||||
print(f"\n >>> IMAGE GENERATED! <<<")
|
||||
print(f" Files: {imgs2}")
|
||||
print(f" Time: {elapsed}s")
|
||||
else:
|
||||
print(f"\n Queue empty, no image. Checking log for errors...")
|
||||
print(sh(c, 'tail -30 /tmp/comfyui.log'))
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Check process still alive
|
||||
alive = sh(c, 'pgrep -f "python3.*main.py" >/dev/null 2>&1 && echo YES || echo NO', timeout=5)
|
||||
if alive == 'NO':
|
||||
print(f"\n *** ComfyUI CRASHED! ***")
|
||||
print(sh(c, 'tail -40 /tmp/comfyui.log'))
|
||||
break
|
||||
|
||||
time.sleep(15)
|
||||
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Just start ComfyUI and submit workflow. All patches applied."""
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, t=30):
|
||||
ch = c.get_transport().open_session()
|
||||
ch.settimeout(t)
|
||||
ch.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
o = b""
|
||||
while True:
|
||||
try:
|
||||
d = ch.recv(65536)
|
||||
if not d: break
|
||||
o += d
|
||||
except: break
|
||||
ch.close()
|
||||
return o.decode(errors='replace').strip()
|
||||
|
||||
# Kill any leftover, clean logs
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 1')
|
||||
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
|
||||
|
||||
# Start
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(5)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
print(f"Started PID: {pid}")
|
||||
|
||||
# Wait for HTTP
|
||||
for i in range(60):
|
||||
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', t=5)
|
||||
if '200' in r:
|
||||
print(f"HTTP ready ({i*2}s)")
|
||||
break
|
||||
time.sleep(2)
|
||||
|
||||
# Confirm SHARED
|
||||
log = ''
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: pass
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if any(x in s for x in ['vram state', 'SHARED', 'Device:']): print(f" {s}")
|
||||
|
||||
# Submit
|
||||
wf = {"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(wf))
|
||||
print("Submitting...")
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f" {resp[:120]}")
|
||||
|
||||
# Monitor
|
||||
t0 = time.time()
|
||||
shown = set()
|
||||
for i in range(150):
|
||||
el = int(time.time() - t0)
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if s and s not in shown and any(x in s for x in ['/8', 'loaded', 'load device', 'offload device',
|
||||
'Requested', 'VAE', 'Prompt executed', 'Error', 'OOM', 'CUDA']):
|
||||
if 'FETCH' not in s and 'audio_vae' not in s and 'split attention' not in s:
|
||||
gpu = sh('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null', t=3)
|
||||
print(f" [{el:>3}s] GPU:{gpu}% {s[-110:]}")
|
||||
shown.add(s)
|
||||
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', t=5)
|
||||
if imgs:
|
||||
print(f"\n*** DONE in {el}s! ***")
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if 'Prompt executed' in s: print(f" {s}")
|
||||
break
|
||||
|
||||
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', t=5) == 'N':
|
||||
print(f"\nCRASHED at {el}s!")
|
||||
for l in log.split('\n')[-15:]:
|
||||
if l.strip(): print(f" {l.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(4)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
@@ -0,0 +1,358 @@
|
||||
"""BC-250 Full GPU Fix: Verify ROCm, diagnose VRAM, start ComfyUI on GPU, generate image."""
|
||||
import paramiko
|
||||
import time
|
||||
import json
|
||||
import sys
|
||||
|
||||
SSH_HOST = '192.168.178.150'
|
||||
SSH_USER = 'fabian'
|
||||
SSH_KEY = r'C:\Users\fabia\.ssh\id_ed25519'
|
||||
|
||||
def ssh_connect():
|
||||
k = paramiko.Ed25519Key.from_private_key_file(SSH_KEY)
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(SSH_HOST, username=SSH_USER, pkey=k, timeout=15)
|
||||
return c
|
||||
|
||||
def run(c, cmd, timeout=30):
|
||||
"""Run command via fish shell, return stdout."""
|
||||
wrapped = f'bash -c {repr(cmd)}'
|
||||
_, o, e = c.exec_command(wrapped, timeout=timeout)
|
||||
return o.read().decode(errors='replace').strip()
|
||||
|
||||
def run_full(c, cmd, timeout=30):
|
||||
"""Run command, return (stdout, stderr)."""
|
||||
wrapped = f'bash -c {repr(cmd)}'
|
||||
_, o, e = c.exec_command(wrapped, timeout=timeout)
|
||||
return o.read().decode(errors='replace').strip(), e.read().decode(errors='replace').strip()
|
||||
|
||||
# ============================================================
|
||||
# PHASE 1: Kill any remnants
|
||||
# ============================================================
|
||||
print("="*60)
|
||||
print("PHASE 1: Clean slate")
|
||||
print("="*60)
|
||||
c = ssh_connect()
|
||||
run(c, 'pkill -f "python.*main.py" 2>/dev/null; pkill -f comfyui 2>/dev/null')
|
||||
time.sleep(2)
|
||||
leftover = run(c, 'pgrep -af "python.*main.py" 2>/dev/null')
|
||||
if leftover:
|
||||
print(f"WARNING: Still running: {leftover}")
|
||||
run(c, 'pkill -9 -f "python.*main.py" 2>/dev/null')
|
||||
time.sleep(1)
|
||||
print("ComfyUI killed. Clean slate.")
|
||||
|
||||
# ============================================================
|
||||
# PHASE 2: Verify ROCm + PyTorch GPU
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print("PHASE 2: Verify ROCm + PyTorch GPU access")
|
||||
print("="*60)
|
||||
|
||||
# Set GPU env vars for ALL subsequent commands
|
||||
GPU_ENV = (
|
||||
'export HSA_OVERRIDE_GFX_VERSION=10.1.0; '
|
||||
'export HIP_VISIBLE_DEVICES=0; '
|
||||
'export HSA_ENABLE_SDMA=0; '
|
||||
'export HSA_TOOLS_LIB=""; '
|
||||
'export HSA_TOOLS_REPORT_LOAD_FAILURE=0; '
|
||||
'export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False; '
|
||||
)
|
||||
|
||||
# Check rocminfo
|
||||
out = run(c, f'{GPU_ENV} rocminfo 2>&1 | grep -E "Name:|Marketing Name:|gfx" | head -10')
|
||||
print(f"ROCm devices:\n{out}")
|
||||
|
||||
# Check PyTorch GPU
|
||||
gpu_test = f'''{GPU_ENV} cd ~/ComfyUI && source ~/comfyui-env/bin/activate.fish 2>/dev/null; . ~/comfyui-env/bin/activate 2>/dev/null; python3 -c "
|
||||
import torch
|
||||
print(f'PyTorch: {{torch.__version__}}')
|
||||
print(f'CUDA available: {{torch.cuda.is_available()}}')
|
||||
print(f'Device count: {{torch.cuda.device_count()}}')
|
||||
if torch.cuda.is_available():
|
||||
print(f'Device name: {{torch.cuda.get_device_name(0)}}')
|
||||
free, total = torch.cuda.mem_get_info(0)
|
||||
print(f'VRAM: {{free//1024//1024}}MB free / {{total//1024//1024}}MB total')
|
||||
# Quick GPU compute test
|
||||
x = torch.randn(1024, 1024, device='cuda', dtype=torch.float16)
|
||||
y = torch.mm(x, x)
|
||||
print(f'GPU compute test: OK (result sum={{y.sum().item():.1f}})')
|
||||
del x, y
|
||||
torch.cuda.empty_cache()
|
||||
else:
|
||||
print('ERROR: GPU NOT AVAILABLE')
|
||||
import sys; sys.exit(1)
|
||||
"'''
|
||||
out, err = run_full(c, gpu_test, timeout=60)
|
||||
print(out)
|
||||
if err:
|
||||
print(f"STDERR: {err}")
|
||||
if 'ERROR: GPU NOT AVAILABLE' in out or 'CUDA available: False' in out:
|
||||
print("\n*** FATAL: PyTorch cannot see the GPU! ***")
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
print("\nGPU verified OK!")
|
||||
|
||||
# ============================================================
|
||||
# PHASE 3: Start ComfyUI with correct GPU flags
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print("PHASE 3: Start ComfyUI with GPU")
|
||||
print("="*60)
|
||||
|
||||
# The key insight: --novram was offloading EVERYTHING to CPU (0 MB on GPU)
|
||||
# For this APU with shared memory, --lowvram is better:
|
||||
# it keeps compute on GPU but swaps model layers in/out
|
||||
# We also use --force-fp16 to reduce memory pressure
|
||||
# --cpu-vae to avoid the known VAE decode hang on this GPU
|
||||
|
||||
COMFYUI_CMD = (
|
||||
f'{GPU_ENV} '
|
||||
'export OMP_NUM_THREADS=12; '
|
||||
'export MKL_NUM_THREADS=12; '
|
||||
'export OPENBLAS_NUM_THREADS=12; '
|
||||
'export MIOPEN_FIND_MODE=1; ' # Fast MIOpen kernel search
|
||||
'cd ~/ComfyUI && '
|
||||
'source ~/comfyui-env/bin/activate 2>/dev/null; . ~/comfyui-env/bin/activate 2>/dev/null; '
|
||||
'nohup python3 main.py '
|
||||
'--listen 0.0.0.0 --port 8188 '
|
||||
'--lowvram '
|
||||
'--force-fp16 '
|
||||
'--cpu-vae '
|
||||
'--disable-smart-memory '
|
||||
'> /tmp/comfyui.log 2>&1 &'
|
||||
)
|
||||
|
||||
# Truncate old log first
|
||||
run(c, 'truncate -s 0 /tmp/comfyui.log 2>/dev/null; touch /tmp/comfyui.log')
|
||||
print("Starting ComfyUI with: --lowvram --force-fp16 --cpu-vae --disable-smart-memory")
|
||||
print("(--lowvram keeps compute on GPU, swaps layers; --novram was wrong - it put everything on CPU)")
|
||||
run(c, COMFYUI_CMD)
|
||||
time.sleep(3)
|
||||
|
||||
# Verify it started
|
||||
pid = run(c, 'pgrep -f "python.*main.py" 2>/dev/null')
|
||||
if not pid:
|
||||
print("ERROR: ComfyUI failed to start!")
|
||||
log = run(c, 'cat /tmp/comfyui.log')
|
||||
print(f"Log:\n{log}")
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
print(f"ComfyUI started, PID: {pid}")
|
||||
|
||||
# Wait for server ready
|
||||
print("Waiting for server ready...")
|
||||
for i in range(60):
|
||||
try:
|
||||
resp = run(c, 'curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if resp == '200':
|
||||
print(f"Server ready after {i*3}s!")
|
||||
break
|
||||
except:
|
||||
pass
|
||||
# Also check for crash
|
||||
log_tail = run(c, 'tail -3 /tmp/comfyui.log 2>/dev/null')
|
||||
if 'Traceback' in log_tail or 'Error' in log_tail:
|
||||
print(f"Server log issue: {log_tail}")
|
||||
if i % 5 == 0 and i > 0:
|
||||
print(f" [{i*3}s] Still waiting... log: {log_tail[-80:]}")
|
||||
time.sleep(3)
|
||||
else:
|
||||
print("TIMEOUT waiting for ComfyUI!")
|
||||
log = run(c, 'tail -30 /tmp/comfyui.log')
|
||||
print(f"Log:\n{log}")
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
|
||||
# Print startup log to confirm flags
|
||||
log = run(c, 'head -20 /tmp/comfyui.log')
|
||||
print(f"\nStartup log:\n{log}")
|
||||
|
||||
# ============================================================
|
||||
# PHASE 4: Submit workflow via SFTP
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print("PHASE 4: Submit workflow")
|
||||
print("="*60)
|
||||
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {
|
||||
"unet_name": "z_image_turbo-Q5_K_S.gguf"
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {
|
||||
"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf",
|
||||
"type": "qwen_image"
|
||||
}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {
|
||||
"vae_name": "ae.safetensors"
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A red fox in a snowy forest, photorealistic, highly detailed",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"batch_size": 1
|
||||
}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"positive": ["4", 0],
|
||||
"negative": ["4", 0],
|
||||
"latent_image": ["5", 0],
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {
|
||||
"samples": ["6", 0],
|
||||
"vae": ["3", 0]
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {
|
||||
"images": ["7", 0],
|
||||
"filename_prefix": "ZImageTurbo_GPU"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Write via SFTP
|
||||
sftp = c.open_sftp()
|
||||
wf_json = json.dumps(workflow)
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(wf_json)
|
||||
sftp.close()
|
||||
print("Workflow written to /tmp/wf.json via SFTP")
|
||||
|
||||
# Verify JSON
|
||||
verify = run(c, 'python3 -c "import json; d=json.load(open(\'/tmp/wf.json\')); print(f\'Nodes: {list(d[chr(34)+chr(34) if False else \"prompt\"].keys())}\')"')
|
||||
print(f"Verify: {verify}")
|
||||
|
||||
# Submit
|
||||
resp = run(c, 'curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json 2>/dev/null')
|
||||
print(f"Submit response: {resp}")
|
||||
|
||||
if 'error' in resp.lower() and 'prompt_id' not in resp.lower():
|
||||
print(f"\n*** SUBMISSION ERROR ***")
|
||||
# Check what went wrong
|
||||
log = run(c, 'tail -10 /tmp/comfyui.log')
|
||||
print(f"Log: {log}")
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
resp_data = json.loads(resp)
|
||||
prompt_id = resp_data.get('prompt_id', 'unknown')
|
||||
print(f"Prompt ID: {prompt_id}")
|
||||
except:
|
||||
print("Could not parse response, continuing anyway...")
|
||||
|
||||
# ============================================================
|
||||
# PHASE 5: Monitor generation with GPU tracking
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print("PHASE 5: Monitor generation (GPU must be active!)")
|
||||
print("="*60)
|
||||
|
||||
start_time = time.time()
|
||||
last_log_len = 0
|
||||
|
||||
for i in range(120): # Up to 30 minutes
|
||||
elapsed = int(time.time() - start_time)
|
||||
|
||||
# GPU metrics
|
||||
gpu_pct = run(c, 'cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null')
|
||||
gpu_temp = run(c, 'cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null')
|
||||
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
|
||||
|
||||
# GPU power
|
||||
gpu_power = run(c, f'{GPU_ENV} rocm-smi -P 2>&1 | grep "Graphics Package" | grep -oP "[\\d.]+" | head -1')
|
||||
|
||||
# Process info
|
||||
proc = run(c, 'ps -p $(pgrep -f "python.*main.py" | head -1) -o %cpu,%mem,rss --no-headers 2>/dev/null')
|
||||
|
||||
# Log tail
|
||||
log = run(c, 'tail -5 /tmp/comfyui.log 2>/dev/null')
|
||||
last_line = log.split('\n')[-1] if log else ''
|
||||
|
||||
# Output files
|
||||
files = run(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null')
|
||||
|
||||
# Queue
|
||||
queue = run(c, 'curl -s http://127.0.0.1:8188/queue 2>/dev/null')
|
||||
|
||||
status = f"[{elapsed:>4}s] GPU:{gpu_pct:>3}% {temp_c}C {gpu_power}W | proc:{proc} | {last_line[-100:]}"
|
||||
print(status)
|
||||
|
||||
# SUCCESS: Image generated!
|
||||
if files:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"*** SUCCESS! IMAGE GENERATED! ***")
|
||||
print(f"Files: {files}")
|
||||
print(f"Total time: {elapsed}s")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Print final log
|
||||
final_log = run(c, 'tail -20 /tmp/comfyui.log 2>/dev/null')
|
||||
print(f"\nFinal log:\n{final_log}")
|
||||
break
|
||||
|
||||
# Check if queue is empty (job done or failed)
|
||||
try:
|
||||
qdata = json.loads(queue)
|
||||
running = len(qdata.get('queue_running', []))
|
||||
pending = len(qdata.get('queue_pending', []))
|
||||
if running == 0 and pending == 0 and elapsed > 30:
|
||||
print(f"\nQueue empty after {elapsed}s. Checking if image was saved...")
|
||||
time.sleep(2)
|
||||
files = run(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null')
|
||||
if files:
|
||||
print(f"*** SUCCESS! {files}")
|
||||
else:
|
||||
print("No image. Checking log for errors:")
|
||||
err_log = run(c, 'tail -30 /tmp/comfyui.log 2>/dev/null')
|
||||
print(err_log)
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
# Check for process death
|
||||
alive = run(c, 'pgrep -f "python.*main.py" 2>/dev/null')
|
||||
if not alive:
|
||||
print("\n*** ComfyUI process died! ***")
|
||||
crash_log = run(c, 'tail -40 /tmp/comfyui.log 2>/dev/null')
|
||||
print(f"Crash log:\n{crash_log}")
|
||||
break
|
||||
|
||||
time.sleep(15)
|
||||
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix GPU inference: kill stuck, diagnose, restart with --novram, test."""
|
||||
import paramiko, json, time, textwrap
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=60, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}\n {desc}\n{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
combined = out.strip()
|
||||
if combined:
|
||||
lines = combined.split('\n')
|
||||
if len(lines) > 50:
|
||||
print(f" ... ({len(lines)} lines, showing last 50)")
|
||||
print('\n'.join(lines[-50:]))
|
||||
else:
|
||||
print(combined)
|
||||
if err.strip():
|
||||
for line in err.strip().split('\n')[-10:]:
|
||||
print(f" STDERR: {line}")
|
||||
return rc, out, err
|
||||
|
||||
# ============================================================
|
||||
# STEP 1: Kill stuck ComfyUI
|
||||
# ============================================================
|
||||
run("pkill -f 'python3 main.py' 2>/dev/null; sleep 2; "
|
||||
"pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1; "
|
||||
"echo 'Killed.'", desc="Kill stuck ComfyUI")
|
||||
|
||||
# ============================================================
|
||||
# STEP 2: Check dmesg for GPU errors
|
||||
# ============================================================
|
||||
run("dmesg | grep -i -E 'amdgpu|error|fault|gpu|kiq|gfx' | tail -30",
|
||||
desc="Check dmesg for GPU errors")
|
||||
|
||||
# ============================================================
|
||||
# STEP 3: Quick GPU sanity test
|
||||
# ============================================================
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
||||
"HSA_OVERRIDE_GFX_VERSION=10.1.0 HSA_ENABLE_SDMA=0 "
|
||||
"python3 -c \""
|
||||
"import torch; "
|
||||
"print(f\\\"CUDA available: {torch.cuda.is_available()}\\\"); "
|
||||
"print(f\\\"Device: {torch.cuda.get_device_name(0)}\\\"); "
|
||||
"a = torch.randn(1024, 1024, device=\\\"cuda\\\"); "
|
||||
"b = torch.randn(1024, 1024, device=\\\"cuda\\\"); "
|
||||
"c = a @ b; "
|
||||
"print(f\\\"Matmul result shape: {c.shape}, sum: {c.sum().item():.2f}\\\"); "
|
||||
"# Test fp16 "
|
||||
"a16 = a.half(); b16 = b.half(); c16 = a16 @ b16; "
|
||||
"print(f\\\"FP16 matmul OK: {c16.shape}\\\"); "
|
||||
"print(f\\\"Free VRAM: {torch.cuda.mem_get_info()[0]/1024**2:.0f} MB\\\"); "
|
||||
"print(f\\\"Total VRAM: {torch.cuda.mem_get_info()[1]/1024**2:.0f} MB\\\"); "
|
||||
"print(\\\"GPU SANITY: PASS\\\")\"'",
|
||||
desc="Quick GPU sanity test")
|
||||
|
||||
# ============================================================
|
||||
# STEP 4: Check what ComfyUI flags are available
|
||||
# ============================================================
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && "
|
||||
"python3 main.py --help 2>&1 | grep -E \"novram|lowvram|cpu|fp16|force|vram|disable-smart|channels\"'",
|
||||
desc="ComfyUI VRAM-related flags")
|
||||
|
||||
# ============================================================
|
||||
# STEP 5: Write new startup script with --novram
|
||||
# ============================================================
|
||||
startup_script = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
# BC-250 ComfyUI Launcher - GPU mode with aggressive offloading
|
||||
|
||||
# GPU identity
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
|
||||
# Disable SDMA (known issue on Cyan Skillfish)
|
||||
export HSA_ENABLE_SDMA=0
|
||||
|
||||
# Suppress HSA tool warnings
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# Threading: use all 12 cores for CPU-side work
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
|
||||
# HIP memory: allow expandable segments to reduce fragmentation
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:True
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
# --novram: most aggressive offloading - keeps almost nothing on GPU,
|
||||
# sends individual layers to GPU one at a time during forward pass.
|
||||
# This is needed because BC-250 has only ~7.6GB shared VRAM.
|
||||
# --disable-smart-memory: prevents ComfyUI from trying to be clever about memory
|
||||
# --force-fp16: force fp16 to halve VRAM usage
|
||||
exec python3 main.py --listen 0.0.0.0 --port 8188 --novram --force-fp16 --disable-smart-memory
|
||||
""")
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
||||
f.write(startup_script)
|
||||
sftp.close()
|
||||
run("chmod +x /home/fabian/start_comfyui.sh", desc="Make script executable")
|
||||
print("\n Startup script updated with --novram --force-fp16 --disable-smart-memory")
|
||||
|
||||
# ============================================================
|
||||
# STEP 6: Launch ComfyUI with new settings
|
||||
# ============================================================
|
||||
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'; sleep 1; echo 'Launched'",
|
||||
desc="Launch ComfyUI with --novram")
|
||||
|
||||
# Wait for server to be ready
|
||||
print("\n Waiting for server to start...")
|
||||
for i in range(30):
|
||||
time.sleep(3)
|
||||
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null || echo 0'")
|
||||
code = out.strip()
|
||||
if code == '200':
|
||||
print(f" Server ready after {(i+1)*3}s!")
|
||||
break
|
||||
# Check log for errors
|
||||
rc2, log, _ = run("tail -3 /home/fabian/comfyui.log 2>/dev/null")
|
||||
if 'Error' in log or 'error' in log.lower():
|
||||
print(f" Log: {log.strip()}")
|
||||
print(f" [{(i+1)*3}s] HTTP {code}...")
|
||||
else:
|
||||
print(" Server didn't start in 90s!")
|
||||
run("tail -40 /home/fabian/comfyui.log", desc="Startup log")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
|
||||
# Confirm server info
|
||||
run("tail -30 /home/fabian/comfyui.log", desc="Startup log")
|
||||
|
||||
# ============================================================
|
||||
# STEP 7: Submit test workflow (smaller 512x512 image first)
|
||||
# ============================================================
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A red fox in a snowy forest, photorealistic",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"seed": 12345,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": ["4", 0],
|
||||
"negative": ["5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sftp2 = ssh.open_sftp()
|
||||
with sftp2.open('/tmp/zimage_workflow.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
sftp2.close()
|
||||
|
||||
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H \"Content-Type: application/json\" "
|
||||
"-d @/tmp/zimage_workflow.json'",
|
||||
desc="Submit 512x512 test workflow")
|
||||
|
||||
prompt_id = None
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f"\n API ERROR: {resp['error']}")
|
||||
if 'node_errors' in resp:
|
||||
for nid, e in resp['node_errors'].items():
|
||||
print(f" Node {nid}: {e}")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
prompt_id = resp.get('prompt_id', 'unknown')
|
||||
print(f"\n Prompt ID: {prompt_id}")
|
||||
except:
|
||||
print(f" Raw response: {out.strip()[:500]}")
|
||||
|
||||
# ============================================================
|
||||
# STEP 8: Monitor generation
|
||||
# ============================================================
|
||||
print("\n Monitoring GPU generation...")
|
||||
last_log = ""
|
||||
for i in range(120): # up to 30 minutes
|
||||
time.sleep(15)
|
||||
|
||||
# CPU + GPU status
|
||||
rc, status, _ = run("bash -c '"
|
||||
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
||||
" MEM=$(ps -p $PID -o %mem --no-headers); "
|
||||
" THREADS=$(ps -p $PID -o nlwp --no-headers); "
|
||||
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
|
||||
" GPU_USE=$(cat /sys/class/drm/card1/device/gpu_busy_percent 2>/dev/null || echo N/A); "
|
||||
" VRAM_USED=$(cat /sys/class/drm/card1/device/mem_info_vram_used 2>/dev/null || echo 0); "
|
||||
" VRAM_TOTAL=$(cat /sys/class/drm/card1/device/mem_info_vram_total 2>/dev/null || echo 1); "
|
||||
" echo \"CPU:${CPU}% MEM:${MEM}% THR:${THREADS} LOAD:${LOAD} GPU:${GPU_USE}% VRAM:$((VRAM_USED/1048576))/$((VRAM_TOTAL/1048576))MB\"; "
|
||||
"else echo DEAD; fi'")
|
||||
|
||||
rc, log, _ = run("bash -c 'tail -8 /home/fabian/comfyui.log 2>/dev/null'")
|
||||
log_lines = log.strip()
|
||||
|
||||
# Show status
|
||||
status_line = status.strip()
|
||||
print(f" [{i+1}] {(i+1)*15}s | {status_line}")
|
||||
|
||||
# Show new log lines
|
||||
if log_lines != last_log:
|
||||
new_part = log_lines
|
||||
for line in new_part.split('\n')[-5:]:
|
||||
if line.strip():
|
||||
print(f" LOG: {line.strip()}")
|
||||
last_log = log_lines
|
||||
|
||||
if 'DEAD' in status_line:
|
||||
print("\n ComfyUI DIED!")
|
||||
run("tail -60 /home/fabian/comfyui.log", desc="Death log")
|
||||
break
|
||||
|
||||
if 'Prompt executed in' in log_lines:
|
||||
print(f"\n SUCCESS! Image generated at check {i+1} (~{(i+1)*15}s)")
|
||||
break
|
||||
|
||||
if 'Error' in log_lines or 'Traceback' in log_lines:
|
||||
print("\n ERROR detected!")
|
||||
run("tail -60 /home/fabian/comfyui.log", desc="Error log")
|
||||
break
|
||||
|
||||
# Final check
|
||||
run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null", desc="Output files")
|
||||
run("tail -25 /home/fabian/comfyui.log", desc="Final log")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Single SSH connection: kill stuck, check flags, update startup, launch, submit, monitor.
|
||||
Properly closes connection when done.
|
||||
"""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
import sys
|
||||
|
||||
KEY = r'C:\Users\fabia\.ssh\id_ed25519'
|
||||
HOST = '192.168.178.150'
|
||||
USER = 'fabian'
|
||||
|
||||
def connect():
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
for attempt in range(5):
|
||||
try:
|
||||
ssh.connect(HOST, username=USER, key_filename=KEY, timeout=10)
|
||||
return ssh
|
||||
except Exception as e:
|
||||
print(f" SSH attempt {attempt+1}/5 failed: {e}")
|
||||
time.sleep(10)
|
||||
print("FATAL: Cannot connect to BC-250")
|
||||
sys.exit(1)
|
||||
|
||||
def run(ssh, cmd, timeout=120):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
return out, err
|
||||
|
||||
def main():
|
||||
ssh = connect()
|
||||
print("Connected to BC-250.\n")
|
||||
|
||||
try:
|
||||
# ── 1. Kill stuck ComfyUI ──
|
||||
print("=== STEP 1: Kill stuck ComfyUI ===")
|
||||
out, _ = run(ssh, "pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; "
|
||||
"pgrep -f 'python3 main.py' || echo 'all_dead'")
|
||||
print(f" {out.strip()}")
|
||||
|
||||
# ── 2. Check available flags ──
|
||||
print("\n=== STEP 2: ComfyUI flags ===")
|
||||
out, err = run(ssh, "bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
||||
"cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'")
|
||||
combined = out + err
|
||||
for line in combined.split('\n'):
|
||||
low = line.lower()
|
||||
if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram',
|
||||
'memory', 'offload', 'precision', 'novram', 'lowvram']):
|
||||
print(f" {line.strip()}")
|
||||
|
||||
# ── 3. Write startup script ──
|
||||
print("\n=== STEP 3: Update startup script ===")
|
||||
script = r"""#!/bin/bash
|
||||
# BC-250 ComfyUI Launcher - GPU inference with CPU VAE decode
|
||||
|
||||
# GPU identity
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# Use all 12 CPU cores
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
# --novram: aggressive offload, one layer at a time to GPU
|
||||
# --force-fp16: halve VRAM for diffusion
|
||||
# --cpu-vae: VAE decode on CPU (GPU hangs on float32 VAE conv2d)
|
||||
# --disable-smart-memory: no memory heuristics
|
||||
exec python3 main.py \
|
||||
--listen 0.0.0.0 --port 8188 \
|
||||
--novram \
|
||||
--force-fp16 \
|
||||
--cpu-vae \
|
||||
--disable-smart-memory
|
||||
"""
|
||||
# Write via heredoc to avoid SFTP
|
||||
escaped = script.replace("'", "'\\''")
|
||||
out, _ = run(ssh, f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n"
|
||||
f"chmod +x /home/fabian/start_comfyui.sh && echo 'written'")
|
||||
print(f" {out.strip()}")
|
||||
|
||||
# ── 4. Launch ComfyUI ──
|
||||
print("\n=== STEP 4: Launch ComfyUI ===")
|
||||
run(ssh, "nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &")
|
||||
time.sleep(2)
|
||||
|
||||
print(" Waiting for server...")
|
||||
for i in range(40):
|
||||
time.sleep(3)
|
||||
out, _ = run(ssh, "curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0")
|
||||
code = out.strip()
|
||||
if code == '200':
|
||||
print(f" Server ready ({(i+1)*3}s)")
|
||||
break
|
||||
if i % 5 == 4:
|
||||
log, _ = run(ssh, "tail -2 /home/fabian/comfyui.log 2>/dev/null")
|
||||
last = [l.strip() for l in log.strip().split('\n') if l.strip()][-1:]
|
||||
print(f" [{(i+1)*3}s] HTTP {code} ... {last[0][:80] if last else ''}")
|
||||
else:
|
||||
print(" Server didn't start in 120s!")
|
||||
out, _ = run(ssh, "tail -40 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
return
|
||||
|
||||
# Show startup log
|
||||
out, _ = run(ssh, "tail -20 /home/fabian/comfyui.log 2>/dev/null")
|
||||
for line in out.strip().split('\n'):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH'):
|
||||
print(f" LOG: {l[:120]}")
|
||||
|
||||
# ── 5. Submit workflow ──
|
||||
print("\n=== STEP 5: Submit workflow (512x512, 8 steps) ===")
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}},
|
||||
"6": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"7": {"class_type": "KSampler",
|
||||
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple",
|
||||
"positive": ["4", 0], "negative": ["5", 0],
|
||||
"latent_image": ["6", 0], "denoise": 1.0}},
|
||||
"8": {"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
|
||||
"9": {"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
|
||||
}
|
||||
}
|
||||
|
||||
wf_json = json.dumps(workflow).replace("'", "'\\''")
|
||||
out, _ = run(ssh, f"curl -s -X POST http://localhost:8188/prompt "
|
||||
f"-H 'Content-Type: application/json' "
|
||||
f"-d '{wf_json}'")
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f" API ERROR: {resp['error']}")
|
||||
if 'node_errors' in resp:
|
||||
for nid, e in resp['node_errors'].items():
|
||||
print(f" Node {nid}: {e}")
|
||||
return
|
||||
print(f" Prompt ID: {resp.get('prompt_id')}")
|
||||
except:
|
||||
print(f" Response: {out.strip()[:500]}")
|
||||
|
||||
# ── 6. Monitor generation ──
|
||||
print("\n=== STEP 6: Monitoring generation ===")
|
||||
last_log = ""
|
||||
for i in range(120): # up to 30 minutes
|
||||
time.sleep(15)
|
||||
|
||||
stats, _ = run(ssh, "bash -c '"
|
||||
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
||||
" MEM=$(ps -p $PID -o rss --no-headers); "
|
||||
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
|
||||
" echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:${LOAD}\"; "
|
||||
"else echo DEAD; fi'")
|
||||
|
||||
log, _ = run(ssh, "tail -10 /home/fabian/comfyui.log 2>/dev/null")
|
||||
log_s = log.strip()
|
||||
|
||||
elapsed = (i+1) * 15
|
||||
m, s = divmod(elapsed, 60)
|
||||
print(f" [{m}m{s:02d}s] {stats.strip()}")
|
||||
|
||||
if log_s != last_log:
|
||||
for line in reversed(log_s.split('\n')):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
|
||||
print(f" LOG: {l[:120]}")
|
||||
break
|
||||
last_log = log_s
|
||||
|
||||
if 'DEAD' in stats:
|
||||
print("\n PROCESS DIED!")
|
||||
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
break
|
||||
|
||||
if 'Prompt executed in' in log_s:
|
||||
print(f"\n SUCCESS! Image generated!")
|
||||
out, _ = run(ssh, "tail -30 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
break
|
||||
|
||||
if 'Traceback' in log_s or 'RuntimeError' in log_s:
|
||||
print("\n ERROR detected!")
|
||||
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
|
||||
print(out)
|
||||
break
|
||||
|
||||
# ── 7. Check output ──
|
||||
print("\n=== Output files ===")
|
||||
out, _ = run(ssh, "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null")
|
||||
print(out.strip())
|
||||
|
||||
finally:
|
||||
ssh.close()
|
||||
print("\nSSH connection closed.")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fix: Add MIOpen fast-find, pre-warm GPU, restart ComfyUI, generate.
|
||||
All GPU ops confirmed working. The hang is likely cold MIOpen kernel cache.
|
||||
Single SSH connection.
|
||||
"""
|
||||
import paramiko, json, time, sys
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
for attempt in range(5):
|
||||
try:
|
||||
ssh.connect('192.168.178.150', username='fabian',
|
||||
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f" SSH attempt {attempt+1}/5: {e}")
|
||||
time.sleep(10)
|
||||
else:
|
||||
print("FATAL: Cannot connect"); sys.exit(1)
|
||||
|
||||
def run(cmd, timeout=300):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
return out, err
|
||||
|
||||
try:
|
||||
# 1. Kill any lingering ComfyUI
|
||||
print("=== Kill any ComfyUI ===")
|
||||
run("pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1")
|
||||
print(" Done")
|
||||
|
||||
# 2. Pre-warm MIOpen kernel cache with DiT-like operations
|
||||
print("\n=== Pre-warming MIOpen kernel cache ===")
|
||||
print(" This compiles HIP kernels that Z-Image-Turbo will need.")
|
||||
print(" First run after reboot is slow (kernel compilation)...")
|
||||
|
||||
warmup_code = r"""
|
||||
import torch, torch.nn as nn, time
|
||||
|
||||
# Simulate Z-Image-Turbo DiT operations
|
||||
device = 'cuda'
|
||||
dtype = torch.float16
|
||||
|
||||
print("Warming up HIP kernels for DiT inference...")
|
||||
t0 = time.time()
|
||||
|
||||
# 1. Linear layers (DiT blocks)
|
||||
print(" Linear layers...", end=" ", flush=True)
|
||||
for size in [(1024,1024), (4096,1024), (1024,4096)]:
|
||||
l = nn.Linear(*size).to(device, dtype)
|
||||
x = torch.randn(1, 64, size[0], device=device, dtype=dtype)
|
||||
y = l(x)
|
||||
del l, x, y
|
||||
torch.cuda.synchronize()
|
||||
print(f"{time.time()-t0:.1f}s")
|
||||
|
||||
# 2. Attention (SDPA - the core of DiT)
|
||||
print(" Scaled dot-product attention...", end=" ", flush=True)
|
||||
t1 = time.time()
|
||||
for heads in [8, 16, 24]:
|
||||
q = torch.randn(1, heads, 64, 64, device=device, dtype=dtype)
|
||||
k = torch.randn(1, heads, 64, 64, device=device, dtype=dtype)
|
||||
v = torch.randn(1, heads, 64, 64, device=device, dtype=dtype)
|
||||
y = torch.nn.functional.scaled_dot_product_attention(q, k, v)
|
||||
del q, k, v, y
|
||||
torch.cuda.synchronize()
|
||||
print(f"{time.time()-t1:.1f}s")
|
||||
|
||||
# 3. LayerNorm / RMSNorm
|
||||
print(" Normalization layers...", end=" ", flush=True)
|
||||
t1 = time.time()
|
||||
for dim in [1024, 2048, 4096]:
|
||||
ln = nn.LayerNorm(dim).to(device, dtype)
|
||||
x = torch.randn(1, 64, dim, device=device, dtype=dtype)
|
||||
y = ln(x)
|
||||
del ln, x, y
|
||||
torch.cuda.synchronize()
|
||||
print(f"{time.time()-t1:.1f}s")
|
||||
|
||||
# 4. Conv2d (VAE-like, but we'll run VAE on CPU)
|
||||
print(" Conv2d layers...", end=" ", flush=True)
|
||||
t1 = time.time()
|
||||
for ch in [64, 128, 256]:
|
||||
c = nn.Conv2d(ch, ch, 3, padding=1).to(device, dtype)
|
||||
x = torch.randn(1, ch, 32, 32, device=device, dtype=dtype)
|
||||
y = c(x)
|
||||
del c, x, y
|
||||
torch.cuda.synchronize()
|
||||
print(f"{time.time()-t1:.1f}s")
|
||||
|
||||
# 5. Full mini-DiT forward pass simulation
|
||||
print(" Mini-DiT forward pass simulation...", end=" ", flush=True)
|
||||
t1 = time.time()
|
||||
hidden = 1024
|
||||
seq_len = 256
|
||||
heads = 16
|
||||
head_dim = hidden // heads
|
||||
# Simulate a DiT block
|
||||
x = torch.randn(1, seq_len, hidden, device=device, dtype=dtype)
|
||||
norm = nn.LayerNorm(hidden).to(device, dtype)
|
||||
qkv = nn.Linear(hidden, hidden*3).to(device, dtype)
|
||||
proj = nn.Linear(hidden, hidden).to(device, dtype)
|
||||
ff1 = nn.Linear(hidden, hidden*4).to(device, dtype)
|
||||
ff2 = nn.Linear(hidden*4, hidden).to(device, dtype)
|
||||
for step in range(3):
|
||||
h = norm(x)
|
||||
q, k, v = qkv(h).chunk(3, dim=-1)
|
||||
q = q.view(1, seq_len, heads, head_dim).transpose(1,2)
|
||||
k = k.view(1, seq_len, heads, head_dim).transpose(1,2)
|
||||
v = v.view(1, seq_len, heads, head_dim).transpose(1,2)
|
||||
attn = torch.nn.functional.scaled_dot_product_attention(q, k, v)
|
||||
attn = attn.transpose(1,2).contiguous().view(1, seq_len, hidden)
|
||||
x = x + proj(attn)
|
||||
x = x + ff2(torch.nn.functional.gelu(ff1(norm(x))))
|
||||
torch.cuda.synchronize()
|
||||
print(f"{time.time()-t1:.1f}s")
|
||||
|
||||
total = time.time() - t0
|
||||
print(f"\nGPU kernel warmup complete in {total:.1f}s")
|
||||
print(f"VRAM used: {torch.cuda.memory_allocated()//1048576} MB")
|
||||
torch.cuda.empty_cache()
|
||||
print(f"VRAM after cleanup: {torch.cuda.memory_allocated()//1048576} MB")
|
||||
print("WARMUP_DONE")
|
||||
"""
|
||||
# Write warmup script
|
||||
run(f"cat > /tmp/gpu_warmup.py << 'PYEOF'\n{warmup_code}\nPYEOF")
|
||||
|
||||
out, err = run("bash -c 'source ~/comfyui-env/bin/activate && "
|
||||
"HSA_OVERRIDE_GFX_VERSION=10.1.0 HSA_ENABLE_SDMA=0 "
|
||||
"MIOPEN_FIND_MODE=3 MIOPEN_FIND_ENFORCE=3 "
|
||||
"python3 /tmp/gpu_warmup.py' 2>&1", timeout=300)
|
||||
print(out.strip())
|
||||
if 'WARMUP_DONE' not in out:
|
||||
print(f" WARNING: Warmup may have failed")
|
||||
print(f" STDERR: {err.strip()[:500]}")
|
||||
|
||||
# 3. Update startup script with MIOpen settings
|
||||
print("\n=== Update startup script ===")
|
||||
script = """#!/bin/bash
|
||||
# BC-250 ComfyUI Launcher — GPU (ROCm) + CPU VAE
|
||||
|
||||
# GPU identity (Cyan Skillfish gfx1013 -> gfx1010)
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# MIOpen: fast kernel selection (avoid long auto-tune on first run)
|
||||
export MIOPEN_FIND_MODE=3
|
||||
export MIOPEN_FIND_ENFORCE=3
|
||||
|
||||
# Use all 12 CPU cores for CPU-side work (dequant, text encoding)
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
# --novram: offload models to RAM, send layers to GPU one at a time
|
||||
# --force-fp16: fp16 diffusion to halve VRAM usage
|
||||
# --cpu-vae: VAE decode on CPU (GPU hangs on full VAE forward pass)
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--novram \\
|
||||
--force-fp16 \\
|
||||
--cpu-vae
|
||||
"""
|
||||
run(f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n"
|
||||
f"chmod +x /home/fabian/start_comfyui.sh")
|
||||
print(" Written with MIOpen fast-find + --novram --force-fp16 --cpu-vae")
|
||||
|
||||
# 4. Launch ComfyUI
|
||||
print("\n=== Launch ComfyUI ===")
|
||||
run("nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &")
|
||||
time.sleep(2)
|
||||
|
||||
print(" Waiting for server...")
|
||||
for i in range(50):
|
||||
time.sleep(3)
|
||||
out, _ = run("curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0")
|
||||
if out.strip() == '200':
|
||||
print(f" Server ready ({(i+1)*3}s)")
|
||||
break
|
||||
if i % 5 == 4:
|
||||
log, _ = run("tail -2 /home/fabian/comfyui.log 2>/dev/null")
|
||||
last = [l.strip() for l in log.strip().split('\n') if l.strip()]
|
||||
print(f" [{(i+1)*3}s] ... {last[-1][:80] if last else ''}")
|
||||
else:
|
||||
print(" Timeout!")
|
||||
out, _ = run("tail -40 /home/fabian/comfyui.log")
|
||||
print(out)
|
||||
sys.exit(1)
|
||||
|
||||
# 5. Submit workflow
|
||||
print("\n=== Submit workflow (512x512, 8 steps) ===")
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}},
|
||||
"6": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"7": {"class_type": "KSampler",
|
||||
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple",
|
||||
"positive": ["4", 0], "negative": ["5", 0],
|
||||
"latent_image": ["6", 0], "denoise": 1.0}},
|
||||
"8": {"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
|
||||
"9": {"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
|
||||
}
|
||||
}
|
||||
|
||||
wf_json = json.dumps(workflow)
|
||||
# Write workflow to file to avoid shell escaping issues
|
||||
run(f"cat > /tmp/zimage_wf.json << 'JSONEOF'\n{wf_json}\nJSONEOF")
|
||||
out, _ = run("curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H 'Content-Type: application/json' "
|
||||
"-d @/tmp/zimage_wf.json")
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f" API ERROR: {resp['error']}")
|
||||
sys.exit(1)
|
||||
print(f" Prompt ID: {resp.get('prompt_id')}")
|
||||
except:
|
||||
print(f" Response: {out.strip()[:500]}")
|
||||
|
||||
# 6. Monitor — wait up to 15 minutes (first run can be slow due to kernel cache)
|
||||
print("\n=== Monitoring generation (GPU kernels may compile on first step) ===")
|
||||
last_log = ""
|
||||
for i in range(60): # up to 15 minutes
|
||||
time.sleep(15)
|
||||
|
||||
stats, _ = run("bash -c '"
|
||||
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
||||
" RSS=$(ps -p $PID -o rss --no-headers); "
|
||||
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1); "
|
||||
" GPU_T=$(cat /sys/class/drm/card1/device/hwmon/hwmon*/temp1_input 2>/dev/null || echo 0); "
|
||||
" echo \"CPU:${CPU}% RSS:$((RSS/1024))M LOAD:${LOAD} GPU:$((GPU_T/1000))C\"; "
|
||||
"else echo DEAD; fi'")
|
||||
|
||||
log, _ = run("tail -12 /home/fabian/comfyui.log 2>/dev/null")
|
||||
log_s = log.strip()
|
||||
|
||||
elapsed = (i+1) * 15
|
||||
m, s = divmod(elapsed, 60)
|
||||
stats_s = stats.strip()
|
||||
print(f" [{m}m{s:02d}s] {stats_s}")
|
||||
|
||||
# Show new log content
|
||||
if log_s != last_log:
|
||||
for line in reversed(log_s.split('\n')):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
|
||||
print(f" LOG: {l[:120]}")
|
||||
break
|
||||
last_log = log_s
|
||||
|
||||
if 'DEAD' in stats_s:
|
||||
print("\n PROCESS DIED!")
|
||||
out, _ = run("tail -60 /home/fabian/comfyui.log")
|
||||
print(out)
|
||||
break
|
||||
|
||||
if 'Prompt executed in' in log_s:
|
||||
print(f"\n SUCCESS! Image generated!")
|
||||
out, _ = run("tail -25 /home/fabian/comfyui.log")
|
||||
print(out)
|
||||
break
|
||||
|
||||
if 'Traceback' in log_s or 'RuntimeError' in log_s:
|
||||
print("\n ERROR detected!")
|
||||
out, _ = run("tail -60 /home/fabian/comfyui.log")
|
||||
print(out)
|
||||
break
|
||||
|
||||
# 7. Check output
|
||||
print("\n=== Output files ===")
|
||||
out, _ = run("ls -lah ~/ComfyUI/output/ 2>/dev/null")
|
||||
print(out.strip())
|
||||
|
||||
# 8. Check history
|
||||
out, _ = run("curl -s http://localhost:8188/history 2>/dev/null")
|
||||
try:
|
||||
h = json.loads(out)
|
||||
for pid, info in h.items():
|
||||
status = info.get('status', {})
|
||||
outputs = info.get('outputs', {})
|
||||
print(f"\n Prompt {pid[:12]}...: status={status}")
|
||||
if outputs:
|
||||
for nid, nout in outputs.items():
|
||||
if isinstance(nout, dict) and 'images' in nout:
|
||||
for img in nout['images']:
|
||||
print(f" Image: {img.get('filename', 'unknown')}")
|
||||
except:
|
||||
pass
|
||||
|
||||
finally:
|
||||
ssh.close()
|
||||
print("\nSSH connection closed.")
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Monitor ComfyUI generation progress - poll every 20s."""
|
||||
import paramiko, json, time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
return stdout.read().decode()
|
||||
|
||||
last_log_hash = ""
|
||||
for i in range(90): # up to 30 minutes
|
||||
time.sleep(20)
|
||||
|
||||
# Process stats
|
||||
stats = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
||||
" MEM=$(ps -p $PID -o rss --no-headers); "
|
||||
" echo \"CPU:${CPU}% RSS:$((MEM/1024))MB LOAD:$(cat /proc/loadavg | cut -d\" \" -f1-3)\"; "
|
||||
"else echo DEAD; fi'").strip()
|
||||
|
||||
# GPU
|
||||
gpu = run("bash -c 'rocm-smi --showuse --showmemuse 2>/dev/null | grep -E \"GPU|%\" | head -5 || echo no-gpu'").strip()
|
||||
|
||||
# Log tail
|
||||
log = run("tail -10 /home/fabian/comfyui.log 2>/dev/null").strip()
|
||||
log_hash = hash(log)
|
||||
|
||||
elapsed = (i+1) * 20
|
||||
mins = elapsed // 60
|
||||
secs = elapsed % 60
|
||||
print(f"[{mins}m{secs:02d}s] {stats}")
|
||||
|
||||
# Show GPU line
|
||||
for line in gpu.split('\n'):
|
||||
if '%' in line or 'GPU' in line:
|
||||
print(f" GPU: {line.strip()}")
|
||||
break
|
||||
|
||||
# Show last meaningful log line
|
||||
if log_hash != last_log_hash:
|
||||
for line in reversed(log.split('\n')):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH'):
|
||||
print(f" LOG: {l}")
|
||||
break
|
||||
last_log_hash = log_hash
|
||||
|
||||
if 'DEAD' in stats:
|
||||
print("\nPROCESS DIED!")
|
||||
print(run("tail -60 /home/fabian/comfyui.log 2>/dev/null"))
|
||||
break
|
||||
|
||||
if 'Prompt executed in' in log:
|
||||
print(f"\nSUCCESS! Image generated!")
|
||||
print(run("tail -30 /home/fabian/comfyui.log 2>/dev/null"))
|
||||
print("\n=== OUTPUT FILES ===")
|
||||
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null"))
|
||||
break
|
||||
|
||||
if 'Traceback' in log or 'CUDA out of memory' in log or 'RuntimeError' in log:
|
||||
print(f"\nERROR!")
|
||||
print(run("tail -60 /home/fabian/comfyui.log 2>/dev/null"))
|
||||
break
|
||||
|
||||
else:
|
||||
print("\nTimed out after 30 minutes")
|
||||
print(run("tail -40 /home/fabian/comfyui.log 2>/dev/null"))
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,58 @@
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
print("Connected. Monitoring sampling progress...")
|
||||
|
||||
for i in range(40): # up to 10 minutes
|
||||
# Check log for sampling progress
|
||||
_, o, _ = c.exec_command('tail -5 /tmp/comfyui.log 2>/dev/null')
|
||||
log = o.read().decode(errors='replace').strip()
|
||||
|
||||
# Check GPU temp
|
||||
_, o, _ = c.exec_command('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null')
|
||||
temp = o.read().decode().strip()
|
||||
temp_c = int(temp) // 1000 if temp.isdigit() else '?'
|
||||
|
||||
# Check GPU usage
|
||||
_, o, _ = c.exec_command('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null')
|
||||
gpu_pct = o.read().decode().strip()
|
||||
|
||||
# Check output dir for generated images
|
||||
_, o, _ = c.exec_command('ls ~/ComfyUI/output/*.png 2>/dev/null')
|
||||
files = o.read().decode().strip()
|
||||
|
||||
# Check queue
|
||||
_, o, _ = c.exec_command('curl -s http://127.0.0.1:8188/queue 2>/dev/null')
|
||||
queue_raw = o.read().decode().strip()
|
||||
|
||||
# Parse last log line for progress
|
||||
last_line = log.split('\n')[-1] if log else ''
|
||||
print(f"[{i*15:>3}s] GPU:{gpu_pct}% temp:{temp_c}C | {last_line[-120:]}")
|
||||
|
||||
if files:
|
||||
print(f"\n*** IMAGE GENERATED! ***")
|
||||
print(f"Files: {files}")
|
||||
# Get last 10 lines of log for timing info
|
||||
_, o, _ = c.exec_command('tail -10 /tmp/comfyui.log 2>/dev/null')
|
||||
print(o.read().decode(errors='replace'))
|
||||
break
|
||||
|
||||
try:
|
||||
qdata = json.loads(queue_raw)
|
||||
running = len(qdata.get('queue_running', []))
|
||||
pending = len(qdata.get('queue_pending', []))
|
||||
if running == 0 and pending == 0 and i > 3:
|
||||
print("\nQueue empty - job finished or failed. Last 30 log lines:")
|
||||
_, o, _ = c.exec_command('tail -30 /tmp/comfyui.log 2>/dev/null')
|
||||
print(o.read().decode(errors='replace'))
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
time.sleep(15)
|
||||
|
||||
c.close()
|
||||
print("Monitor done.")
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Fix: Back to --novram (proven working for GPU sampling) + --cpu-vae."""
|
||||
import paramiko, time, json, textwrap
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
# Use bash array to avoid quoting issues
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
def sftp_write(path, content):
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open(path, 'w') as f:
|
||||
f.write(content)
|
||||
sftp.close()
|
||||
|
||||
def sftp_read(path):
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open(path, 'r') as f:
|
||||
data = f.read().decode(errors='replace')
|
||||
sftp.close()
|
||||
return data
|
||||
|
||||
# ---- STEP 1: Kill ----
|
||||
print("STEP 1: Kill ComfyUI")
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
print(" Killed.")
|
||||
|
||||
# ---- STEP 2: Write launcher ----
|
||||
print("\nSTEP 2: Write launcher with --novram (PROVEN to work on this APU)")
|
||||
launcher = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export MIOPEN_FIND_MODE=1
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
# --novram = model weights on CPU, GPU only for compute (correct for shared-memory APU)
|
||||
# --cpu-vae = VAE decode on CPU (fixes known hang on this GPU)
|
||||
# --force-fp16 = half precision to save memory
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--novram \\
|
||||
--force-fp16 \\
|
||||
--cpu-vae \\
|
||||
--disable-smart-memory
|
||||
""")
|
||||
sftp_write('/tmp/run_comfyui.sh', launcher)
|
||||
sh('chmod +x /tmp/run_comfyui.sh')
|
||||
print(" Written: --novram --force-fp16 --cpu-vae --disable-smart-memory")
|
||||
|
||||
# ---- STEP 3: Start ----
|
||||
print("\nSTEP 3: Start ComfyUI")
|
||||
sh('rm -f /tmp/comfyui.log; touch /tmp/comfyui.log')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
if not pid:
|
||||
print(" FAILED!")
|
||||
print(sftp_read('/tmp/comfyui.log'))
|
||||
c.close()
|
||||
exit(1)
|
||||
print(f" PID: {pid}")
|
||||
|
||||
# ---- STEP 4: Wait for HTTP 200 ----
|
||||
print("\nSTEP 4: Wait for server ready", end='', flush=True)
|
||||
for i in range(120):
|
||||
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if '200' in code:
|
||||
print(f" READY ({i*2}s)")
|
||||
break
|
||||
# Check if process died
|
||||
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
|
||||
if alive == 'N':
|
||||
print("\n Process died!")
|
||||
print(sftp_read('/tmp/comfyui.log'))
|
||||
c.close()
|
||||
exit(1)
|
||||
if i % 10 == 0 and i > 0:
|
||||
log = sftp_read('/tmp/comfyui.log')
|
||||
lines = [l for l in log.split('\n') if l.strip()]
|
||||
print(f"\n [{i*2}s] {lines[-1][:80] if lines else '...'}", end='', flush=True)
|
||||
else:
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
else:
|
||||
print("\n TIMEOUT!")
|
||||
print(sftp_read('/tmp/comfyui.log')[-1000:])
|
||||
c.close()
|
||||
exit(1)
|
||||
|
||||
# Verify startup flags
|
||||
log = sftp_read('/tmp/comfyui.log')
|
||||
if 'NO_VRAM' in log or 'NOVRAM' in log.upper():
|
||||
print(" Confirmed: NOVRAM mode (GPU compute only, model on CPU)")
|
||||
for line in log.split('\n'):
|
||||
if 'vram state' in line.lower():
|
||||
print(f" {line.strip()}")
|
||||
if 'Device:' in line:
|
||||
print(f" {line.strip()}")
|
||||
|
||||
# ---- STEP 5: Submit workflow ----
|
||||
print("\nSTEP 5: Submit workflow")
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
|
||||
}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}
|
||||
}
|
||||
sftp_write('/tmp/wf.json', json.dumps(workflow))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json', timeout=10)
|
||||
print(f" Response: {resp[:200]}")
|
||||
if 'prompt_id' not in resp:
|
||||
print(" FAILED!")
|
||||
c.close()
|
||||
exit(1)
|
||||
prompt_id = json.loads(resp).get('prompt_id', '?')
|
||||
print(f" Prompt ID: {prompt_id}")
|
||||
|
||||
# ---- STEP 6: Monitor ----
|
||||
print("\nSTEP 6: Monitor (expect GPU power >100W during sampling)")
|
||||
t0 = time.time()
|
||||
sampling_seen = False
|
||||
|
||||
for i in range(200):
|
||||
elapsed = int(time.time() - t0)
|
||||
|
||||
gpu_temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
|
||||
|
||||
# Read log via SFTP to avoid shell issues
|
||||
try:
|
||||
log = sftp_read('/tmp/comfyui.log')
|
||||
except:
|
||||
log = ''
|
||||
|
||||
lines = log.strip().split('\n')
|
||||
# Find last meaningful line (skip manager spam)
|
||||
last = ''
|
||||
for line in reversed(lines):
|
||||
if 'FETCH ComfyRegistry' not in line and 'All startup tasks' not in line and 'FETCH DATA' not in line and line.strip():
|
||||
last = line.strip()
|
||||
break
|
||||
|
||||
# Detect sampling progress
|
||||
for line in lines:
|
||||
if '/8' in line and 'it/s' in line:
|
||||
sampling_seen = True
|
||||
|
||||
print(f" [{elapsed:>4}s] {temp_c}C | {last[-100:]}")
|
||||
|
||||
# Check for output image
|
||||
imgs = sh('ls ~/ComfyUI/output/*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** IMAGE GENERATED! ***")
|
||||
print(f" File: {imgs}")
|
||||
print(f" Total: {elapsed}s")
|
||||
# Show last 15 lines
|
||||
for line in lines[-15:]:
|
||||
if line.strip():
|
||||
print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
# Check queue
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 30:
|
||||
time.sleep(3)
|
||||
imgs = sh('ls ~/ComfyUI/output/*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** IMAGE GENERATED! ***")
|
||||
print(f" File: {imgs}")
|
||||
else:
|
||||
print(f"\n Queue empty, no image. Error in log:")
|
||||
for line in lines[-20:]:
|
||||
if line.strip():
|
||||
print(f" {line}")
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
# Check process alive
|
||||
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
|
||||
if alive == 'N':
|
||||
print(f"\n *** CRASHED ***")
|
||||
for line in lines[-30:]:
|
||||
if line.strip():
|
||||
print(f" {line}")
|
||||
break
|
||||
|
||||
time.sleep(15)
|
||||
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Patch ComfyUI: Force VAE to GPU even in --novram mode. Restart and test."""
|
||||
import paramiko, time, json, textwrap
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# =============================================
|
||||
# STEP 1: Kill ComfyUI
|
||||
# =============================================
|
||||
print("1) Kill ComfyUI")
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
|
||||
# =============================================
|
||||
# STEP 2: Read and patch model_management.py
|
||||
# =============================================
|
||||
print("2) Patch model_management.py — force VAE to GPU")
|
||||
|
||||
# First, read the file to understand the structure
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
mgmt = f.read().decode()
|
||||
|
||||
print(f" File size: {len(mgmt)} bytes")
|
||||
|
||||
# Find the vae_offload_device function
|
||||
# In ComfyUI, with NO_VRAM, vae_offload_device() returns CPU
|
||||
# We need to make it return GPU instead
|
||||
|
||||
# Also find vae_dtype — it's set to float32 by default, we want float16
|
||||
|
||||
# Let's search for relevant functions
|
||||
for i, line in enumerate(mgmt.split('\n')):
|
||||
if 'def vae_offload_device' in line or 'def vae_dtype' in line or 'def vae_device' in line:
|
||||
print(f" Line {i+1}: {line.strip()}")
|
||||
|
||||
# Also check what functions exist
|
||||
found = []
|
||||
for i, line in enumerate(mgmt.split('\n')):
|
||||
if line.startswith('def ') or (line.startswith(' ') and 'def ' in line[:12]):
|
||||
if 'vae' in line.lower():
|
||||
found.append((i+1, line.strip()))
|
||||
for ln, l in found:
|
||||
print(f" L{ln}: {l}")
|
||||
|
||||
# Let's read the specific area around these functions
|
||||
lines = mgmt.split('\n')
|
||||
|
||||
# Find and show context around vae functions
|
||||
for keyword in ['vae_offload_device', 'vae_dtype', 'vae_device']:
|
||||
for i, line in enumerate(lines):
|
||||
if f'def {keyword}' in line:
|
||||
start = max(0, i-2)
|
||||
end = min(len(lines), i+15)
|
||||
print(f"\n --- {keyword} (L{i+1}) ---")
|
||||
for j in range(start, end):
|
||||
print(f" {j+1:>5}: {lines[j]}")
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\n Reading complete. Will patch next.")
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Fix: keep UNet+VAE both on GPU (shared memory). No offloading."""
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=30):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# Kill first
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
print("Killed ComfyUI")
|
||||
|
||||
# Read model_management.py
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
code = f.read().decode()
|
||||
|
||||
lines = code.split('\n')
|
||||
|
||||
# Show current offload functions to understand exact code
|
||||
print("\n=== Finding offload functions ===")
|
||||
for i, line in enumerate(lines):
|
||||
if 'def unet_offload_device' in line or 'def vae_offload_device' in line:
|
||||
print(f"\n--- {line.strip()} at line {i+1} ---")
|
||||
for j in range(i, min(i+10, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
|
||||
# ============ PATCH unet_offload_device ============
|
||||
# Current: returns CPU unless HIGH_VRAM
|
||||
# Fix: also return GPU for SHARED (APU shared memory = no point offloading)
|
||||
old_unet = None
|
||||
new_unet = None
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if 'def unet_offload_device' in line:
|
||||
# Grab the function body (next ~6 lines)
|
||||
chunk = '\n'.join(lines[i:i+8])
|
||||
print(f"\n=== unet_offload_device chunk ===\n{chunk}")
|
||||
|
||||
# The function checks HIGH_VRAM only. Add SHARED.
|
||||
if 'HIGH_VRAM' in chunk and 'SHARED' not in chunk:
|
||||
old_unet = chunk
|
||||
new_unet = chunk.replace(
|
||||
'vram_state == VRAMState.HIGH_VRAM',
|
||||
'vram_state == VRAMState.HIGH_VRAM or vram_state == VRAMState.SHARED'
|
||||
)
|
||||
print(f"\n -> Will patch to include SHARED")
|
||||
elif 'SHARED' in chunk:
|
||||
print(f"\n -> Already patched for SHARED")
|
||||
break
|
||||
|
||||
# ============ PATCH vae_offload_device ============
|
||||
# Current: returns CPU unless --gpu-only
|
||||
# Fix: also return GPU for SHARED
|
||||
old_vae = None
|
||||
new_vae = None
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if 'def vae_offload_device' in line:
|
||||
chunk = '\n'.join(lines[i:i+8])
|
||||
print(f"\n=== vae_offload_device chunk ===\n{chunk}")
|
||||
|
||||
if 'args.gpu_only' in chunk and 'SHARED' not in chunk:
|
||||
old_vae = chunk
|
||||
new_vae = chunk.replace(
|
||||
'args.gpu_only',
|
||||
'args.gpu_only or vram_state == VRAMState.SHARED'
|
||||
)
|
||||
print(f"\n -> Will patch to include SHARED")
|
||||
elif 'SHARED' in chunk:
|
||||
print(f"\n -> Already patched for SHARED")
|
||||
break
|
||||
|
||||
# Also check text_encoder_offload_device
|
||||
for i, line in enumerate(lines):
|
||||
if 'def text_encoder_offload_device' in line:
|
||||
chunk = '\n'.join(lines[i:i+8])
|
||||
print(f"\n=== text_encoder_offload_device chunk ===\n{chunk}")
|
||||
break
|
||||
|
||||
# Apply patches
|
||||
patched = False
|
||||
if old_unet and new_unet:
|
||||
code = code.replace(old_unet, new_unet)
|
||||
patched = True
|
||||
print("\n[OK] Patched unet_offload_device")
|
||||
|
||||
if old_vae and new_vae:
|
||||
code = code.replace(old_vae, new_vae)
|
||||
patched = True
|
||||
print("[OK] Patched vae_offload_device")
|
||||
|
||||
if patched:
|
||||
# Backup and write
|
||||
sh('cp /home/fabian/ComfyUI/comfy/model_management.py /home/fabian/ComfyUI/comfy/model_management.py.bak2')
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
|
||||
f.write(code)
|
||||
print("[OK] Written to disk")
|
||||
else:
|
||||
print("[INFO] No patches needed (already applied or code changed)")
|
||||
|
||||
# Verify
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
verify = f.read().decode()
|
||||
for i, line in enumerate(verify.split('\n')):
|
||||
if 'def unet_offload_device' in line or 'def vae_offload_device' in line:
|
||||
print(f"\n--- VERIFY {line.strip()} ---")
|
||||
for j in range(i, min(i+8, len(verify.split(chr(10))))):
|
||||
print(f" {j+1}: {verify.split(chr(10))[j]}")
|
||||
|
||||
# ============ RESTART ============
|
||||
print("\n=== RESTARTING ===")
|
||||
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
print(f"PID: {pid}")
|
||||
|
||||
# Wait for ready
|
||||
print("Waiting for HTTP", end='', flush=True)
|
||||
for i in range(90):
|
||||
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if '200' in r:
|
||||
print(f" OK ({i*2}s)")
|
||||
break
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
else:
|
||||
print(" TIMEOUT")
|
||||
|
||||
# Check SHARED mode active
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(x in s.lower() for x in ['vram state', 'shared', 'device:', 'total vram']):
|
||||
print(f" {s}")
|
||||
|
||||
# Submit workflow
|
||||
print("\nSubmitting workflow...")
|
||||
wf = {"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(wf))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f" {resp[:150]}")
|
||||
|
||||
# Monitor - watch for UNet+VAE both on GPU, fast VAE
|
||||
print("\nMonitoring...")
|
||||
t0 = time.time()
|
||||
for i in range(120):
|
||||
el = int(time.time() - t0)
|
||||
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
# GPU usage
|
||||
gpu = sh('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null || echo ?', timeout=5)
|
||||
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
tc = int(temp)//1000 if temp.isdigit() else '?'
|
||||
|
||||
# Get latest progress line
|
||||
last_progress = ''
|
||||
last_line = ''
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if '/8' in s and ('it/s' in s or 's/it' in s): last_progress = s
|
||||
if 'loaded' in s.lower() or 'VAE' in s or 'Requested' in s or 'Prompt executed' in s:
|
||||
last_line = s
|
||||
if s and 'FETCH' not in s: last_line = s
|
||||
|
||||
status = last_progress or last_line
|
||||
print(f" [{el:>3}s] GPU:{gpu}% {tc}C | {status[-100:]}")
|
||||
|
||||
# Check for output
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** IMAGE DONE! *** {imgs}")
|
||||
print(f" Wall time: {el}s")
|
||||
# Print key log lines
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(x in s for x in ['loaded', 'load device', 'offload device', 'Prompt executed', '/8', 'Requested', 'VAE']):
|
||||
if 'FETCH' not in s:
|
||||
print(f" {s}")
|
||||
break
|
||||
|
||||
# Check queue empty
|
||||
if el > 30:
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending'):
|
||||
time.sleep(3)
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** DONE: {imgs} ***")
|
||||
else:
|
||||
print(f"\n Queue empty, no image. Error?")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip() and 'FETCH' not in line: print(f" {line.strip()}")
|
||||
break
|
||||
except: pass
|
||||
|
||||
# Check alive
|
||||
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
|
||||
print("\n CRASHED!")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,31 @@
|
||||
import paramiko
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
# Full log
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if not s or 'FETCH' in s or 'startup tasks' in s or 'DEPRECATION' in s: continue
|
||||
print(s)
|
||||
|
||||
# Process + GPU
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(10)
|
||||
chan.exec_command('/bin/bash -c "echo === PROC ===; ps aux | grep main.py | grep -v grep; echo === GPU ===; rocm-smi 2>/dev/null | head -12; echo === VRAM ===; rocm-smi --showmeminfo vram 2>/dev/null"')
|
||||
o = b""
|
||||
while True:
|
||||
try:
|
||||
ch = chan.recv(65536)
|
||||
if not ch: break
|
||||
o += ch
|
||||
except: break
|
||||
chan.close()
|
||||
print(o.decode(errors='replace'))
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quick status check."""
|
||||
import paramiko
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
def run(cmd):
|
||||
_, so, se = ssh.exec_command(cmd, timeout=15)
|
||||
return so.read().decode()
|
||||
|
||||
print("=== PROCESS ===")
|
||||
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); ps -p $PID -o pid,%cpu,%mem,nlwp --no-headers 2>/dev/null; echo LOAD: $(cat /proc/loadavg)'"))
|
||||
|
||||
print("=== GPU ===")
|
||||
print(run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null | tail -8"))
|
||||
|
||||
print("=== LOG (last 15) ===")
|
||||
print(run("tail -15 /home/fabian/comfyui.log 2>/dev/null"))
|
||||
|
||||
print("=== MEMORY ===")
|
||||
print(run("free -h"))
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Read raw ComfyUI log - no filtering, no quoting issues."""
|
||||
import paramiko
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
|
||||
# Read the ENTIRE log via SFTP - no shell, no grep, no quoting
|
||||
sftp = c.open_sftp()
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
|
||||
lines = log.split('\n')
|
||||
print(f"Total log lines: {len(lines)}")
|
||||
print()
|
||||
|
||||
# Print everything that's NOT ComfyUI-Manager registry spam
|
||||
for line in lines:
|
||||
if 'FETCH ComfyRegistry' in line:
|
||||
continue
|
||||
if 'All startup tasks' in line:
|
||||
continue
|
||||
if line.strip():
|
||||
print(line)
|
||||
except Exception as e:
|
||||
print(f"Error reading log: {e}")
|
||||
finally:
|
||||
sftp.close()
|
||||
|
||||
# Also check: is the process actually using GPU memory?
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(10)
|
||||
chan.exec_command('/bin/bash -c "rocm-smi --showmeminfo vram 2>/dev/null"')
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
print("\n=== VRAM Info ===")
|
||||
print(out.decode(errors='replace'))
|
||||
|
||||
c.close()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Read more of model_management.py — find how --novram affects GPU compute."""
|
||||
import paramiko
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
mgmt = f.read().decode()
|
||||
|
||||
lines = mgmt.split('\n')
|
||||
|
||||
# Find all functions related to device/offload
|
||||
print("=== KEY FUNCTIONS ===")
|
||||
for keyword in ['unet_offload_device', 'unet_device', 'NO_VRAM', 'should_use', 'get_torch_device',
|
||||
'def text_encoder_device', 'def text_encoder_offload', 'VRAMState']:
|
||||
for i, line in enumerate(lines):
|
||||
if keyword in line and ('def ' in line or 'class ' in line or '=' in line[:50]):
|
||||
print(f" L{i+1}: {line.strip()[:100]}")
|
||||
|
||||
# Show VRAMState enum
|
||||
print("\n=== VRAMState ===")
|
||||
for i, line in enumerate(lines):
|
||||
if 'class VRAMState' in line or (i > 0 and 'VRAMState' in lines[i-1] and 'class' in lines[i-1]):
|
||||
for j in range(i, min(i+15, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
break
|
||||
|
||||
# Show unet_offload_device
|
||||
print("\n=== unet_offload_device ===")
|
||||
for i, line in enumerate(lines):
|
||||
if 'def unet_offload_device' in line:
|
||||
for j in range(max(0,i-2), min(i+15, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
|
||||
# Show text_encoder functions
|
||||
print("\n=== text_encoder_device ===")
|
||||
for i, line in enumerate(lines):
|
||||
if 'def text_encoder_device' in line:
|
||||
for j in range(max(0,i-2), min(i+12, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
|
||||
print("\n=== text_encoder_offload_device ===")
|
||||
for i, line in enumerate(lines):
|
||||
if 'def text_encoder_offload_device' in line:
|
||||
for j in range(max(0,i-2), min(i+12, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
|
||||
# Show how NO_VRAM is used in loading logic
|
||||
print("\n=== NO_VRAM usage in model loading ===")
|
||||
for i, line in enumerate(lines):
|
||||
if 'NO_VRAM' in line:
|
||||
print(f" L{i+1}: {line.strip()[:120]}")
|
||||
|
||||
# Show the VRAM state setting logic
|
||||
print("\n=== vram_state assignment ===")
|
||||
for i, line in enumerate(lines):
|
||||
if 'vram_state' in line and ('=' in line) and 'VRAMState' in line:
|
||||
print(f" L{i+1}: {line.strip()[:120]}")
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Read the SHARED vram state logic and CLI args."""
|
||||
import paramiko
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
# Read model_management.py around line 440-470 (where SHARED is set)
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
mgmt = f.read().decode()
|
||||
lines = mgmt.split('\n')
|
||||
|
||||
print("=== L430-475: VRAM state setting ===")
|
||||
for i in range(429, min(475, len(lines))):
|
||||
print(f" {i+1}: {lines[i]}")
|
||||
|
||||
print("\n=== L750-790: NO_VRAM model loading ===")
|
||||
for i in range(749, min(790, len(lines))):
|
||||
print(f" {i+1}: {lines[i]}")
|
||||
|
||||
print("\n=== L850-870: Smart memory / offload ===")
|
||||
for i in range(849, min(870, len(lines))):
|
||||
print(f" {i+1}: {lines[i]}")
|
||||
|
||||
# Check CLI args for shared memory
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/cli_args.py', 'r') as f:
|
||||
cli = f.read().decode()
|
||||
|
||||
print("\n=== CLI args with 'shared' or 'SHARED' ===")
|
||||
for i, line in enumerate(cli.split('\n')):
|
||||
if 'shared' in line.lower():
|
||||
print(f" L{i+1}: {line.strip()}")
|
||||
|
||||
# Check what --gpu-only does
|
||||
print("\n=== CLI args with 'gpu_only' ===")
|
||||
for i, line in enumerate(cli.split('\n')):
|
||||
if 'gpu_only' in line.lower() or 'gpu-only' in line.lower():
|
||||
print(f" L{i+1}: {line.strip()}")
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Reconnect, check patches, fix missing ones, restart."""
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, t=30):
|
||||
ch = c.get_transport().open_session()
|
||||
ch.settimeout(t)
|
||||
ch.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
o = b""
|
||||
while True:
|
||||
try:
|
||||
d = ch.recv(65536)
|
||||
if not d: break
|
||||
o += d
|
||||
except: break
|
||||
ch.close()
|
||||
return o.decode(errors='replace').strip()
|
||||
|
||||
# Read current state
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
code = f.read().decode()
|
||||
lines = code.split('\n')
|
||||
|
||||
# Show ALL offload functions
|
||||
for fname in ['unet_offload_device', 'vae_offload_device', 'text_encoder_offload_device']:
|
||||
for i, line in enumerate(lines):
|
||||
if f'def {fname}' in line:
|
||||
print(f"\n=== {fname} (L{i+1}) ===")
|
||||
for j in range(i, min(i+10, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
break
|
||||
|
||||
# Also show unet_inital_load_device
|
||||
for i, line in enumerate(lines):
|
||||
if 'def unet_inital_load_device' in line:
|
||||
print(f"\n=== unet_inital_load_device (L{i+1}) ===")
|
||||
for j in range(i, min(i+10, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
break
|
||||
|
||||
# Show SHARED patch
|
||||
for i, line in enumerate(lines):
|
||||
if 'COMFYUI_SHARED_MEMORY' in line:
|
||||
print(f"\n=== SHARED patch (L{i+1}) ===")
|
||||
for j in range(max(0,i-2), min(i+5, len(lines))):
|
||||
print(f" {j+1}: {lines[j]}")
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone reading.")
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Fix: Remove --novram so ComfyUI actually uses the GPU for compute."""
|
||||
import paramiko, time, json, textwrap
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
|
||||
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# STEP 1: Kill
|
||||
print("STEP 1: Kill")
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
print(" Done")
|
||||
|
||||
# STEP 2: Write new launcher - NO --novram, NO --lowvram
|
||||
# ComfyUI sees 7602MB VRAM → will use NORMAL_VRAM mode → GPU compute
|
||||
print("\nSTEP 2: New launcher (NO memory flags = auto GPU)")
|
||||
launcher = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
# GPU
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
# Threading
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
# MIOpen - use fast mode, persistent cache
|
||||
export MIOPEN_FIND_MODE=3
|
||||
export MIOPEN_LOG_LEVEL=3
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
# NO --novram, NO --lowvram = ComfyUI auto-detects 7602MB VRAM = GPU compute
|
||||
# --force-fp16 = half precision saves memory
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--force-fp16
|
||||
""")
|
||||
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
|
||||
f.write(launcher)
|
||||
sh('chmod +x /tmp/run_comfyui.sh')
|
||||
print(" Flags: --force-fp16 ONLY (auto VRAM mode)")
|
||||
|
||||
# STEP 3: Start
|
||||
print("\nSTEP 3: Start ComfyUI")
|
||||
sh('rm -f /tmp/comfyui.log; touch /tmp/comfyui.log')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
print(f" PID: {pid}")
|
||||
|
||||
# STEP 4: Wait for ready
|
||||
print("\nSTEP 4: Wait for HTTP ready", end='', flush=True)
|
||||
for i in range(120):
|
||||
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if '200' in code:
|
||||
print(f" READY ({i*2}s)")
|
||||
break
|
||||
if i % 10 == 0 and i > 0:
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
lines = [l.strip() for l in log.split('\n') if l.strip() and 'FETCH' not in l and 'DEPRECATION' not in l]
|
||||
print(f"\n [{i*2}s] {lines[-1][:80] if lines else '...'}", end='', flush=True)
|
||||
except: pass
|
||||
else:
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
else:
|
||||
print("\n TIMEOUT!")
|
||||
exit(1)
|
||||
|
||||
# Show VRAM state
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['vram state', 'Device:', 'Total VRAM', 'VRAM', 'pytorch version']):
|
||||
print(f" {s}")
|
||||
|
||||
# STEP 5: Submit workflow
|
||||
print("\nSTEP 5: Submit workflow")
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
|
||||
}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}
|
||||
}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json', timeout=10)
|
||||
print(f" {resp[:150]}")
|
||||
if 'prompt_id' not in resp:
|
||||
print(" FAILED!")
|
||||
exit(1)
|
||||
|
||||
# STEP 6: Monitor - focus on GPU usage and sampling speed
|
||||
print("\nSTEP 6: Monitor")
|
||||
print(" First step may be slow (MIOpen kernel compilation). Be patient.")
|
||||
t0 = time.time()
|
||||
for i in range(200):
|
||||
elapsed = int(time.time() - t0)
|
||||
|
||||
gpu_temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
|
||||
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
# Find sampling progress and last meaningful line
|
||||
sampling = ''
|
||||
last = ''
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if '/8' in s and ('it/s' in s or 's/it' in s):
|
||||
sampling = s
|
||||
if s and 'FETCH' not in s and 'startup tasks' not in s and 'DEPRECATION' not in s:
|
||||
last = s
|
||||
|
||||
display = sampling if sampling else last[-100:]
|
||||
print(f" [{elapsed:>4}s] {temp_c}C | {display}")
|
||||
|
||||
# Check output
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
# Get timing from log
|
||||
exec_time = ''
|
||||
for line in log.split('\n'):
|
||||
if 'Prompt executed in' in line:
|
||||
exec_time = line.strip()
|
||||
print(f"\n *** IMAGE GENERATED! ***")
|
||||
print(f" File: {imgs}")
|
||||
print(f" {exec_time}")
|
||||
print(f" Wall time: {elapsed}s")
|
||||
|
||||
# Show vram state and model loading details
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['loaded completely', 'loaded partially', 'vram state', '/8']):
|
||||
print(f" {s}")
|
||||
break
|
||||
|
||||
# Check queue empty
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 30:
|
||||
time.sleep(3)
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** IMAGE GENERATED: {imgs} ***")
|
||||
else:
|
||||
print(f"\n Queue empty, no image. Log errors:")
|
||||
for line in log.split('\n')[-25:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
except: pass
|
||||
|
||||
# Process alive?
|
||||
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
|
||||
if alive == 'N':
|
||||
print(f"\n *** CRASHED ***")
|
||||
for line in log.split('\n')[-30:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(15)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Recreate launcher (lost on reboot) and start ComfyUI."""
|
||||
import paramiko, time, json, sys
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, t=30):
|
||||
stdin, stdout, stderr = c.exec_command(f"bash -lc '{cmd}'", timeout=t)
|
||||
return stdout.read().decode(errors='replace').strip()
|
||||
|
||||
# Recreate launcher (wiped by reboot since /tmp)
|
||||
launcher = """#!/bin/bash
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export MIOPEN_FIND_MODE=3
|
||||
export COMFYUI_SHARED_MEMORY=1
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--force-fp16 \\
|
||||
--fp16-vae
|
||||
"""
|
||||
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
|
||||
f.write(launcher)
|
||||
sh("chmod +x /tmp/run_comfyui.sh")
|
||||
print("Launcher recreated")
|
||||
|
||||
# Verify patches
|
||||
p = sh("grep -c COMFYUI_SHARED_MEMORY ~/ComfyUI/comfy/model_management.py")
|
||||
print(f"SHARED patch refs: {p}")
|
||||
p2 = sh("grep 'def unet_offload_device' -A2 ~/ComfyUI/comfy/model_management.py | head -3")
|
||||
print(f"unet_offload: {p2}")
|
||||
p3 = sh("grep 'def vae_offload_device' -A2 ~/ComfyUI/comfy/model_management.py | head -3")
|
||||
print(f"vae_offload: {p3}")
|
||||
|
||||
# Start
|
||||
sh("rm -f /tmp/comfyui.log")
|
||||
sh("nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &")
|
||||
time.sleep(4)
|
||||
pid = sh("pgrep -f 'python3.*main.py'")
|
||||
print(f"PID: {pid}")
|
||||
|
||||
if not pid:
|
||||
print("FAILED! Log:")
|
||||
print(sh("cat /tmp/comfyui.log 2>/dev/null"))
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
|
||||
# Wait for HTTP
|
||||
print("Waiting for HTTP...", end='', flush=True)
|
||||
for i in range(90):
|
||||
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null')
|
||||
if '200' in code:
|
||||
print(f" ready ({i*2}s)")
|
||||
break
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
else:
|
||||
print(" TIMEOUT")
|
||||
print(sh("tail -30 /tmp/comfyui.log"))
|
||||
c.close()
|
||||
sys.exit(1)
|
||||
|
||||
# Log state
|
||||
log = sh("cat /tmp/comfyui.log")
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if any(x in s for x in ['vram state', 'SHARED', 'Device:', 'Total VRAM']):
|
||||
print(f" {s}")
|
||||
|
||||
# Submit
|
||||
wf = {"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(wf))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f"Submitted: {resp[:100]}")
|
||||
|
||||
# Monitor
|
||||
t0 = time.time()
|
||||
shown = set()
|
||||
for _ in range(200):
|
||||
el = int(time.time() - t0)
|
||||
log = sh("cat /tmp/comfyui.log 2>/dev/null")
|
||||
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if s not in shown and any(x in s for x in ['/8', 'loaded completely', 'load device', 'offload device', 'Requested to load', 'Prompt executed', 'Error', 'OOM']):
|
||||
if 'FETCH' not in s and 'audio' not in s and 'split attention' not in s:
|
||||
gpu = sh("cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null")
|
||||
print(f" [{el:>3}s] GPU:{gpu}% | {s[-110:]}")
|
||||
shown.add(s)
|
||||
|
||||
if 'Prompt executed' in log:
|
||||
print(f"\n*** DONE in {el}s! ***")
|
||||
imgs = sh("ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null")
|
||||
print(f" Images: {imgs}")
|
||||
break
|
||||
|
||||
alive = sh("pgrep -c -f 'python3.*main.py' 2>/dev/null")
|
||||
if alive == '0':
|
||||
print(f"\nCRASHED at {el}s!")
|
||||
for l in log.split('\n')[-20:]:
|
||||
if l.strip(): print(f" {l.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recon the BC-250 for PyTorch/ComfyUI installation."""
|
||||
import paramiko, sys
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
cmds = [
|
||||
("Python version", "python3 --version 2>&1"),
|
||||
("pip version", "pip --version 2>&1 || pip3 --version 2>&1"),
|
||||
("Disk space", "df -h / /home 2>&1"),
|
||||
("RAM", "free -h 2>&1"),
|
||||
("CPU cores", "nproc 2>&1"),
|
||||
("ROCm version", "cat /opt/rocm/.info/version 2>/dev/null || echo 'no version file'; ls /opt/rocm/lib/libamdhip64.so* 2>&1"),
|
||||
("hipcc", "which hipcc 2>&1 && hipcc --version 2>&1 | head -5"),
|
||||
("rocminfo GPU", "HSA_OVERRIDE_GFX_VERSION=10.1.0 rocminfo 2>&1 | grep -E 'Marketing|gfx|Name:' | head -10"),
|
||||
("Existing PyTorch", "python3 -c 'import torch; print(torch.__version__); print(torch.cuda.is_available()); print(torch.version.hip)' 2>&1"),
|
||||
("Existing venvs", "ls -la ~/venv* ~/env* ~/.local/lib/python*/site-packages/torch* 2>&1 | head -20"),
|
||||
("git version", "git --version 2>&1"),
|
||||
("cmake version", "cmake --version 2>&1 | head -1"),
|
||||
("ninja version", "ninja --version 2>&1"),
|
||||
("Available Python packages", "python3 -m venv --help >/dev/null 2>&1 && echo 'venv OK' || echo 'venv missing'"),
|
||||
("Swap", "swapon --show 2>&1"),
|
||||
("GPU device check", "ls -la /dev/kfd /dev/dri/render* 2>&1"),
|
||||
("Existing ComfyUI", "ls -la ~/ComfyUI 2>&1 || echo 'not found'"),
|
||||
("pacman cmake/ninja", "pacman -Q cmake ninja 2>&1"),
|
||||
]
|
||||
|
||||
for label, cmd in cmds:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {label}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=30)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
print(f"STDERR: {err.strip()}")
|
||||
|
||||
ssh.close()
|
||||
print("\n\nDone.")
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check Python versions and z-image-turbo info on BC-250."""
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
cmds = [
|
||||
("Python 3.12 available?", "pacman -Ss python | grep -E 'python3\\.1[0-3]|python 3\\.' 2>&1 | head -20"),
|
||||
("All python packages", "pacman -Q | grep python 2>&1 | head -30"),
|
||||
("pip via python", "python3 -m pip --version 2>&1"),
|
||||
("pip package", "pacman -Q python-pip 2>&1"),
|
||||
("check pyenv", "which pyenv 2>&1; pacman -Q pyenv 2>&1"),
|
||||
("check python3.12", "which python3.12 2>&1; pacman -Q python312 2>&1; ls /usr/bin/python3.1* 2>&1"),
|
||||
("ninja available", "pacman -Ss '^ninja$' 2>&1 | head -5"),
|
||||
("check ccache", "which ccache 2>&1; pacman -Q ccache 2>&1"),
|
||||
("check z-image-turbo", "pacman -Ss z-image 2>&1; pip3 search z-image-turbo 2>&1 || true"),
|
||||
("check huggingface tools", "pacman -Q | grep -i hugging 2>&1; python3 -c 'import huggingface_hub' 2>&1 || true"),
|
||||
]
|
||||
|
||||
for label, cmd in cmds:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {label}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=30)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
print(f"STDERR: {err.strip()}")
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Post-reboot: start patched ComfyUI, submit, monitor. Single connection."""
|
||||
import paramiko, time, json, sys
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
|
||||
def sh(cmd, t=30):
|
||||
stdin, stdout, stderr = c.exec_command(f"bash -lc '{cmd}'", timeout=t)
|
||||
return stdout.read().decode(errors='replace').strip()
|
||||
|
||||
# Verify patches survived reboot
|
||||
p = sh("grep -c SHARED ~/ComfyUI/comfy/model_management.py")
|
||||
print(f"SHARED refs in code: {p}")
|
||||
|
||||
# Start ComfyUI
|
||||
sh("rm -f /tmp/comfyui.log")
|
||||
sh("nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &")
|
||||
time.sleep(4)
|
||||
pid = sh("pgrep -f main.py")
|
||||
print(f"PID: {pid}")
|
||||
|
||||
# Wait for HTTP ready
|
||||
print("Waiting for HTTP...", end='', flush=True)
|
||||
for i in range(90):
|
||||
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null')
|
||||
if '200' in code:
|
||||
print(f" ready ({i*2}s)")
|
||||
break
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
else:
|
||||
print(" TIMEOUT")
|
||||
print(sh("tail -30 /tmp/comfyui.log"))
|
||||
sys.exit(1)
|
||||
|
||||
# Show startup state
|
||||
log = sh("cat /tmp/comfyui.log")
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if any(x in s for x in ['vram state', 'SHARED', 'Device:', 'Total VRAM']):
|
||||
print(f" {s}")
|
||||
|
||||
# Submit workflow
|
||||
wf = json.dumps({"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}})
|
||||
# Write workflow and submit
|
||||
sh(f"echo '{wf}' > /tmp/wf.json")
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f"Submitted: {resp[:100]}")
|
||||
|
||||
# Monitor
|
||||
t0 = time.time()
|
||||
shown = set()
|
||||
for _ in range(200):
|
||||
el = int(time.time() - t0)
|
||||
log = sh("cat /tmp/comfyui.log 2>/dev/null")
|
||||
|
||||
for l in log.split('\n'):
|
||||
s = l.strip()
|
||||
if s not in shown and any(x in s for x in ['/8', 'loaded completely', 'load device', 'Requested to load', 'Prompt executed', 'Error', 'OOM']):
|
||||
if 'FETCH' not in s and 'audio' not in s:
|
||||
gpu = sh("cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null")
|
||||
print(f" [{el:>3}s] GPU:{gpu}% | {s[-110:]}")
|
||||
shown.add(s)
|
||||
|
||||
if 'Prompt executed' in log:
|
||||
print(f"\n*** DONE in {el}s! ***")
|
||||
imgs = sh("ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null")
|
||||
print(f" Images: {imgs}")
|
||||
break
|
||||
|
||||
alive = sh("pgrep -c -f main.py 2>/dev/null")
|
||||
if alive == '0':
|
||||
print(f"\nCRASHED at {el}s!")
|
||||
for l in log.split('\n')[-15:]:
|
||||
if l.strip(): print(f" {l.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
c.close()
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Patch ComfyUI for BC-250 APU: Use SHARED VRAM mode + force fp16 VAE.
|
||||
This is the correct mode for an APU where CPU and GPU share the same physical memory."""
|
||||
import paramiko, time, json, textwrap
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# ======================================
|
||||
# 1) Kill
|
||||
# ======================================
|
||||
print("1) Kill ComfyUI")
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
|
||||
# ======================================
|
||||
# 2) Backup + Patch model_management.py
|
||||
# ======================================
|
||||
print("2) Patch model_management.py: SHARED mode for APU")
|
||||
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
code = f.read().decode()
|
||||
|
||||
# Backup
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py.bak', 'w') as f:
|
||||
f.write(code)
|
||||
print(" Backup saved")
|
||||
|
||||
# PATCH 1: After MPS sets SHARED, also set SHARED for this AMD APU
|
||||
# Current code (L458-462):
|
||||
# if cpu_state != CPUState.GPU:
|
||||
# vram_state = VRAMState.DISABLED
|
||||
# if cpu_state == CPUState.MPS:
|
||||
# vram_state = VRAMState.SHARED
|
||||
#
|
||||
# We add: if the GPU has shared memory (small dedicated VRAM), set SHARED
|
||||
|
||||
old_block = '''if cpu_state == CPUState.MPS:
|
||||
vram_state = VRAMState.SHARED
|
||||
|
||||
logging.info(f"Set vram state to: {vram_state.name}")'''
|
||||
|
||||
new_block = '''if cpu_state == CPUState.MPS:
|
||||
vram_state = VRAMState.SHARED
|
||||
|
||||
# BC-250 APU: shared memory between CPU and GPU. Dedicated VRAM is tiny (512MB)
|
||||
# but the full system RAM is accessible to both. SHARED mode loads models
|
||||
# directly on GPU (zero-copy for shared memory APUs).
|
||||
if cpu_state == CPUState.GPU and vram_state not in (VRAMState.DISABLED, VRAMState.SHARED):
|
||||
try:
|
||||
import os
|
||||
if os.environ.get("COMFYUI_SHARED_MEMORY") == "1":
|
||||
vram_state = VRAMState.SHARED
|
||||
logging.info("Forcing SHARED vram state (COMFYUI_SHARED_MEMORY=1)")
|
||||
except:
|
||||
pass
|
||||
|
||||
logging.info(f"Set vram state to: {vram_state.name}")'''
|
||||
|
||||
if old_block in code:
|
||||
code = code.replace(old_block, new_block)
|
||||
print(" PATCH 1 applied: COMFYUI_SHARED_MEMORY env var support")
|
||||
else:
|
||||
print(" PATCH 1: Could not find exact block, trying alternate...")
|
||||
# Try line by line
|
||||
lines = code.split('\n')
|
||||
for i, line in enumerate(lines):
|
||||
if 'cpu_state == CPUState.MPS' in line and 'SHARED' in lines[i+1] if i+1 < len(lines) else '':
|
||||
# Insert after the MPS block
|
||||
insert_idx = i + 2 # After "vram_state = VRAMState.SHARED"
|
||||
patch_lines = [
|
||||
'',
|
||||
'# BC-250 APU shared memory support',
|
||||
'if cpu_state == CPUState.GPU and vram_state not in (VRAMState.DISABLED, VRAMState.SHARED):',
|
||||
' try:',
|
||||
' import os',
|
||||
' if os.environ.get("COMFYUI_SHARED_MEMORY") == "1":',
|
||||
' vram_state = VRAMState.SHARED',
|
||||
' logging.info("Forcing SHARED vram state (COMFYUI_SHARED_MEMORY=1)")',
|
||||
' except:',
|
||||
' pass',
|
||||
]
|
||||
for j, pl in enumerate(patch_lines):
|
||||
lines.insert(insert_idx + j, pl)
|
||||
code = '\n'.join(lines)
|
||||
print(f" PATCH 1 applied (alternate) at line {insert_idx}")
|
||||
break
|
||||
|
||||
# Write patched file
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
|
||||
f.write(code)
|
||||
print(" File written")
|
||||
|
||||
# Verify patch
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
verify = f.read().decode()
|
||||
if 'COMFYUI_SHARED_MEMORY' in verify:
|
||||
print(" Patch verified!")
|
||||
else:
|
||||
print(" ERROR: Patch not found in file!")
|
||||
|
||||
# ======================================
|
||||
# 3) Write launcher with SHARED mode
|
||||
# ======================================
|
||||
print("3) Write launcher with COMFYUI_SHARED_MEMORY=1")
|
||||
launcher = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
# GPU
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
# Threading
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
# MIOpen
|
||||
export MIOPEN_FIND_MODE=3
|
||||
# Shared memory APU mode: CPU and GPU share the same physical RAM
|
||||
export COMFYUI_SHARED_MEMORY=1
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
# --force-fp16: half precision (saves memory)
|
||||
# --fp16-vae: VAE in fp16 (320MB instead of 640MB, fits in GPU memory)
|
||||
# SHARED mode: models load directly on GPU, no offloading overhead
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--force-fp16 \\
|
||||
--fp16-vae
|
||||
""")
|
||||
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
|
||||
f.write(launcher)
|
||||
sh('chmod +x /tmp/run_comfyui.sh')
|
||||
print(" Flags: --force-fp16 --fp16-vae + COMFYUI_SHARED_MEMORY=1")
|
||||
|
||||
# ======================================
|
||||
# 4) Start
|
||||
# ======================================
|
||||
print("4) Start ComfyUI")
|
||||
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
print(f" PID: {pid}")
|
||||
|
||||
# ======================================
|
||||
# 5) Wait ready
|
||||
# ======================================
|
||||
print("5) Wait HTTP", end='', flush=True)
|
||||
for i in range(90):
|
||||
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if '200' in code:
|
||||
print(f" OK ({i*2}s)")
|
||||
break
|
||||
if i % 10 == 0 and i > 0:
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
ls = [l.strip() for l in log.split('\n') if l.strip() and 'FETCH' not in l and 'DEPRECATION' not in l]
|
||||
print(f"\n [{i*2}s] {ls[-1][:80] if ls else ''}", end='', flush=True)
|
||||
except: pass
|
||||
else:
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
|
||||
# Verify SHARED mode
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['vram state', 'SHARED', 'Device:', 'Total VRAM', 'pytorch version']):
|
||||
print(f" {s}")
|
||||
|
||||
# ======================================
|
||||
# 6) Submit
|
||||
# ======================================
|
||||
print("6) Submit")
|
||||
wf = {"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 999, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(wf))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f" {resp[:150]}")
|
||||
|
||||
# ======================================
|
||||
# 7) Monitor
|
||||
# ======================================
|
||||
print("7) Monitor (SHARED mode = everything on GPU)")
|
||||
t0 = time.time()
|
||||
for i in range(200):
|
||||
el = int(time.time() - t0)
|
||||
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
tc = int(temp)//1000 if temp.isdigit() else '?'
|
||||
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
samp = ''
|
||||
last = ''
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if '/8' in s and ('it/s' in s or 's/it' in s): samp = s
|
||||
if s and 'FETCH' not in s and 'startup' not in s and 'DEPRECATION' not in s: last = s
|
||||
|
||||
print(f" [{el:>4}s] {tc}C | {(samp or last)[-90:]}")
|
||||
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
et = ''
|
||||
for line in log.split('\n'):
|
||||
if 'Prompt executed' in line: et = line.strip()
|
||||
print(f"\n *** DONE! ***")
|
||||
print(f" File: {imgs}")
|
||||
print(f" {et}")
|
||||
print(f" Wall: {el}s")
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['loaded', '/8', 'Prompt executed', 'VAE load', 'Requested']):
|
||||
if 'FETCH' not in s:
|
||||
print(f" {s}")
|
||||
break
|
||||
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and el > 30:
|
||||
time.sleep(2)
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** DONE: {imgs} ***")
|
||||
else:
|
||||
print(f"\n Queue empty, no image. Log:")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip() and 'FETCH' not in line: print(f" {line.strip()}")
|
||||
break
|
||||
except: pass
|
||||
|
||||
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
|
||||
print("\n CRASHED!")
|
||||
for line in log.split('\n')[-30:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Quick: check current state, patch for SHARED mode, restart."""
|
||||
import paramiko, time, json, textwrap
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# Check current cmdline
|
||||
print("=== Current process ===")
|
||||
print(sh('ps aux | grep main.py | grep -v grep'))
|
||||
|
||||
# Check if patch exists
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
code = f.read().decode()
|
||||
print(f"\n=== Patch status: {'APPLIED' if 'COMFYUI_SHARED_MEMORY' in code else 'NOT applied'} ===")
|
||||
|
||||
# Check current vram state in log
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if 'vram state' in s or 'VAE load' in s or 'Device:' in s:
|
||||
print(f" {s}")
|
||||
except: pass
|
||||
|
||||
# =============================
|
||||
# APPLY PATCH if not done
|
||||
# =============================
|
||||
if 'COMFYUI_SHARED_MEMORY' not in code:
|
||||
print("\nApplying SHARED memory patch...")
|
||||
# Backup
|
||||
sh('cp /home/fabian/ComfyUI/comfy/model_management.py /home/fabian/ComfyUI/comfy/model_management.py.bak')
|
||||
|
||||
old = 'if cpu_state == CPUState.MPS:\n vram_state = VRAMState.SHARED'
|
||||
new = '''if cpu_state == CPUState.MPS:
|
||||
vram_state = VRAMState.SHARED
|
||||
|
||||
# Shared memory APU: CPU+GPU share physical RAM (e.g. AMD BC-250)
|
||||
import os as _os
|
||||
if _os.environ.get("COMFYUI_SHARED_MEMORY") == "1" and cpu_state == CPUState.GPU:
|
||||
vram_state = VRAMState.SHARED
|
||||
logging.info("SHARED vram: APU shared memory mode enabled")'''
|
||||
|
||||
if old in code:
|
||||
code = code.replace(old, new)
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
|
||||
f.write(code)
|
||||
print(" Patch applied!")
|
||||
else:
|
||||
print(" ERROR: Could not find patch target. Dumping area:")
|
||||
for i, line in enumerate(code.split('\n')):
|
||||
if 'MPS' in line and 'SHARED' in code.split('\n')[i+1] if i+1 < len(code.split('\n')) else '':
|
||||
for j in range(max(0,i-3), min(i+5, len(code.split('\n')))):
|
||||
print(f" {j+1}: {code.split(chr(10))[j]}")
|
||||
else:
|
||||
print(" Patch already applied, good.")
|
||||
|
||||
# =============================
|
||||
# KILL + RESTART with SHARED
|
||||
# =============================
|
||||
print("\nKilling ComfyUI...")
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
|
||||
launcher = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export MIOPEN_FIND_MODE=3
|
||||
# APU shared memory mode: everything on GPU
|
||||
export COMFYUI_SHARED_MEMORY=1
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
# SHARED mode: models load on GPU directly (shared memory = zero copy)
|
||||
# --force-fp16: half precision for models
|
||||
# --fp16-vae: VAE in fp16 (160MB, fast on GPU)
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--force-fp16 \\
|
||||
--fp16-vae
|
||||
""")
|
||||
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
|
||||
f.write(launcher)
|
||||
sh('chmod +x /tmp/run_comfyui.sh')
|
||||
|
||||
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
print(f"Started PID: {pid}")
|
||||
|
||||
# Wait for ready
|
||||
print("Waiting for HTTP", end='', flush=True)
|
||||
for i in range(90):
|
||||
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if '200' in r:
|
||||
print(f" OK ({i*2}s)")
|
||||
break
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
|
||||
# Verify SHARED mode
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['vram state', 'SHARED', 'Device:', 'Total VRAM']):
|
||||
print(f" {s}")
|
||||
|
||||
# Submit test
|
||||
print("\nSubmitting test workflow...")
|
||||
wf = {"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
|
||||
"latent_image": ["5", 0], "seed": 12345, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
|
||||
}}
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(wf))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
|
||||
print(f" {resp[:120]}")
|
||||
|
||||
# Monitor
|
||||
print("\nMonitoring (SHARED = VAE on GPU, everything on GPU)...")
|
||||
t0 = time.time()
|
||||
for i in range(200):
|
||||
el = int(time.time() - t0)
|
||||
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
tc = int(temp)//1000 if temp.isdigit() else '?'
|
||||
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except: log = ''
|
||||
|
||||
samp = ''
|
||||
last = ''
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if '/8' in s and ('it/s' in s or 's/it' in s): samp = s
|
||||
if s and 'FETCH' not in s and 'startup' not in s and 'DEPRECATION' not in s: last = s
|
||||
|
||||
print(f" [{el:>4}s] {tc}C | {(samp or last)[-90:]}")
|
||||
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
et = ''
|
||||
for line in log.split('\n'):
|
||||
if 'Prompt executed' in line: et = line.strip()
|
||||
print(f"\n *** DONE! *** {imgs}")
|
||||
print(f" {et}")
|
||||
print(f" Wall: {el}s")
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if any(k in s for k in ['loaded', '/8', 'Prompt executed', 'VAE load', 'Requested']):
|
||||
if 'FETCH' not in s: print(f" {s}")
|
||||
break
|
||||
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and el > 30:
|
||||
time.sleep(2)
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n *** DONE: {imgs} ***")
|
||||
else:
|
||||
print(f"\n Queue empty, no image:")
|
||||
for line in log.split('\n')[-20:]:
|
||||
s = line.strip()
|
||||
if s and 'FETCH' not in s: print(f" {s}")
|
||||
break
|
||||
except: pass
|
||||
|
||||
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
|
||||
print("\n CRASHED!")
|
||||
for line in log.split('\n')[-30:]:
|
||||
if line.strip(): print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Upload a self-contained bash script and run it in ONE SSH session."""
|
||||
import paramiko, time
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
# Write the entire fix+restart+test as ONE bash script
|
||||
script = r'''#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Kill any running ComfyUI
|
||||
pkill -9 -f "python3.*main.py" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Clean
|
||||
rm -f /tmp/comfyui.log
|
||||
rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png
|
||||
|
||||
# Start ComfyUI
|
||||
echo "Starting ComfyUI..."
|
||||
nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &
|
||||
sleep 3
|
||||
PID=$(pgrep -f "python3.*main.py" | head -1)
|
||||
echo "PID: $PID"
|
||||
|
||||
if [ -z "$PID" ]; then
|
||||
echo "FAILED TO START!"
|
||||
cat /tmp/comfyui.log 2>/dev/null | tail -20
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for HTTP
|
||||
echo "Waiting for HTTP..."
|
||||
for i in $(seq 1 90); do
|
||||
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null || echo 000)
|
||||
if [ "$CODE" = "200" ]; then
|
||||
echo "Ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Show key startup info
|
||||
grep -E "vram state|SHARED|Device:|Total VRAM|load device|offload" /tmp/comfyui.log 2>/dev/null || true
|
||||
|
||||
# Submit workflow
|
||||
echo ""
|
||||
echo "Submitting workflow..."
|
||||
cat > /tmp/wf.json << 'WFEOF'
|
||||
{"prompt":{"1":{"class_type":"UnetLoaderGGUF","inputs":{"unet_name":"z_image_turbo-Q5_K_S.gguf"}},"2":{"class_type":"CLIPLoaderGGUF","inputs":{"clip_name":"Qwen3-4B.i1-Q5_K_S.gguf","type":"qwen_image"}},"3":{"class_type":"VAELoader","inputs":{"vae_name":"ae.safetensors"}},"4":{"class_type":"CLIPTextEncode","inputs":{"text":"A red fox in a snowy forest, photorealistic","clip":["2",0]}},"5":{"class_type":"EmptyLatentImage","inputs":{"width":512,"height":512,"batch_size":1}},"6":{"class_type":"KSampler","inputs":{"model":["1",0],"positive":["4",0],"negative":["4",0],"latent_image":["5",0],"seed":99999,"steps":8,"cfg":1.0,"sampler_name":"euler","scheduler":"simple","denoise":1.0}},"7":{"class_type":"VAEDecode","inputs":{"samples":["6",0],"vae":["3",0]}},"8":{"class_type":"SaveImage","inputs":{"images":["7",0],"filename_prefix":"ZImageTurbo_GPU"}}}}
|
||||
WFEOF
|
||||
|
||||
RESP=$(curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json)
|
||||
echo "Response: ${RESP:0:120}"
|
||||
|
||||
# Monitor
|
||||
echo ""
|
||||
echo "Monitoring..."
|
||||
START=$(date +%s)
|
||||
LAST=""
|
||||
while true; do
|
||||
NOW=$(date +%s)
|
||||
ELAPSED=$((NOW - START))
|
||||
|
||||
if [ $ELAPSED -gt 600 ]; then
|
||||
echo "TIMEOUT after 600s"
|
||||
tail -20 /tmp/comfyui.log
|
||||
break
|
||||
fi
|
||||
|
||||
# Check for output image
|
||||
if ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null; then
|
||||
echo ""
|
||||
echo "*** IMAGE DONE in ${ELAPSED}s! ***"
|
||||
grep -E "loaded|load device|offload|Prompt executed|/8" /tmp/comfyui.log 2>/dev/null | grep -v FETCH || true
|
||||
break
|
||||
fi
|
||||
|
||||
# Show progress
|
||||
LINE=$(grep -E "/8|loaded|Requested|VAE|Prompt executed|Error|OOM" /tmp/comfyui.log 2>/dev/null | grep -v FETCH | grep -v audio_vae | grep -v "split attention" | tail -1)
|
||||
GPU=$(cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null || echo "?")
|
||||
|
||||
if [ "$LINE" != "$LAST" ] && [ -n "$LINE" ]; then
|
||||
echo "[${ELAPSED}s] GPU:${GPU}% ${LINE:0:110}"
|
||||
LAST="$LINE"
|
||||
fi
|
||||
|
||||
# Check alive
|
||||
if ! pgrep -f "python3.*main.py" > /dev/null 2>&1; then
|
||||
echo "CRASHED at ${ELAPSED}s!"
|
||||
tail -20 /tmp/comfyui.log
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 3
|
||||
done
|
||||
'''
|
||||
|
||||
with sftp.open('/tmp/fix_and_run.sh', 'w') as f:
|
||||
f.write(script)
|
||||
sftp.close()
|
||||
|
||||
# Execute in ONE session
|
||||
print("Running fix+restart+monitor on BC-250...")
|
||||
stdin, stdout, stderr = c.exec_command('bash /tmp/fix_and_run.sh', timeout=660)
|
||||
# Stream output
|
||||
for line in iter(stdout.readline, ''):
|
||||
print(line.rstrip())
|
||||
err = stderr.read().decode(errors='replace').strip()
|
||||
if err:
|
||||
for l in err.split('\n')[-10:]:
|
||||
if l.strip(): print(f"STDERR: {l.strip()}")
|
||||
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Quick status: is ComfyUI still running and what's the log say?"""
|
||||
import paramiko
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
|
||||
sftp = c.open_sftp()
|
||||
|
||||
# Read full log
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
|
||||
sftp.close()
|
||||
|
||||
lines = log.split('\n')
|
||||
print(f"Total lines: {len(lines)}")
|
||||
print()
|
||||
|
||||
# Show only meaningful lines
|
||||
for line in lines:
|
||||
s = line.strip()
|
||||
if not s:
|
||||
continue
|
||||
if 'FETCH ComfyRegistry' in s or 'All startup tasks' in s or 'FETCH DATA' in s:
|
||||
continue
|
||||
print(s)
|
||||
|
||||
# Check process + GPU
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(10)
|
||||
chan.exec_command('/bin/bash -c "echo; echo === PROCESS ===; ps aux | grep python3 | grep -v grep; echo; echo === GPU ===; rocm-smi 2>/dev/null | head -12; echo; echo === OUTPUT ===; ls -la ~/ComfyUI/output/ 2>/dev/null; echo; echo === QUEUE ===; curl -s http://127.0.0.1:8188/queue 2>/dev/null"')
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
print(out.decode(errors='replace'))
|
||||
|
||||
c.close()
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Start PyTorch build on BC-250 properly using SFTP for the script."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 60:
|
||||
print(f" ... ({len(lines)} lines, showing last 60)")
|
||||
print('\n'.join(lines[-60:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Upload the build script via SFTP
|
||||
build_script = '''#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
LOG="/home/fabian/pytorch_build.log"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
echo "=========================================="
|
||||
echo " PyTorch Build for ROCm gfx1010 (BC-250)"
|
||||
echo " Started: $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
|
||||
cd /home/fabian/pytorch
|
||||
|
||||
# ROCm build configuration
|
||||
export USE_ROCM=1
|
||||
export USE_CUDA=0
|
||||
export PYTORCH_ROCM_ARCH="gfx1010"
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export ROCM_PATH=/opt/rocm
|
||||
export HIP_PATH=/opt/rocm
|
||||
export CMAKE_PREFIX_PATH="/opt/rocm;$(python3 -c 'import sys; print(sys.prefix)')"
|
||||
export PATH=/opt/rocm/bin:$PATH
|
||||
|
||||
# Build settings
|
||||
export USE_NINJA=1
|
||||
export CMAKE_GENERATOR=Ninja
|
||||
export MAX_JOBS=6
|
||||
export USE_CCACHE=1
|
||||
export CCACHE_DIR=/home/fabian/.ccache
|
||||
|
||||
# Disable unnecessary components for faster build
|
||||
export USE_FBGEMM=0
|
||||
export USE_KINETO=0
|
||||
export USE_CUPTI_SO=0
|
||||
export USE_NCCL=0
|
||||
export USE_DISTRIBUTED=0
|
||||
export USE_TENSORPIPE=0
|
||||
export USE_GLOO=0
|
||||
export USE_MPI=0
|
||||
export USE_OPENMP=1
|
||||
export USE_MKLDNN=1
|
||||
export BUILD_TEST=0
|
||||
export USE_CUDNN=0
|
||||
|
||||
echo ""
|
||||
echo "Build config:"
|
||||
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
|
||||
echo " USE_ROCM=$USE_ROCM"
|
||||
echo " MAX_JOBS=$MAX_JOBS"
|
||||
echo " Python: $(python3 --version)"
|
||||
echo " hipcc: $(hipcc --version 2>&1 | head -1)"
|
||||
echo " ROCm: $(cat /opt/rocm/.info/version)"
|
||||
echo ""
|
||||
|
||||
# Install requirements
|
||||
echo "Installing PyTorch requirements..."
|
||||
pip install -r requirements.txt 2>&1 | tail -10
|
||||
echo ""
|
||||
|
||||
# Clean any partial build
|
||||
echo "Cleaning previous build artifacts..."
|
||||
python3 setup.py clean 2>&1 || true
|
||||
echo ""
|
||||
|
||||
# Build the wheel
|
||||
echo "Starting PyTorch build..."
|
||||
echo "=========================================="
|
||||
python3 setup.py bdist_wheel 2>&1
|
||||
|
||||
BUILD_RC=$?
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " Build exit code: $BUILD_RC"
|
||||
echo " Finished: $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
if [ $BUILD_RC -eq 0 ]; then
|
||||
echo ""
|
||||
echo "Wheel files:"
|
||||
ls -lh dist/*.whl 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "Installing wheel..."
|
||||
pip install dist/*.whl 2>&1
|
||||
|
||||
echo ""
|
||||
echo "=== VERIFICATION ==="
|
||||
python3 -c "
|
||||
import torch
|
||||
print(f'PyTorch version: {torch.__version__}')
|
||||
print(f'HIP version: {torch.version.hip}')
|
||||
print(f'CUDA available (HIP): {torch.cuda.is_available()}')
|
||||
if torch.cuda.is_available():
|
||||
print(f'Device name: {torch.cuda.get_device_name(0)}')
|
||||
print(f'Device count: {torch.cuda.device_count()}')
|
||||
t = torch.randn(4, 4, device=\"cuda\")
|
||||
print(f'Tensor on GPU: {t.device}')
|
||||
print(f'Tensor sum: {t.sum().item():.4f}')
|
||||
print('GPU COMPUTE: WORKING')
|
||||
else:
|
||||
print('WARNING: CUDA/HIP not available')
|
||||
" 2>&1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "BUILD_COMPLETE_RC=$BUILD_RC"
|
||||
'''
|
||||
|
||||
print("Uploading build script via SFTP...")
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/build_pytorch.sh', 'w') as f:
|
||||
f.write(build_script)
|
||||
sftp.close()
|
||||
|
||||
run("chmod +x /home/fabian/build_pytorch.sh", desc="Make executable")
|
||||
|
||||
# Remove old log if it exists
|
||||
run("rm -f /home/fabian/pytorch_build.log", desc="Clean old log")
|
||||
|
||||
# Start the build using nohup inside bash (not fish)
|
||||
# Using bash explicitly to avoid fish issues with nohup
|
||||
run("bash -c 'nohup bash /home/fabian/build_pytorch.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
|
||||
desc="Start build in background")
|
||||
|
||||
# Wait for it to actually start
|
||||
time.sleep(10)
|
||||
|
||||
# Verify it's running
|
||||
run("pgrep -fa 'build_pytorch\\|setup.py' | head -10",
|
||||
desc="Verify build is running")
|
||||
|
||||
# Check initial log
|
||||
time.sleep(5)
|
||||
run("cat /home/fabian/pytorch_build.log 2>/dev/null | head -30 || echo 'Log not yet available'",
|
||||
desc="Initial build log")
|
||||
|
||||
# Monitor for first compile steps
|
||||
time.sleep(30)
|
||||
run("tail -30 /home/fabian/pytorch_build.log 2>/dev/null || echo 'Waiting for log...'",
|
||||
desc="Build progress after 30 seconds")
|
||||
|
||||
ssh.close()
|
||||
print("\n" + "="*60)
|
||||
print(" PyTorch build running on BC-250!")
|
||||
print(" Monitor: tail -f ~/pytorch_build.log")
|
||||
print("="*60)
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quick single-connection status check."""
|
||||
import paramiko, json
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian',
|
||||
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
|
||||
|
||||
def run(cmd):
|
||||
_, so, se = ssh.exec_command(cmd, timeout=15)
|
||||
return so.read().decode()
|
||||
|
||||
try:
|
||||
# Is ComfyUI running?
|
||||
print("=== PROCESS ===")
|
||||
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" ps -p $PID -o pid,%cpu,%mem,nlwp,etime --no-headers; "
|
||||
"else echo NOT_RUNNING; fi'").strip())
|
||||
|
||||
# Log
|
||||
print("\n=== LOG (last 20) ===")
|
||||
print(run("tail -20 /home/fabian/comfyui.log 2>/dev/null").strip())
|
||||
|
||||
# GPU
|
||||
print("\n=== GPU ===")
|
||||
gpu = run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null | grep -E '0x|GPU%'")
|
||||
print(gpu.strip() if gpu.strip() else "no output")
|
||||
|
||||
# Output files
|
||||
print("\n=== OUTPUT ===")
|
||||
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null").strip())
|
||||
|
||||
# Queue
|
||||
print("\n=== QUEUE ===")
|
||||
q = run("curl -s http://localhost:8188/queue 2>/dev/null")
|
||||
if q.strip():
|
||||
qj = json.loads(q)
|
||||
print(f"Running: {len(qj.get('queue_running',[]))}, Pending: {len(qj.get('queue_pending',[]))}")
|
||||
else:
|
||||
print("Server not responding")
|
||||
finally:
|
||||
ssh.close()
|
||||
print("\nSSH closed.")
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Step 1: Install build dependencies on BC-250 for PyTorch build."""
|
||||
import paramiko
|
||||
import sys
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
print(out.strip()[-2000:]) # Last 2000 chars
|
||||
if err.strip():
|
||||
# Filter out common noise
|
||||
lines = [l for l in err.strip().split('\n') if not l.startswith('warning:')]
|
||||
if lines:
|
||||
print(f"STDERR: {chr(10).join(lines[-20:])}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Install build tools
|
||||
run("sudo pacman -S --needed --noconfirm python-pip ninja ccache",
|
||||
desc="Install pip, ninja, ccache")
|
||||
|
||||
# Install PyTorch build dependencies
|
||||
run("sudo pacman -S --needed --noconfirm cmake blas lapack openblas "
|
||||
"python-numpy python-pyyaml python-typing_extensions "
|
||||
"intel-oneapi-mkl 2>/dev/null; echo done",
|
||||
desc="Install build dependencies (cmake, blas, numpy, etc.)")
|
||||
|
||||
# Install additional deps that PyTorch needs
|
||||
run("sudo pacman -S --needed --noconfirm python-cffi python-setuptools "
|
||||
"python-wheel python-filelock python-sympy python-networkx",
|
||||
desc="Install Python dependencies")
|
||||
|
||||
# Verify installs
|
||||
run("pip --version && ninja --version && ccache --version | head -1 && cmake --version | head -1",
|
||||
desc="Verify installations")
|
||||
|
||||
# Check pip can install packages
|
||||
run("pip install --user --upgrade pip setuptools wheel 2>&1 | tail -5",
|
||||
desc="Upgrade pip/setuptools")
|
||||
|
||||
ssh.close()
|
||||
print("\n\nDone — build dependencies installed.")
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Step 1: Kill stuck ComfyUI and check flags."""
|
||||
import paramiko
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=60):
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
print(out.strip() if out.strip() else "")
|
||||
if err.strip():
|
||||
for l in err.strip().split('\n')[-5:]:
|
||||
print(f"STDERR: {l}")
|
||||
|
||||
print("=== Kill stuck ===")
|
||||
run("pkill -f 'python3 main.py' 2>/dev/null; sleep 2; pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1; echo killed")
|
||||
|
||||
print("\n=== Check flags ===")
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && python3 main.py --help 2>&1 | grep -i -E \"vae|fp16|fp32|force|cpu|novram|lowvram\"'")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Step 2: Create venv and clone PyTorch on BC-250."""
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=600, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
# Print last portion for long outputs
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 50:
|
||||
print(f" ... ({len(lines)} lines, showing last 50)")
|
||||
print('\n'.join(lines[-50:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = [l for l in err.strip().split('\n') if 'warning:' not in l.lower()]
|
||||
if lines:
|
||||
if len(lines) > 30:
|
||||
print(f"STDERR ({len(lines)} lines, last 30):")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(f"STDERR: {chr(10).join(lines)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Create the venv
|
||||
run("python3 -m venv ~/comfyui-env --system-site-packages",
|
||||
desc="Create venv with system site-packages (for numpy, etc.)")
|
||||
|
||||
# Activate and install basic build deps in venv
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && pip install --upgrade pip setuptools wheel'",
|
||||
desc="Upgrade pip in venv")
|
||||
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && pip install cmake ninja pyyaml typing-extensions cffi future six requests dataclasses filelock sympy networkx jinja2 numpy'",
|
||||
desc="Install PyTorch build deps in venv", timeout=120)
|
||||
|
||||
# Check if PyTorch source already exists
|
||||
rc, out, _ = run("test -d ~/pytorch && echo EXISTS || echo MISSING",
|
||||
desc="Check for existing PyTorch source")
|
||||
|
||||
if "EXISTS" in out:
|
||||
print("\n PyTorch source directory exists. Checking if it's a valid repo...")
|
||||
run("cd ~/pytorch && git log --oneline -1 2>&1", desc="Check PyTorch repo")
|
||||
else:
|
||||
# Clone PyTorch — this is the big download
|
||||
print("\n Cloning PyTorch (this will take a while)...")
|
||||
run("git clone --depth 1 --recursive --shallow-submodules https://github.com/pytorch/pytorch.git ~/pytorch 2>&1 | tail -20",
|
||||
desc="Clone PyTorch (shallow, with submodules)",
|
||||
timeout=1200) # 20 minutes timeout
|
||||
|
||||
# Verify clone
|
||||
run("ls -la ~/pytorch/setup.py ~/pytorch/torch/ 2>&1 | head -5",
|
||||
desc="Verify PyTorch source")
|
||||
|
||||
run("cd ~/pytorch && git log --oneline -1",
|
||||
desc="PyTorch version")
|
||||
|
||||
run("du -sh ~/pytorch",
|
||||
desc="PyTorch source size")
|
||||
|
||||
ssh.close()
|
||||
print("\n\nDone — venv created and PyTorch cloned.")
|
||||
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Step 3: Build PyTorch from source for ROCm gfx1010 on BC-250.
|
||||
|
||||
This build will take a long time (1-3 hours on 12 cores).
|
||||
We run it non-interactively via nohup so it survives SSH disconnects.
|
||||
"""
|
||||
import paramiko
|
||||
import sys
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=600, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 50:
|
||||
print(f" ... ({len(lines)} lines, showing last 50)")
|
||||
print('\n'.join(lines[-50:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = [l for l in err.strip().split('\n')]
|
||||
if lines:
|
||||
show = lines[-30:] if len(lines) > 30 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# First, create the build script on the BC-250
|
||||
build_script = r'''#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
LOG="/home/fabian/pytorch_build.log"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
echo "=========================================="
|
||||
echo " PyTorch Build for ROCm gfx1010 (BC-250)"
|
||||
echo " Started: $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
# Activate venv
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
|
||||
# Go to PyTorch source
|
||||
cd /home/fabian/pytorch
|
||||
|
||||
# Set environment for ROCm build
|
||||
export USE_ROCM=1
|
||||
export USE_CUDA=0
|
||||
export PYTORCH_ROCM_ARCH="gfx1010"
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export ROCM_PATH=/opt/rocm
|
||||
export HIP_PATH=/opt/rocm
|
||||
export CMAKE_PREFIX_PATH=/opt/rocm
|
||||
export PATH=/opt/rocm/bin:$PATH
|
||||
|
||||
# Use ninja for faster builds
|
||||
export USE_NINJA=1
|
||||
export CMAKE_GENERATOR=Ninja
|
||||
|
||||
# Limit parallel jobs to avoid OOM (14GB RAM + 14GB swap)
|
||||
# Each compilation unit can use ~1-2GB during link, so limit to 6 jobs
|
||||
export MAX_JOBS=6
|
||||
|
||||
# Use ccache to speed up rebuilds
|
||||
export USE_CCACHE=1
|
||||
export CCACHE_DIR=/home/fabian/.ccache
|
||||
|
||||
# Disable unnecessary components to speed up build
|
||||
export USE_FBGEMM=0
|
||||
export USE_KINETO=0
|
||||
export USE_CUPTI_SO=0
|
||||
export USE_NCCL=0
|
||||
export USE_DISTRIBUTED=0
|
||||
export USE_TENSORPIPE=0
|
||||
export USE_GLOO=0
|
||||
export USE_MPI=0
|
||||
export USE_OPENMP=1
|
||||
export USE_MKLDNN=1
|
||||
export BUILD_TEST=0
|
||||
|
||||
# Disable CUDA-specific stuff
|
||||
export USE_CUDNN=0
|
||||
|
||||
echo ""
|
||||
echo "Build config:"
|
||||
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
|
||||
echo " USE_ROCM=$USE_ROCM"
|
||||
echo " MAX_JOBS=$MAX_JOBS"
|
||||
echo " USE_CCACHE=$USE_CCACHE"
|
||||
echo " Python: $(python3 --version)"
|
||||
echo " hipcc: $(hipcc --version 2>&1 | head -1)"
|
||||
echo ""
|
||||
|
||||
# Install requirements
|
||||
echo "Installing PyTorch requirements..."
|
||||
pip install -r requirements.txt 2>&1 | tail -5
|
||||
|
||||
# Run the build
|
||||
echo ""
|
||||
echo "Starting PyTorch build... (this will take 1-3 hours)"
|
||||
echo "=========================================="
|
||||
python3 setup.py bdist_wheel 2>&1
|
||||
|
||||
BUILD_RC=$?
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " Build finished with exit code: $BUILD_RC"
|
||||
echo " Time: $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
if [ $BUILD_RC -eq 0 ]; then
|
||||
echo "Wheel file:"
|
||||
ls -lh dist/*.whl 2>/dev/null || echo "No wheel found, trying develop install..."
|
||||
|
||||
# Install the wheel
|
||||
echo "Installing PyTorch wheel..."
|
||||
pip install dist/*.whl 2>&1 | tail -5
|
||||
|
||||
# Verify
|
||||
echo ""
|
||||
echo "Verification:"
|
||||
python3 -c "import torch; print(f'PyTorch {torch.__version__}'); print(f'ROCm: {torch.version.hip}'); print(f'CUDA available: {torch.cuda.is_available()}'); print(f'Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"N/A\"}')"
|
||||
fi
|
||||
|
||||
echo "BUILD_COMPLETE_RC=$BUILD_RC" >> "$LOG"
|
||||
'''
|
||||
|
||||
# Write the build script to BC-250
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/build_pytorch.sh', 'w') as f:
|
||||
f.write(build_script)
|
||||
sftp.close()
|
||||
|
||||
run("chmod +x /home/fabian/build_pytorch.sh", desc="Make build script executable")
|
||||
|
||||
# Check if a build is already running
|
||||
rc, out, _ = run("pgrep -f 'setup.py bdist_wheel' || echo 'NOT RUNNING'",
|
||||
desc="Check if build is already running")
|
||||
|
||||
if 'NOT RUNNING' not in out:
|
||||
print("\n BUILD IS ALREADY RUNNING — not starting a new one.")
|
||||
print(" Monitor with: tail -f ~/pytorch_build.log")
|
||||
else:
|
||||
# Start the build in background using nohup
|
||||
# This way it survives SSH disconnects
|
||||
run("nohup bash /home/fabian/build_pytorch.sh > /dev/null 2>&1 &",
|
||||
desc="Starting PyTorch build in background (nohup)")
|
||||
|
||||
# Give it a moment to start
|
||||
time.sleep(5)
|
||||
|
||||
# Verify it started
|
||||
run("pgrep -fa 'build_pytorch.sh' || pgrep -fa 'setup.py' || echo 'WARNING: Build may have failed to start'",
|
||||
desc="Verify build process started")
|
||||
|
||||
# Check initial log output
|
||||
time.sleep(10)
|
||||
run("tail -30 /home/fabian/pytorch_build.log 2>/dev/null || echo 'Log not yet created'",
|
||||
desc="Initial build log output")
|
||||
|
||||
ssh.close()
|
||||
print("\n" + "="*60)
|
||||
print(" PyTorch build started in background on BC-250!")
|
||||
print(" Monitor: ssh fabian@BC-250 'tail -f ~/pytorch_build.log'")
|
||||
print(" Check status: ssh fabian@BC-250 'pgrep -fa setup.py'")
|
||||
print("="*60)
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install missing ROCm math libraries for PyTorch build on BC-250."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 60:
|
||||
print(f" ... ({len(lines)} lines, showing last 60)")
|
||||
print('\n'.join(lines[-60:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Find all available ROCm packages
|
||||
run("pacman -Ss rocm | grep -E '^cachyos|^extra|^core' | head -40",
|
||||
desc="Available ROCm packages")
|
||||
|
||||
# Install all ROCm math/compute libraries needed by PyTorch
|
||||
run("sudo pacman -S --needed --noconfirm "
|
||||
"hiprand rocrand "
|
||||
"hipblas rocblas "
|
||||
"hipfft rocfft "
|
||||
"hipsparse rocsparse "
|
||||
"hipsolver rocsolver "
|
||||
"miopen-hip "
|
||||
"rocprim hipcub "
|
||||
"rocthrust "
|
||||
"rccl "
|
||||
"hipblaslt "
|
||||
"roctracer "
|
||||
"2>&1 | tail -40",
|
||||
desc="Install ROCm math libraries",
|
||||
timeout=600)
|
||||
|
||||
# Verify hiprand is now available
|
||||
run("find /opt/rocm -name 'hiprandConfig.cmake' -o -name 'hiprand-config.cmake' 2>/dev/null | head -5",
|
||||
desc="Verify hiprand cmake config")
|
||||
|
||||
# Check all libraries
|
||||
run("ls /opt/rocm/lib/libhiprand.so /opt/rocm/lib/librocblas.so /opt/rocm/lib/libhipblas.so /opt/rocm/lib/librocfft.so /opt/rocm/lib/libMIOpen.so 2>&1",
|
||||
desc="Verify key libraries exist")
|
||||
|
||||
# Clean the failed build and restart
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && cd ~/pytorch && python3 setup.py clean 2>&1 | tail -5'",
|
||||
desc="Clean failed build")
|
||||
|
||||
# Restart build
|
||||
run("rm -f /home/fabian/pytorch_build.log", desc="Clean old log")
|
||||
run("bash -c 'nohup bash /home/fabian/build_pytorch.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
|
||||
desc="Restart PyTorch build")
|
||||
|
||||
time.sleep(15)
|
||||
run("pgrep -fa 'setup.py\\|cmake\\|ninja' | head -10",
|
||||
desc="Verify build restarted")
|
||||
|
||||
time.sleep(45)
|
||||
run("tail -40 /home/fabian/pytorch_build.log 2>/dev/null",
|
||||
desc="Build progress")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone — ROCm libs installed and build restarted.")
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install ComfyUI and dependencies on BC-250."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 60:
|
||||
print(f" ... ({len(lines)} lines, showing last 60)")
|
||||
print('\n'.join(lines[-60:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Clone ComfyUI
|
||||
run("git clone --depth 1 https://github.com/comfyanonymous/ComfyUI.git ~/ComfyUI 2>&1 | tail -10",
|
||||
desc="Clone ComfyUI")
|
||||
|
||||
# Install ComfyUI requirements in venv
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && cd ~/ComfyUI && pip install -r requirements.txt 2>&1 | tail -20'",
|
||||
desc="Install ComfyUI requirements",
|
||||
timeout=300)
|
||||
|
||||
# Install diffusers from source (needed for ZImagePipeline)
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && pip install git+https://github.com/huggingface/diffusers 2>&1 | tail -10'",
|
||||
desc="Install diffusers from source (for ZImagePipeline)",
|
||||
timeout=300)
|
||||
|
||||
# Install additional deps that ComfyUI/Z-Image might need
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && pip install transformers accelerate safetensors sentencepiece huggingface_hub aiohttp einops torchvision 2>&1 | tail -15'",
|
||||
desc="Install transformers, accelerate, etc.",
|
||||
timeout=300)
|
||||
|
||||
# Install ComfyUI-Manager (custom node manager)
|
||||
run("git clone --depth 1 https://github.com/Comfy-Org/ComfyUI-Manager.git ~/ComfyUI/custom_nodes/ComfyUI-Manager 2>&1 | tail -5",
|
||||
desc="Install ComfyUI-Manager")
|
||||
|
||||
# Install ComfyUI-GGUF (needed for GGUF checkpoint format)
|
||||
run("git clone --depth 1 https://github.com/city96/ComfyUI-GGUF.git ~/ComfyUI/custom_nodes/ComfyUI-GGUF 2>&1 | tail -5",
|
||||
desc="Install ComfyUI-GGUF nodes")
|
||||
|
||||
# Install GGUF dependencies
|
||||
run("bash -c 'source ~/comfyui-env/bin/activate && pip install gguf 2>&1 | tail -5'",
|
||||
desc="Install gguf Python package")
|
||||
|
||||
# Install Z-Image Power Nodes
|
||||
run("git clone --depth 1 https://github.com/martin-rizzo/ComfyUI-ZImagePowerNodes.git ~/ComfyUI/custom_nodes/ComfyUI-ZImagePowerNodes 2>&1 | tail -5",
|
||||
desc="Install Z-Image Power Nodes")
|
||||
|
||||
# Verify ComfyUI structure
|
||||
run("ls -la ~/ComfyUI/main.py ~/ComfyUI/custom_nodes/ 2>&1",
|
||||
desc="Verify ComfyUI structure")
|
||||
|
||||
run("ls ~/ComfyUI/custom_nodes/",
|
||||
desc="Custom nodes installed")
|
||||
|
||||
# Create model directories
|
||||
run("mkdir -p ~/ComfyUI/models/diffusion_models ~/ComfyUI/models/text_encoders ~/ComfyUI/models/vae ~/ComfyUI/models/checkpoints",
|
||||
desc="Create model directories")
|
||||
|
||||
# Quick test: can ComfyUI import?
|
||||
run("""bash -c 'source ~/comfyui-env/bin/activate && \
|
||||
HSA_OVERRIDE_GFX_VERSION=10.1.0 \
|
||||
HIP_VISIBLE_DEVICES=0 \
|
||||
HSA_ENABLE_SDMA=0 \
|
||||
cd ~/ComfyUI && python3 -c "
|
||||
import torch
|
||||
print(f\\"torch {torch.__version__} hip={torch.version.hip} cuda={torch.cuda.is_available()}\\")
|
||||
import comfy
|
||||
print(\\"ComfyUI import OK\\")
|
||||
" 2>&1'""",
|
||||
desc="Test ComfyUI import",
|
||||
timeout=60)
|
||||
|
||||
ssh.close()
|
||||
print("\nDone — ComfyUI installed.")
|
||||
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download Z-Image-Turbo GGUF model checkpoints on BC-250.
|
||||
|
||||
Files needed (GGUF format — memory-efficient for 14GB RAM):
|
||||
1. z_image_turbo-Q5_K_S.gguf (5.19 GB) → diffusion_models/
|
||||
2. Qwen3-4B.i1-Q5_K_S.gguf (2.82 GB) → text_encoders/
|
||||
3. ae.safetensors (335 MB) → vae/
|
||||
Total: ~8.35 GB
|
||||
"""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=3600, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 30:
|
||||
print(f" ... ({len(lines)} lines, showing last 30)")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-15:] if len(lines) > 15 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Create download script for background execution
|
||||
dl_script = '''#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
LOG="/home/fabian/model_download.log"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
source /home/fabian/comfyui-env/bin/activate
|
||||
|
||||
COMFY="$HOME/ComfyUI"
|
||||
echo "=========================================="
|
||||
echo " Downloading Z-Image-Turbo GGUF Models"
|
||||
echo " Started: $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
# 1. Diffusion model (5.19 GB)
|
||||
echo ""
|
||||
echo "[1/3] Downloading z_image_turbo-Q5_K_S.gguf (5.19 GB)..."
|
||||
if [ -f "$COMFY/models/diffusion_models/z_image_turbo-Q5_K_S.gguf" ]; then
|
||||
echo " Already exists, skipping."
|
||||
else
|
||||
HF_XET_HIGH_PERFORMANCE=1 huggingface-cli download \
|
||||
jayn7/Z-Image-Turbo-GGUF \
|
||||
z_image_turbo-Q5_K_S.gguf \
|
||||
--local-dir "$COMFY/models/diffusion_models/" \
|
||||
--local-dir-use-symlinks False
|
||||
echo " Done."
|
||||
fi
|
||||
|
||||
# 2. Text encoder Qwen3-4B (2.82 GB)
|
||||
echo ""
|
||||
echo "[2/3] Downloading Qwen3-4B.i1-Q5_K_S.gguf (2.82 GB)..."
|
||||
if [ -f "$COMFY/models/text_encoders/Qwen3-4B.i1-Q5_K_S.gguf" ]; then
|
||||
echo " Already exists, skipping."
|
||||
else
|
||||
HF_XET_HIGH_PERFORMANCE=1 huggingface-cli download \
|
||||
mradermacher/Qwen3-4B-i1-GGUF \
|
||||
Qwen3-4B.i1-Q5_K_S.gguf \
|
||||
--local-dir "$COMFY/models/text_encoders/" \
|
||||
--local-dir-use-symlinks False
|
||||
echo " Done."
|
||||
fi
|
||||
|
||||
# 3. VAE (335 MB)
|
||||
echo ""
|
||||
echo "[3/3] Downloading ae.safetensors (VAE, 335 MB)..."
|
||||
if [ -f "$COMFY/models/vae/ae.safetensors" ]; then
|
||||
echo " Already exists, skipping."
|
||||
else
|
||||
HF_XET_HIGH_PERFORMANCE=1 huggingface-cli download \
|
||||
Comfy-Org/z_image_turbo \
|
||||
split_files/vae/ae.safetensors \
|
||||
--local-dir "$COMFY/models/vae/" \
|
||||
--local-dir-use-symlinks False
|
||||
# Move from subdirectory if needed
|
||||
if [ -f "$COMFY/models/vae/split_files/vae/ae.safetensors" ]; then
|
||||
mv "$COMFY/models/vae/split_files/vae/ae.safetensors" "$COMFY/models/vae/ae.safetensors"
|
||||
rm -rf "$COMFY/models/vae/split_files"
|
||||
fi
|
||||
echo " Done."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " Model Download Summary"
|
||||
echo "=========================================="
|
||||
echo "Diffusion model:"
|
||||
ls -lh "$COMFY/models/diffusion_models/"*.gguf 2>/dev/null || echo " NOT FOUND"
|
||||
echo "Text encoder:"
|
||||
ls -lh "$COMFY/models/text_encoders/"*.gguf 2>/dev/null || echo " NOT FOUND"
|
||||
echo "VAE:"
|
||||
ls -lh "$COMFY/models/vae/"*.safetensors 2>/dev/null || echo " NOT FOUND"
|
||||
echo ""
|
||||
echo "Total model size:"
|
||||
du -sh "$COMFY/models/"
|
||||
echo ""
|
||||
echo "DOWNLOAD_COMPLETE"
|
||||
echo "Finished: $(date)"
|
||||
'''
|
||||
|
||||
# Upload download script
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/download_models.sh', 'w') as f:
|
||||
f.write(dl_script)
|
||||
sftp.close()
|
||||
|
||||
run("chmod +x /home/fabian/download_models.sh", desc="Make download script executable")
|
||||
run("rm -f /home/fabian/model_download.log", desc="Clean old log")
|
||||
|
||||
# Start download in background
|
||||
run("bash -c 'nohup bash /home/fabian/download_models.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
|
||||
desc="Start model download in background")
|
||||
|
||||
# Wait and check progress
|
||||
time.sleep(10)
|
||||
run("tail -20 /home/fabian/model_download.log 2>/dev/null || echo 'Waiting for log...'",
|
||||
desc="Initial download progress")
|
||||
|
||||
# Keep checking
|
||||
for i in range(6):
|
||||
time.sleep(30)
|
||||
rc, out, _ = run(f"tail -10 /home/fabian/model_download.log 2>/dev/null",
|
||||
desc=f"Download progress check {i+1}")
|
||||
if 'DOWNLOAD_COMPLETE' in out:
|
||||
print("\n ALL DOWNLOADS COMPLETE!")
|
||||
break
|
||||
|
||||
# Final check
|
||||
run("tail -20 /home/fabian/model_download.log 2>/dev/null",
|
||||
desc="Final download status")
|
||||
|
||||
run("du -sh ~/ComfyUI/models/diffusion_models/ ~/ComfyUI/models/text_encoders/ ~/ComfyUI/models/vae/ 2>/dev/null",
|
||||
desc="Model directory sizes")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download Z-Image-Turbo GGUF models on BC-250 using wget."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=7200, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 30:
|
||||
print(f" ... ({len(lines)} lines, showing last 30)")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-15:] if len(lines) > 15 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
dl_script = r'''#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
LOG="/home/fabian/model_download.log"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
COMFY="$HOME/ComfyUI"
|
||||
|
||||
echo "=========================================="
|
||||
echo " Downloading Z-Image-Turbo GGUF Models"
|
||||
echo " Started: $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
# 1. Diffusion model (5.19 GB)
|
||||
echo ""
|
||||
echo "[1/3] Downloading z_image_turbo-Q5_K_S.gguf (5.19 GB)..."
|
||||
DEST1="$COMFY/models/diffusion_models/z_image_turbo-Q5_K_S.gguf"
|
||||
if [ -f "$DEST1" ]; then
|
||||
echo " Already exists ($(du -h "$DEST1" | cut -f1)), skipping."
|
||||
else
|
||||
wget -c -q --show-progress \
|
||||
"https://huggingface.co/jayn7/Z-Image-Turbo-GGUF/resolve/main/z_image_turbo-Q5_K_S.gguf" \
|
||||
-O "$DEST1.tmp"
|
||||
mv "$DEST1.tmp" "$DEST1"
|
||||
echo " Done: $(du -h "$DEST1" | cut -f1)"
|
||||
fi
|
||||
|
||||
# 2. Text encoder Qwen3-4B (2.82 GB)
|
||||
echo ""
|
||||
echo "[2/3] Downloading Qwen3-4B.i1-Q5_K_S.gguf (2.82 GB)..."
|
||||
DEST2="$COMFY/models/text_encoders/Qwen3-4B.i1-Q5_K_S.gguf"
|
||||
if [ -f "$DEST2" ]; then
|
||||
echo " Already exists ($(du -h "$DEST2" | cut -f1)), skipping."
|
||||
else
|
||||
wget -c -q --show-progress \
|
||||
"https://huggingface.co/mradermacher/Qwen3-4B-i1-GGUF/resolve/main/Qwen3-4B.i1-Q5_K_S.gguf" \
|
||||
-O "$DEST2.tmp"
|
||||
mv "$DEST2.tmp" "$DEST2"
|
||||
echo " Done: $(du -h "$DEST2" | cut -f1)"
|
||||
fi
|
||||
|
||||
# 3. VAE (335 MB)
|
||||
echo ""
|
||||
echo "[3/3] Downloading ae.safetensors (VAE, 335 MB)..."
|
||||
DEST3="$COMFY/models/vae/ae.safetensors"
|
||||
if [ -f "$DEST3" ]; then
|
||||
echo " Already exists ($(du -h "$DEST3" | cut -f1)), skipping."
|
||||
else
|
||||
wget -c -q --show-progress \
|
||||
"https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/vae/ae.safetensors" \
|
||||
-O "$DEST3.tmp"
|
||||
mv "$DEST3.tmp" "$DEST3"
|
||||
echo " Done: $(du -h "$DEST3" | cut -f1)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " Model Download Summary"
|
||||
echo "=========================================="
|
||||
echo "Diffusion model:"
|
||||
ls -lh "$COMFY/models/diffusion_models/"*.gguf 2>/dev/null || echo " NOT FOUND"
|
||||
echo "Text encoder:"
|
||||
ls -lh "$COMFY/models/text_encoders/"*.gguf 2>/dev/null || echo " NOT FOUND"
|
||||
echo "VAE:"
|
||||
ls -lh "$COMFY/models/vae/"*.safetensors 2>/dev/null || echo " NOT FOUND"
|
||||
echo ""
|
||||
echo "Total model size:"
|
||||
du -sh "$COMFY/models/"
|
||||
echo ""
|
||||
echo "DOWNLOAD_COMPLETE"
|
||||
echo "Finished: $(date)"
|
||||
'''
|
||||
|
||||
# Upload download script
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/download_models.sh', 'w') as f:
|
||||
f.write(dl_script)
|
||||
sftp.close()
|
||||
|
||||
run("chmod +x /home/fabian/download_models.sh", desc="Make script executable")
|
||||
run("rm -f /home/fabian/model_download.log", desc="Clean old log")
|
||||
run("bash -c 'which wget'", desc="Verify wget exists")
|
||||
|
||||
# Start download in background via nohup
|
||||
run("bash -c 'nohup bash /home/fabian/download_models.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
|
||||
desc="Start model download in background")
|
||||
|
||||
time.sleep(15)
|
||||
run("tail -20 /home/fabian/model_download.log 2>/dev/null || echo 'Waiting for log...'",
|
||||
desc="Initial download progress")
|
||||
|
||||
# Monitor download progress - check every 60s for up to 30 minutes
|
||||
for i in range(30):
|
||||
time.sleep(60)
|
||||
rc, out, _ = run(f"tail -5 /home/fabian/model_download.log 2>/dev/null; echo '---'; "
|
||||
f"ls -lh ~/ComfyUI/models/diffusion_models/ ~/ComfyUI/models/text_encoders/ ~/ComfyUI/models/vae/ 2>/dev/null",
|
||||
desc=f"Progress check {i+1}/30 ({(i+1)}min)")
|
||||
if 'DOWNLOAD_COMPLETE' in out:
|
||||
print("\n ALL DOWNLOADS COMPLETE!")
|
||||
break
|
||||
# Check if background process still running
|
||||
_, pout, _ = run("bash -c 'pgrep -f download_models.sh || echo NOPROCESS'")
|
||||
if 'NOPROCESS' in pout and 'DOWNLOAD_COMPLETE' not in out:
|
||||
print("\n WARNING: Download process exited without completion!")
|
||||
run("cat /home/fabian/model_download.log", desc="Full download log")
|
||||
break
|
||||
|
||||
# Final verification
|
||||
run("tail -25 /home/fabian/model_download.log 2>/dev/null", desc="Final download status")
|
||||
run("du -sh ~/ComfyUI/models/diffusion_models/ ~/ComfyUI/models/text_encoders/ ~/ComfyUI/models/vae/",
|
||||
desc="Model directory sizes")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create ComfyUI startup script and launch it on BC-250."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=120, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 40:
|
||||
print(f" ... ({len(lines)} lines, showing last 40)")
|
||||
print('\n'.join(lines[-40:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 1. Create the startup script
|
||||
# ──────────────────────────────────────────────────────────
|
||||
startup_script = r'''#!/bin/bash
|
||||
# ComfyUI Startup Script for AsRock BC-250 (AMD Cyan Skillfish / ROCm 7.2)
|
||||
# Usage: ~/start_comfyui.sh [--listen] [--port PORT]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# BC-250 AMD GPU Environment Variables
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# Override gfx1013 → gfx1010 (RDNA 1.5 → RDNA 1 compat)
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
|
||||
# Use device 0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
|
||||
# Disable SDMA (avoids queue errors on Cyan Skillfish)
|
||||
export HSA_ENABLE_SDMA=0
|
||||
|
||||
# Suppress tool library warnings
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
|
||||
# PyTorch / ROCm tuning
|
||||
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:True"
|
||||
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
|
||||
|
||||
# Avoid OOM on 14GB shared VRAM — force float16 where possible
|
||||
export COMFY_PRECISION=16
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# Activate Virtual Environment
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
source "$HOME/comfyui-env/bin/activate"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# Launch ComfyUI
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
cd "$HOME/ComfyUI"
|
||||
|
||||
echo "=========================================="
|
||||
echo " ComfyUI on BC-250 (ROCm 7.2)"
|
||||
echo "=========================================="
|
||||
echo " GPU: AMD Cyan Skillfish (gfx1013→gfx1010)"
|
||||
echo " PyTorch: $(python -c 'import torch; print(torch.__version__)')"
|
||||
echo " HIP: $(python -c 'import torch; print(torch.version.hip)')"
|
||||
echo " CUDA: $(python -c 'import torch; print(torch.cuda.is_available())')"
|
||||
echo " Device: $(python -c 'import torch; print(torch.cuda.get_device_name(0) if torch.cuda.is_available() else "N/A")')"
|
||||
echo "=========================================="
|
||||
|
||||
# Default: listen on all interfaces for remote access
|
||||
LISTEN_ARGS="--listen 0.0.0.0 --port 8188"
|
||||
|
||||
# Parse arguments (override defaults if provided)
|
||||
if [ $# -gt 0 ]; then
|
||||
LISTEN_ARGS="$@"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Starting ComfyUI with: $LISTEN_ARGS"
|
||||
echo "Access at: http://$(hostname -I | awk '{print $1}'):8188"
|
||||
echo ""
|
||||
|
||||
exec python main.py $LISTEN_ARGS
|
||||
'''
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 2. Create fish shell wrapper too
|
||||
# ──────────────────────────────────────────────────────────
|
||||
fish_script = r'''#!/usr/bin/env fish
|
||||
# ComfyUI launcher for fish shell on BC-250
|
||||
|
||||
# BC-250 GPU env vars
|
||||
set -gx HSA_OVERRIDE_GFX_VERSION 10.1.0
|
||||
set -gx HIP_VISIBLE_DEVICES 0
|
||||
set -gx HSA_ENABLE_SDMA 0
|
||||
set -gx HSA_TOOLS_LIB ""
|
||||
set -gx HSA_TOOLS_REPORT_LOAD_FAILURE 0
|
||||
set -gx PYTORCH_HIP_ALLOC_CONF "expandable_segments:True"
|
||||
set -gx TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL 1
|
||||
|
||||
# Activate venv
|
||||
source $HOME/comfyui-env/bin/activate.fish
|
||||
|
||||
# Launch
|
||||
cd $HOME/ComfyUI
|
||||
echo "Starting ComfyUI on BC-250..."
|
||||
python main.py --listen 0.0.0.0 --port 8188 $argv
|
||||
'''
|
||||
|
||||
# Upload scripts
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
|
||||
f.write(startup_script)
|
||||
with sftp.open('/home/fabian/start_comfyui.fish', 'w') as f:
|
||||
f.write(fish_script)
|
||||
sftp.close()
|
||||
|
||||
run("chmod +x /home/fabian/start_comfyui.sh /home/fabian/start_comfyui.fish",
|
||||
desc="Make startup scripts executable")
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 3. Quick pre-flight check
|
||||
# ──────────────────────────────────────────────────────────
|
||||
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && "
|
||||
"export HSA_OVERRIDE_GFX_VERSION=10.1.0 && "
|
||||
"export HIP_VISIBLE_DEVICES=0 && "
|
||||
"export HSA_ENABLE_SDMA=0 && "
|
||||
"cd /home/fabian/ComfyUI && "
|
||||
"python -c \""
|
||||
"import torch; "
|
||||
"print(f\\\"PyTorch {torch.__version__}, HIP {torch.version.hip}, CUDA {torch.cuda.is_available()}\\\"); "
|
||||
"print(f\\\"Device: {torch.cuda.get_device_name(0)}\\\"); "
|
||||
"import comfy.model_management; "
|
||||
"print(f\\\"ComfyUI model_management imported OK\\\"); "
|
||||
"\"'",
|
||||
desc="Pre-flight: PyTorch + ComfyUI import check")
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 4. Launch ComfyUI in background
|
||||
# ──────────────────────────────────────────────────────────
|
||||
run("bash -c 'pkill -f \"python main.py\" 2>/dev/null; echo killed || echo no_existing'",
|
||||
desc="Kill any existing ComfyUI process")
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
|
||||
desc="Launch ComfyUI in background")
|
||||
|
||||
# Wait for startup
|
||||
time.sleep(15)
|
||||
run("tail -30 /home/fabian/comfyui.log 2>/dev/null", desc="ComfyUI startup log")
|
||||
|
||||
# Check if port is listening
|
||||
time.sleep(10)
|
||||
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
|
||||
desc="Check if port 8188 is listening")
|
||||
|
||||
run("tail -50 /home/fabian/comfyui.log 2>/dev/null", desc="Full startup log")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix workflow submission: write JSON to file, curl from file, monitor."""
|
||||
import paramiko, json, time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian',
|
||||
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
|
||||
|
||||
def run(cmd, timeout=60):
|
||||
_, so, se = ssh.exec_command(cmd, timeout=timeout)
|
||||
return so.read().decode(), se.read().decode()
|
||||
|
||||
try:
|
||||
# Step 1: Write workflow JSON via SFTP (reliable, no shell escaping)
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}},
|
||||
"6": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"7": {"class_type": "KSampler",
|
||||
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple",
|
||||
"positive": ["4", 0], "negative": ["5", 0],
|
||||
"latent_image": ["6", 0], "denoise": 1.0}},
|
||||
"8": {"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
|
||||
"9": {"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
|
||||
}
|
||||
}
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/wf.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
sftp.close()
|
||||
print("Workflow JSON written via SFTP.")
|
||||
|
||||
# Verify JSON is valid
|
||||
out, _ = run("python3 -c \"import json; d=json.load(open('/tmp/wf.json')); print('nodes:', sorted(d['prompt'].keys()))\"")
|
||||
print(f"Verify: {out.strip()}")
|
||||
|
||||
# Step 2: Check startup script has --cpu-vae
|
||||
out, _ = run("cat /home/fabian/start_comfyui.sh")
|
||||
has_cpu_vae = '--cpu-vae' in out
|
||||
print(f"Startup has --cpu-vae: {has_cpu_vae}")
|
||||
print(f"Startup has --novram: {'--novram' in out}")
|
||||
print(f"Startup has --force-fp16: {'--force-fp16' in out}")
|
||||
|
||||
# Step 3: Submit
|
||||
out, err = run("curl -s -X POST http://localhost:8188/prompt -H 'Content-Type: application/json' -d @/tmp/wf.json")
|
||||
print(f"\nSubmit response: {out.strip()[:500]}")
|
||||
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
except:
|
||||
print(f"Failed to parse response!")
|
||||
raise SystemExit(1)
|
||||
|
||||
if 'error' in resp:
|
||||
print(f"\nAPI ERROR: {resp['error']}")
|
||||
print(f"Details: {resp.get('details','')}")
|
||||
print(f"Node errors: {resp.get('node_errors',{})}")
|
||||
raise SystemExit(1)
|
||||
|
||||
prompt_id = resp.get('prompt_id', 'unknown')
|
||||
print(f"Prompt ID: {prompt_id}")
|
||||
|
||||
# Step 4: Monitor (15s intervals, up to 30 min)
|
||||
print("\nMonitoring generation (GPU sampling + CPU VAE)...")
|
||||
last_log = ""
|
||||
for i in range(120):
|
||||
time.sleep(15)
|
||||
|
||||
stats, _ = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" CPU=$(ps -p $PID -o %cpu --no-headers); "
|
||||
" MEM=$(ps -p $PID -o rss --no-headers); "
|
||||
" echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:$(cut -d\" \" -f1-3 /proc/loadavg)\"; "
|
||||
"else echo DEAD; fi'")
|
||||
|
||||
log, _ = run("tail -8 /home/fabian/comfyui.log 2>/dev/null")
|
||||
|
||||
m, s = divmod((i+1)*15, 60)
|
||||
print(f" [{m}m{s:02d}s] {stats.strip()}")
|
||||
|
||||
if log.strip() != last_log:
|
||||
for line in reversed(log.strip().split('\n')):
|
||||
l = line.strip()
|
||||
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
|
||||
print(f" LOG: {l[:120]}")
|
||||
break
|
||||
last_log = log.strip()
|
||||
|
||||
if 'DEAD' in stats:
|
||||
print("\nPROCESS DIED!")
|
||||
out, _ = run("tail -50 /home/fabian/comfyui.log")
|
||||
print(out)
|
||||
break
|
||||
if 'Prompt executed in' in log:
|
||||
print("\nSUCCESS! Image generated!")
|
||||
out, _ = run("tail -20 /home/fabian/comfyui.log")
|
||||
print(out)
|
||||
break
|
||||
if 'Traceback' in log or 'RuntimeError' in log:
|
||||
print("\nERROR detected!")
|
||||
out, _ = run("tail -50 /home/fabian/comfyui.log")
|
||||
print(out)
|
||||
break
|
||||
|
||||
# Output files
|
||||
print("\n=== Output files ===")
|
||||
out, _ = run("ls -lah /home/fabian/ComfyUI/output/")
|
||||
print(out.strip())
|
||||
|
||||
finally:
|
||||
ssh.close()
|
||||
print("\nSSH closed.")
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test Z-Image-Turbo end-to-end on BC-250 via ComfyUI API.
|
||||
|
||||
1. Query available nodes from ComfyUI
|
||||
2. Build a workflow using Z-Image nodes + GGUF loader
|
||||
3. Submit and wait for image generation
|
||||
"""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 60:
|
||||
print(f" ... ({len(lines)} lines, showing last 60)")
|
||||
print('\n'.join(lines[-60:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# 1. Check ComfyUI is still running
|
||||
run("bash -c 'ss -tlnp | grep 8188'", desc="Verify ComfyUI is running")
|
||||
|
||||
# 2. Get available node types — look for Z-Image and GGUF related nodes
|
||||
run("bash -c 'curl -s http://localhost:8188/object_info 2>/dev/null | python3 -c \""
|
||||
"import sys, json; "
|
||||
"data = json.load(sys.stdin); "
|
||||
"nodes = sorted(data.keys()); "
|
||||
"z_nodes = [n for n in nodes if any(k in n.lower() for k in [\\\"zimage\\\", \\\"z_image\\\", \\\"zi_\\\", \\\"gguf\\\", \\\"unet\\\", \\\"sampler\\\", \\\"vae\\\", \\\"clip\\\", \\\"save\\\", \\\"empty\\\", \\\"latent\\\"])]; "
|
||||
"print(\\\"Relevant nodes:\\\"); "
|
||||
"[print(f\\\" {n}\\\") for n in z_nodes]; "
|
||||
"print(f\\\"\\\\nTotal nodes: {len(nodes)}\\\"); "
|
||||
"\"'",
|
||||
desc="Query ComfyUI for available nodes")
|
||||
|
||||
# 3. Get detailed info on Z-Image and GGUF nodes
|
||||
run("bash -c 'curl -s http://localhost:8188/object_info 2>/dev/null | python3 -c \""
|
||||
"import sys, json; "
|
||||
"data = json.load(sys.stdin); "
|
||||
"targets = [k for k in data if any(t in k.lower() for t in [\\\"zi_\\\", \\\"zimage\\\", \\\"z_image\\\", \\\"gguf\\\"])]; "
|
||||
"for name in sorted(targets): "
|
||||
" info = data[name]; "
|
||||
" print(f\\\"\\\\n=== {name} ===\\\"); "
|
||||
" inp = info.get(\\\"input\\\", {}).get(\\\"required\\\", {}); "
|
||||
" print(f\\\" Required inputs:\\\"); "
|
||||
" for k, v in inp.items(): "
|
||||
" print(f\\\" {k}: {v}\\\"); "
|
||||
" opt = info.get(\\\"input\\\", {}).get(\\\"optional\\\", {}); "
|
||||
" if opt: "
|
||||
" print(f\\\" Optional inputs:\\\"); "
|
||||
" for k, v in opt.items(): "
|
||||
" print(f\\\" {k}: {v}\\\"); "
|
||||
" out = info.get(\\\"output\\\", []); "
|
||||
" print(f\\\" Outputs: {out}\\\"); "
|
||||
"\"'",
|
||||
desc="Get Z-Image and GGUF node details")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Get detailed node info from ComfyUI on BC-250."""
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
# Create a Python script on the remote to query node info
|
||||
query_script = '''#!/usr/bin/env python3
|
||||
import json, urllib.request
|
||||
|
||||
data = json.loads(urllib.request.urlopen("http://localhost:8188/object_info").read())
|
||||
|
||||
# Find all Z-Image and GGUF related nodes
|
||||
targets = [k for k in data if any(t in k.lower() for t in ["zi_", "zimage", "z_image", "gguf", "zi ", "zsampler", "emptyz", "textencodez"])]
|
||||
|
||||
# Also check for standard nodes we need
|
||||
standard = ["UNETLoader", "VAELoader", "VAEDecode", "SaveImage", "EmptyLatentImage", "CLIPTextEncode", "KSampler"]
|
||||
for s in standard:
|
||||
if s in data and s not in targets:
|
||||
targets.append(s)
|
||||
|
||||
for name in sorted(targets):
|
||||
info = data[name]
|
||||
print(f"\\n=== {name} ===")
|
||||
inp = info.get("input", {}).get("required", {})
|
||||
if inp:
|
||||
print(" Required:")
|
||||
for k, v in inp.items():
|
||||
print(f" {k}: {v}")
|
||||
opt = info.get("input", {}).get("optional", {})
|
||||
if opt:
|
||||
print(" Optional:")
|
||||
for k, v in opt.items():
|
||||
print(f" {k}: {v}")
|
||||
out = info.get("output", [])
|
||||
out_names = info.get("output_name", [])
|
||||
print(f" Outputs: {list(zip(out, out_names)) if out_names else out}")
|
||||
|
||||
# Also list ALL nodes with "empty" and "latent" in the name
|
||||
print("\\n\\n=== Nodes with 'empty' or 'z' in name ===")
|
||||
for k in sorted(data.keys()):
|
||||
if "empty" in k.lower() or ("z" in k.lower() and "image" in k.lower()):
|
||||
print(f" {k}")
|
||||
'''
|
||||
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/query_nodes.py', 'w') as f:
|
||||
f.write(query_script)
|
||||
sftp.close()
|
||||
|
||||
_, stdout, stderr = ssh.exec_command("python3 /tmp/query_nodes.py", timeout=30)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
print(out)
|
||||
if err.strip():
|
||||
print(f"STDERR: {err.strip()}")
|
||||
|
||||
ssh.close()
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Submit Z-Image-Turbo workflow to ComfyUI on BC-250 via API."""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=600, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd[:200]}...")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 50:
|
||||
print(f" ... ({len(lines)} lines, showing last 50)")
|
||||
print('\n'.join(lines[-50:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# ComfyUI API prompt workflow for Z-Image-Turbo
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {
|
||||
"unet_name": "z_image_turbo-Q5_K_S.gguf"
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {
|
||||
"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf",
|
||||
"type": "qwen_image"
|
||||
}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {
|
||||
"vae_name": "ae.safetensors"
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A majestic mountain landscape at sunset, golden light illuminating snow-capped peaks, crystal clear lake in the foreground reflecting the sky, photorealistic, 8k, detailed",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "EmptyZImageLatentImage //ZImagePowerNodes",
|
||||
"inputs": {
|
||||
"landscape": True,
|
||||
"ratio": "16:9 (widescreen)",
|
||||
"size": "medium (recommended)",
|
||||
"batch_size": 1
|
||||
}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "ZSamplerTurbo //ZImagePowerNodes",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"positive": ["4", 0],
|
||||
"latent_input": ["5", 0],
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"denoise": 1.0,
|
||||
"initial_noise_calibration": "off",
|
||||
"lowres_bias": False
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {
|
||||
"samples": ["6", 0],
|
||||
"vae": ["3", 0]
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {
|
||||
"images": ["7", 0],
|
||||
"filename_prefix": "ZImageTurbo_BC250_test"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Write workflow to remote
|
||||
workflow_json = json.dumps(workflow)
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/zimage_workflow.json', 'w') as f:
|
||||
f.write(workflow_json)
|
||||
sftp.close()
|
||||
|
||||
# Submit via curl
|
||||
run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H \"Content-Type: application/json\" "
|
||||
"-d @/tmp/zimage_workflow.json'",
|
||||
desc="Submit Z-Image-Turbo workflow to ComfyUI")
|
||||
|
||||
# Monitor the queue and wait for completion
|
||||
time.sleep(5)
|
||||
run("bash -c 'curl -s http://localhost:8188/queue'",
|
||||
desc="Check queue status")
|
||||
|
||||
# Wait and check ComfyUI log for progress
|
||||
for i in range(30):
|
||||
time.sleep(10)
|
||||
rc, out, _ = run(f"bash -c 'tail -20 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc=f"ComfyUI log check {i+1}")
|
||||
if any(kw in out for kw in ['Prompt executed', 'SaveImage', 'output images']):
|
||||
print("\n IMAGE GENERATION COMPLETE!")
|
||||
break
|
||||
if 'error' in out.lower() or 'Error' in out:
|
||||
print("\n ERROR DETECTED — checking full log...")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Full error log")
|
||||
break
|
||||
|
||||
# Check output
|
||||
run("bash -c 'ls -la ~/ComfyUI/output/ 2>/dev/null'",
|
||||
desc="Check output directory")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Submit Z-Image-Turbo workflow using standard KSampler to ComfyUI on BC-250."""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=600, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 50:
|
||||
print(f" ... ({len(lines)} lines, showing last 50)")
|
||||
print('\n'.join(lines[-50:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Z-Image-Turbo workflow using standard KSampler
|
||||
# Turbo models: low steps (8), low/zero CFG (1.0 with cfg_pp or euler works)
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {
|
||||
"unet_name": "z_image_turbo-Q5_K_S.gguf"
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {
|
||||
"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf",
|
||||
"type": "qwen_image"
|
||||
}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {
|
||||
"vae_name": "ae.safetensors"
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A majestic mountain landscape at sunset, golden light illuminating snow-capped peaks, crystal clear lake reflecting the sky, photorealistic",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {
|
||||
"width": 1024,
|
||||
"height": 576,
|
||||
"batch_size": 1
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": ["4", 0],
|
||||
"negative": ["5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {
|
||||
"samples": ["7", 0],
|
||||
"vae": ["3", 0]
|
||||
}
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {
|
||||
"images": ["8", 0],
|
||||
"filename_prefix": "ZImageTurbo_BC250_test"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Write workflow
|
||||
workflow_json = json.dumps(workflow)
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/zimage_workflow2.json', 'w') as f:
|
||||
f.write(workflow_json)
|
||||
sftp.close()
|
||||
|
||||
# Submit
|
||||
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H \"Content-Type: application/json\" "
|
||||
"-d @/tmp/zimage_workflow2.json'",
|
||||
desc="Submit Z-Image-Turbo workflow")
|
||||
|
||||
response = {}
|
||||
try:
|
||||
response = json.loads(out.strip())
|
||||
except:
|
||||
pass
|
||||
|
||||
if 'error' in response:
|
||||
print(f"\nERROR: {response['error']}")
|
||||
if 'node_errors' in response:
|
||||
for node_id, errs in response['node_errors'].items():
|
||||
print(f" Node {node_id} ({errs.get('class_type','')}): {errs.get('errors','')}")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
|
||||
prompt_id = response.get('prompt_id', '')
|
||||
print(f"\nPrompt ID: {prompt_id}")
|
||||
|
||||
# Monitor progress — model loading + 8 sampling steps
|
||||
for i in range(60): # up to 10 minutes
|
||||
time.sleep(10)
|
||||
rc, out, _ = run(f"bash -c 'tail -30 /home/fabian/comfyui.log 2>/dev/null'",
|
||||
desc=f"Progress {i+1} ({(i+1)*10}s)")
|
||||
|
||||
if 'Prompt executed in' in out:
|
||||
print("\n IMAGE GENERATION COMPLETE!")
|
||||
break
|
||||
if 'Exception' in out or 'Traceback' in out:
|
||||
print("\n ERROR during generation!")
|
||||
run("bash -c 'tail -80 /home/fabian/comfyui.log'", desc="Error details")
|
||||
break
|
||||
|
||||
# Check output files
|
||||
run("bash -c 'ls -la ~/ComfyUI/output/ 2>/dev/null'",
|
||||
desc="Output directory")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Submit Z-Image-Turbo workflow and monitor CPU/threading."""
|
||||
import paramiko
|
||||
import json
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=60, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 40:
|
||||
print(f" ... ({len(lines)} lines, showing last 40)")
|
||||
print('\n'.join(lines[-40:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')[-10:]
|
||||
print(f"STDERR: {chr(10).join(lines)}")
|
||||
print(f" Exit: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Workflow JSON
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoaderGGUF",
|
||||
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "ae.safetensors"}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "A majestic mountain landscape at sunset, golden light on snow peaks, crystal lake reflection, photorealistic, 8k",
|
||||
"clip": ["2", 0]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["2", 0]}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 1024, "height": 576, "batch_size": 1}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"seed": 42,
|
||||
"steps": 8,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": ["4", 0],
|
||||
"negative": ["5", 0],
|
||||
"latent_image": ["6", 0],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_BC250"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Upload and submit
|
||||
sftp = ssh.open_sftp()
|
||||
with sftp.open('/tmp/zimage_workflow.json', 'w') as f:
|
||||
f.write(json.dumps(workflow))
|
||||
sftp.close()
|
||||
|
||||
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
|
||||
"-H \"Content-Type: application/json\" "
|
||||
"-d @/tmp/zimage_workflow.json'",
|
||||
desc="Submit workflow")
|
||||
|
||||
try:
|
||||
resp = json.loads(out.strip())
|
||||
if 'error' in resp:
|
||||
print(f"\nERROR: {resp['error']}")
|
||||
if 'node_errors' in resp:
|
||||
for nid, e in resp['node_errors'].items():
|
||||
print(f" Node {nid}: {e.get('errors', [])}")
|
||||
ssh.close()
|
||||
exit(1)
|
||||
print(f"\nPrompt ID: {resp.get('prompt_id', 'unknown')}")
|
||||
except:
|
||||
print(f"Response: {out.strip()[:500]}")
|
||||
|
||||
# Monitor: check CPU usage + log every 15s
|
||||
print("\n Monitoring CPU and progress...")
|
||||
for i in range(80): # up to 20 minutes
|
||||
time.sleep(15)
|
||||
|
||||
# Get CPU usage per-thread of ComfyUI process + total system load
|
||||
rc, cpu_out, _ = run("bash -c '"
|
||||
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
|
||||
"if [ -n \"$PID\" ]; then "
|
||||
" echo \"=== Process CPU ===\"; "
|
||||
" ps -p $PID -o pid,%cpu,%mem,nlwp --no-headers; "
|
||||
" echo \"=== System Load ===\"; "
|
||||
" uptime; "
|
||||
" echo \"=== Per-Core ===\"; "
|
||||
" mpstat -P ALL 1 1 2>/dev/null | tail -15 || cat /proc/loadavg; "
|
||||
"else echo PROCESS_DEAD; fi'")
|
||||
|
||||
rc, log_out, _ = run("bash -c 'tail -5 /home/fabian/comfyui.log 2>/dev/null'")
|
||||
|
||||
if 'PROCESS_DEAD' in (cpu_out or ''):
|
||||
print("\n ComfyUI process died!")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Death log")
|
||||
break
|
||||
if 'Prompt executed in' in (log_out or ''):
|
||||
print(f"\n IMAGE GENERATED! (at check {i+1}, ~{(i+1)*15}s)")
|
||||
run("bash -c 'tail -30 /home/fabian/comfyui.log'", desc="Completion log")
|
||||
break
|
||||
if 'Traceback' in (log_out or '') or 'Exception' in (log_out or ''):
|
||||
print("\n ERROR!")
|
||||
run("bash -c 'tail -60 /home/fabian/comfyui.log'", desc="Error log")
|
||||
break
|
||||
|
||||
print(f" [{i+1}] {(i+1)*15}s elapsed...")
|
||||
|
||||
# Check output
|
||||
run("bash -c 'ls -lah ~/ComfyUI/output/ 2>/dev/null'", desc="Output files")
|
||||
run("bash -c 'tail -20 /home/fabian/comfyui.log 2>/dev/null'", desc="Final log")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Try pre-built PyTorch ROCm from CachyOS repos, test on BC-250."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=300, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
print(f"$ {cmd}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 80:
|
||||
print(f" ... ({len(lines)} lines, showing last 80)")
|
||||
print('\n'.join(lines[-80:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
lines = err.strip().split('\n')
|
||||
show = lines[-20:] if len(lines) > 20 else lines
|
||||
print(f"STDERR: {chr(10).join(show)}")
|
||||
print(f" Exit code: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Check what architectures the pre-built packages support
|
||||
run("pacman -Si python-pytorch-rocm 2>&1 | head -20",
|
||||
desc="Pre-built PyTorch ROCm package info")
|
||||
|
||||
run("pacman -Si python-pytorch-opt-rocm 2>&1 | head -20",
|
||||
desc="Pre-built PyTorch Opt ROCm package info")
|
||||
|
||||
# Install the pre-built package (system-wide, venv will pick it up via --system-site-packages)
|
||||
run("sudo pacman -S --needed --noconfirm python-pytorch-rocm 2>&1 | tail -30",
|
||||
desc="Install pre-built PyTorch ROCm",
|
||||
timeout=600)
|
||||
|
||||
# Test if it works in venv
|
||||
run("""bash -c 'source ~/comfyui-env/bin/activate && \
|
||||
HSA_OVERRIDE_GFX_VERSION=10.1.0 \
|
||||
HIP_VISIBLE_DEVICES=0 \
|
||||
HSA_ENABLE_SDMA=0 \
|
||||
python3 -c "
|
||||
import torch
|
||||
print(f\\"PyTorch version: {torch.__version__}\\")
|
||||
print(f\\"HIP version: {torch.version.hip}\\")
|
||||
print(f\\"CUDA available (HIP): {torch.cuda.is_available()}\\")
|
||||
if torch.cuda.is_available():
|
||||
print(f\\"Device count: {torch.cuda.device_count()}\\")
|
||||
print(f\\"Device name: {torch.cuda.get_device_name(0)}\\")
|
||||
print(f\\"Device arch: {torch.cuda.get_device_capability(0)}\\")
|
||||
# Try a simple tensor operation on GPU
|
||||
t = torch.randn(4, 4, device=\\"cuda\\")
|
||||
print(f\\"Tensor device: {t.device}\\")
|
||||
print(f\\"Tensor sum: {t.sum().item():.4f}\\")
|
||||
# Try matmul
|
||||
a = torch.randn(64, 64, device=\\"cuda\\")
|
||||
b = torch.randn(64, 64, device=\\"cuda\\")
|
||||
c = torch.matmul(a, b)
|
||||
print(f\\"Matmul result shape: {c.shape}\\")
|
||||
print(\\"GPU COMPUTE: WORKING\\")
|
||||
else:
|
||||
print(\\"CUDA/HIP NOT AVAILABLE\\")
|
||||
" 2>&1'""",
|
||||
desc="Test PyTorch on GPU",
|
||||
timeout=120)
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Check if VAE is hanging or running, then fix and restart."""
|
||||
import paramiko, time, json, textwrap
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def sh(cmd, timeout=30):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
# Check if process is truly hung or still computing
|
||||
print("=== GPU MEMORY / ACTIVITY ===")
|
||||
print(sh('rocm-smi --showmemuse --showuse 2>/dev/null | head -20'))
|
||||
print()
|
||||
print(sh('rocm-smi --showmeminfo vram 2>/dev/null'))
|
||||
print()
|
||||
|
||||
# Check CPU usage of the process
|
||||
pid = sh('pgrep -f "python3.*main.py" | head -1')
|
||||
print(f"PID: {pid}")
|
||||
if pid:
|
||||
print(f"CPU%: {sh('ps -p ' + pid + ' -o %cpu,%mem,rss,vsz --no-headers')}")
|
||||
# Check /proc/pid/status for threads
|
||||
print(f"Threads: {sh('grep Threads /proc/' + pid + '/status 2>/dev/null')}")
|
||||
# strace peek - what syscall is it stuck on?
|
||||
print(f"\nStack peek (1s):")
|
||||
print(sh('timeout 2 strace -p ' + pid + ' -c 2>&1 | head -20', timeout=10))
|
||||
|
||||
# Check last log line timestamp
|
||||
print("\n=== LOG LAST LINES ===")
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
lines = [l for l in log.split('\n') if l.strip()]
|
||||
for l in lines[-10:]:
|
||||
print(f" {l.strip()}")
|
||||
|
||||
# Check if output image exists
|
||||
imgs = sh('ls -la ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null')
|
||||
print(f"\nOutput images: {imgs or 'NONE'}")
|
||||
|
||||
# ============================================================
|
||||
# THE FIX: VAE on GPU hangs on this APU. Force CPU VAE but
|
||||
# ensure ALL 12 threads are used via explicit torch patch.
|
||||
# We'll also patch ComfyUI to call torch.set_num_threads(12)
|
||||
# RIGHT BEFORE vae decode, in case something resets it.
|
||||
# ============================================================
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("FIXING: Kill, patch VAE threading, restart with --cpu-vae")
|
||||
print("="*60)
|
||||
|
||||
# Kill
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
print("Killed ComfyUI")
|
||||
|
||||
# Read current model_management.py
|
||||
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
|
||||
mm = f.read().decode()
|
||||
|
||||
# Find and patch vae_device to force CPU + set threads
|
||||
# Current: def vae_device(): ... return vae_dev
|
||||
# We need to find the vae decode path and add threading there
|
||||
# But actually the simplest: patch the VAEDecode node itself
|
||||
|
||||
# Check nodes_latent.py for VAEDecode
|
||||
vae_decode_path = sh('grep -rn "class VAEDecode" ~/ComfyUI/comfy_extras/ ~/ComfyUI/nodes.py 2>/dev/null')
|
||||
print(f"\nVAEDecode location: {vae_decode_path}")
|
||||
|
||||
# Read the relevant file
|
||||
vae_file = ''
|
||||
vae_line = ''
|
||||
for line in vae_decode_path.split('\n'):
|
||||
if 'class VAEDecode' in line and 'Tiled' not in line:
|
||||
parts = line.split(':')
|
||||
vae_file = parts[0]
|
||||
vae_line = parts[1]
|
||||
break
|
||||
print(f"VAE file: {vae_file}, line: {vae_line}")
|
||||
|
||||
if vae_file:
|
||||
with sftp.open(vae_file, 'r') as f:
|
||||
vae_code = f.read().decode()
|
||||
# Show VAEDecode class
|
||||
vae_lines = vae_code.split('\n')
|
||||
start = int(vae_line) - 1
|
||||
print(f"\nVAEDecode class (from line {vae_line}):")
|
||||
for i in range(start, min(start+30, len(vae_lines))):
|
||||
print(f" {i+1}: {vae_lines[i]}")
|
||||
|
||||
# Also check where vae.decode is called in the VAE wrapper
|
||||
print("\n=== VAE decode method location ===")
|
||||
vae_impl = sh('grep -rn "def decode" ~/ComfyUI/comfy/sd.py 2>/dev/null | head -5')
|
||||
print(vae_impl)
|
||||
|
||||
# Read sd.py decode method
|
||||
for line in vae_impl.split('\n'):
|
||||
if 'def decode' in line:
|
||||
parts = line.split(':')
|
||||
sd_file = parts[0]
|
||||
sd_line = int(parts[1])
|
||||
with sftp.open(sd_file, 'r') as f:
|
||||
sd_code = f.read().decode()
|
||||
sd_lines = sd_code.split('\n')
|
||||
print(f"\n{sd_file} decode method:")
|
||||
for i in range(sd_line-2, min(sd_line+40, len(sd_lines))):
|
||||
print(f" {i+1}: {sd_lines[i]}")
|
||||
break
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDiag complete. Next: apply fix.")
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Remove --cpu-vae: Let VAE run on GPU (only 320MB, easily fits). Restart ComfyUI and test."""
|
||||
import paramiko, time, json, textwrap
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
|
||||
|
||||
def sh(cmd, timeout=60):
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
return out.decode(errors='replace').strip()
|
||||
|
||||
def sftp_write(path, content):
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open(path, 'w') as f:
|
||||
f.write(content)
|
||||
sftp.close()
|
||||
|
||||
def sftp_read(path):
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open(path, 'r') as f:
|
||||
data = f.read().decode(errors='replace')
|
||||
sftp.close()
|
||||
return data
|
||||
|
||||
# Kill old
|
||||
print("Killing ComfyUI...")
|
||||
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
|
||||
|
||||
# New launcher WITHOUT --cpu-vae
|
||||
launcher = textwrap.dedent("""\
|
||||
#!/bin/bash
|
||||
export HSA_OVERRIDE_GFX_VERSION=10.1.0
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_ENABLE_SDMA=0
|
||||
export HSA_TOOLS_LIB=""
|
||||
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
|
||||
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
|
||||
export OMP_NUM_THREADS=12
|
||||
export MKL_NUM_THREADS=12
|
||||
export OPENBLAS_NUM_THREADS=12
|
||||
export MIOPEN_FIND_MODE=1
|
||||
|
||||
cd ~/ComfyUI
|
||||
source ~/comfyui-env/bin/activate
|
||||
|
||||
# --novram: model weights on CPU, GPU computes (correct for shared-memory APU)
|
||||
# --force-fp16: half precision
|
||||
# NO --cpu-vae: VAE is only 320MB, runs fine on GPU and much faster
|
||||
exec python3 main.py \\
|
||||
--listen 0.0.0.0 --port 8188 \\
|
||||
--novram \\
|
||||
--force-fp16 \\
|
||||
--disable-smart-memory
|
||||
""")
|
||||
sftp_write('/tmp/run_comfyui.sh', launcher)
|
||||
sh('chmod +x /tmp/run_comfyui.sh')
|
||||
print("Launcher updated: --novram --force-fp16 (NO --cpu-vae)")
|
||||
|
||||
# Start
|
||||
sh('rm -f /tmp/comfyui.log; touch /tmp/comfyui.log')
|
||||
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
|
||||
time.sleep(3)
|
||||
pid = sh('pgrep -f "python3.*main.py"')
|
||||
print(f"PID: {pid}")
|
||||
|
||||
# Wait for ready
|
||||
print("Waiting for server...", end='', flush=True)
|
||||
for i in range(120):
|
||||
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
|
||||
if '200' in code:
|
||||
print(f" READY ({i*2}s)")
|
||||
break
|
||||
if i % 10 == 0 and i > 0:
|
||||
log = sftp_read('/tmp/comfyui.log')
|
||||
lines = [l for l in log.split('\n') if l.strip()]
|
||||
print(f"\n [{i*2}s] {lines[-1][:80] if lines else '...'}", end='', flush=True)
|
||||
else:
|
||||
print('.', end='', flush=True)
|
||||
time.sleep(2)
|
||||
|
||||
# Submit workflow
|
||||
print("\nSubmitting workflow...")
|
||||
workflow = {
|
||||
"prompt": {
|
||||
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
|
||||
"9": {"class_type": "CLIPTextEncode", "inputs": {"text": "", "clip": ["2", 0]}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"model": ["1", 0], "positive": ["4", 0], "negative": ["9", 0],
|
||||
"latent_image": ["5", 0], "seed": 42, "steps": 8, "cfg": 1.0,
|
||||
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
|
||||
}},
|
||||
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
|
||||
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_v2"}}
|
||||
}
|
||||
}
|
||||
sftp_write('/tmp/wf.json', json.dumps(workflow))
|
||||
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json', timeout=10)
|
||||
print(f"Response: {resp[:150]}")
|
||||
|
||||
# Monitor
|
||||
print("\nMonitoring...")
|
||||
t0 = time.time()
|
||||
for i in range(120):
|
||||
elapsed = int(time.time() - t0)
|
||||
|
||||
gpu_temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
|
||||
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
|
||||
|
||||
try:
|
||||
log = sftp_read('/tmp/comfyui.log')
|
||||
except:
|
||||
log = ''
|
||||
|
||||
# Find last meaningful line
|
||||
last = ''
|
||||
sampling = ''
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if '/8' in s and ('it/s' in s or 's/it' in s):
|
||||
sampling = s
|
||||
if s and 'FETCH' not in s and 'startup tasks' not in s and 'DEPRECATION' not in s:
|
||||
last = s
|
||||
|
||||
display = sampling if sampling else last[-100:]
|
||||
print(f" [{elapsed:>4}s] {temp_c}C | {display}")
|
||||
|
||||
# Check output
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_v2*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n*** IMAGE GENERATED! ***")
|
||||
print(f"File: {imgs}")
|
||||
print(f"Total time: {elapsed}s")
|
||||
# Show timing from log
|
||||
for line in log.split('\n')[-15:]:
|
||||
s = line.strip()
|
||||
if s and 'FETCH' not in s and 'startup' not in s:
|
||||
print(f" {s}")
|
||||
break
|
||||
|
||||
# Queue check
|
||||
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
|
||||
try:
|
||||
qd = json.loads(q)
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 30:
|
||||
time.sleep(3)
|
||||
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_v2*.png 2>/dev/null', timeout=5)
|
||||
if imgs:
|
||||
print(f"\n*** IMAGE GENERATED! ***")
|
||||
print(f"File: {imgs}")
|
||||
print(f"Total time: {elapsed}s")
|
||||
else:
|
||||
print(f"\nQueue empty, no image:")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip():
|
||||
print(f" {line.strip()}")
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
|
||||
if alive == 'N':
|
||||
print("\n*** CRASHED ***")
|
||||
for line in log.split('\n')[-30:]:
|
||||
if line.strip():
|
||||
print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
time.sleep(15)
|
||||
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wait for ComfyUI generation to complete on BC-250."""
|
||||
import paramiko
|
||||
import time
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30, desc=""):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
lines = out.strip().split('\n')
|
||||
if len(lines) > 30:
|
||||
print(f" ... ({len(lines)} lines, showing last 30)")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
print(f"STDERR: {err.strip()[-500:]}")
|
||||
print(f" Exit: {rc}")
|
||||
return rc, out, err
|
||||
|
||||
# Monitor every 30s for up to 20 minutes
|
||||
for i in range(40):
|
||||
time.sleep(30)
|
||||
rc, out, _ = run(f"bash -c 'wc -l /home/fabian/comfyui.log; echo \"---\"; tail -5 /home/fabian/comfyui.log'",
|
||||
desc=f"Check {i+1} ({(i+1)*30}s)")
|
||||
|
||||
if 'Prompt executed in' in out:
|
||||
print("\n GENERATION COMPLETE!")
|
||||
run("bash -c 'tail -30 /home/fabian/comfyui.log'", desc="Final log")
|
||||
break
|
||||
if 'Traceback' in out or 'Exception' in out:
|
||||
print("\n ERROR!")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Error log")
|
||||
break
|
||||
|
||||
# Also check process CPU
|
||||
_, pout, _ = run("bash -c 'ps -p 484588 -o %cpu,%mem,vsz,rss --no-headers 2>/dev/null || echo DEAD'")
|
||||
if 'DEAD' in pout:
|
||||
print("\n PROCESS DIED!")
|
||||
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Death log")
|
||||
break
|
||||
|
||||
# Final output check
|
||||
run("bash -c 'ls -la ~/ComfyUI/output/ 2>/dev/null'", desc="Output directory")
|
||||
run("bash -c 'tail -30 /home/fabian/comfyui.log 2>/dev/null'", desc="Final log state")
|
||||
|
||||
ssh.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Monitor ComfyUI - just watch log + GPU until image appears."""
|
||||
import paramiko, time, json
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
|
||||
sftp = c.open_sftp()
|
||||
|
||||
def gpu_info():
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(10)
|
||||
chan.exec_command('/bin/bash -c "cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null; echo SEP; rocm-smi -P 2>&1 | grep Graphics; echo SEP; pgrep -f python3.*main.py >/dev/null && echo ALIVE || echo DEAD"')
|
||||
out = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
out += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
parts = out.decode(errors='replace').split('SEP')
|
||||
temp = parts[0].strip() if len(parts) > 0 else '?'
|
||||
temp_c = int(temp) // 1000 if temp.isdigit() else '?'
|
||||
power = parts[1].strip().split(':')[-1].strip() if len(parts) > 1 else '?'
|
||||
alive = 'ALIVE' in (parts[2] if len(parts) > 2 else '')
|
||||
return temp_c, power, alive
|
||||
|
||||
t0 = time.time()
|
||||
print("Monitoring... GPU should be at high power (>100W) during sampling")
|
||||
|
||||
for i in range(120):
|
||||
elapsed = int(time.time() - t0)
|
||||
temp_c, power, alive = gpu_info()
|
||||
|
||||
# Read log via SFTP
|
||||
try:
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
except:
|
||||
log = ''
|
||||
|
||||
# Find last meaningful line
|
||||
last = ''
|
||||
for line in reversed(log.split('\n')):
|
||||
s = line.strip()
|
||||
if s and 'FETCH' not in s and 'startup tasks' not in s and 'DEPRECATION' not in s:
|
||||
last = s
|
||||
break
|
||||
|
||||
# Check for sampling progress in log
|
||||
sampling = ''
|
||||
for line in log.split('\n'):
|
||||
if '/8' in line and ('it/s' in line or 's/it' in line):
|
||||
sampling = line.strip()
|
||||
|
||||
# Check output
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(5)
|
||||
chan.exec_command('/bin/bash -c "ls ~/ComfyUI/output/*.png 2>/dev/null"')
|
||||
imgs = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
imgs += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
imgs = imgs.decode().strip()
|
||||
|
||||
line_out = f"[{elapsed:>4}s] {temp_c}C {power} | "
|
||||
if sampling:
|
||||
line_out += sampling[-80:]
|
||||
else:
|
||||
line_out += last[-80:]
|
||||
print(line_out)
|
||||
|
||||
if imgs:
|
||||
print(f"\n*** IMAGE GENERATED! ***")
|
||||
print(f"File: {imgs}")
|
||||
print(f"Time: {elapsed}s")
|
||||
# Print last 15 log lines
|
||||
for line in log.split('\n')[-15:]:
|
||||
if line.strip():
|
||||
print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
if not alive:
|
||||
print("\n*** PROCESS DEAD ***")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip():
|
||||
print(f" {line.strip()}")
|
||||
break
|
||||
|
||||
# Check queue empty
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(5)
|
||||
chan.exec_command('/bin/bash -c "curl -s http://127.0.0.1:8188/queue 2>/dev/null"')
|
||||
qraw = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
qraw += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
try:
|
||||
qd = json.loads(qraw.decode())
|
||||
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 30:
|
||||
time.sleep(2)
|
||||
# Final image check
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(5)
|
||||
chan.exec_command('/bin/bash -c "ls ~/ComfyUI/output/*.png 2>/dev/null"')
|
||||
imgs2 = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = chan.recv(65536)
|
||||
if not chunk: break
|
||||
imgs2 += chunk
|
||||
except: break
|
||||
chan.close()
|
||||
if imgs2.decode().strip():
|
||||
print(f"\n*** IMAGE GENERATED! ***")
|
||||
print(f"File: {imgs2.decode().strip()}")
|
||||
print(f"Time: {elapsed}s")
|
||||
else:
|
||||
print(f"\nQueue empty, no image. Last log lines:")
|
||||
for line in log.split('\n')[-20:]:
|
||||
if line.strip():
|
||||
print(f" {line.strip()}")
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
time.sleep(15)
|
||||
|
||||
sftp.close()
|
||||
c.close()
|
||||
print("\nDone.")
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Diagnose: why is KSampler on CPU?"""
|
||||
import paramiko
|
||||
|
||||
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
||||
sftp = c.open_sftp()
|
||||
|
||||
print("=== LOG (meaningful) ===")
|
||||
with sftp.open('/tmp/comfyui.log', 'r') as f:
|
||||
log = f.read().decode(errors='replace')
|
||||
for line in log.split('\n'):
|
||||
s = line.strip()
|
||||
if not s or 'FETCH' in s or 'startup tasks' in s or 'DEPRECATION' in s:
|
||||
continue
|
||||
print(s)
|
||||
|
||||
print("\n=== LAUNCHER ===")
|
||||
for p in ['/tmp/run_comfyui.sh', '/home/fabian/start_comfyui.sh']:
|
||||
try:
|
||||
with sftp.open(p, 'r') as f:
|
||||
print(f"--- {p} ---")
|
||||
print(f.read().decode())
|
||||
break
|
||||
except: pass
|
||||
|
||||
print("\n=== PROCESS ===")
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(10)
|
||||
chan.exec_command('/bin/bash -c "ps aux | grep main.py | grep -v grep"')
|
||||
o = b""
|
||||
while True:
|
||||
try:
|
||||
ch = chan.recv(65536)
|
||||
if not ch: break
|
||||
o += ch
|
||||
except: break
|
||||
chan.close()
|
||||
print(o.decode(errors='replace'))
|
||||
|
||||
print("=== GPU SPEED TEST ===")
|
||||
ts = '#!/bin/bash\nexport HSA_OVERRIDE_GFX_VERSION=10.1.0\nexport HIP_VISIBLE_DEVICES=0\nexport HSA_ENABLE_SDMA=0\nsource ~/comfyui-env/bin/activate\npython3 -c "\nimport torch,time\nprint(\"CUDA:\",torch.cuda.is_available())\nif torch.cuda.is_available():\n print(\"Dev:\",torch.cuda.get_device_name(0))\n x=torch.randn(2048,2048,device=\"cuda\",dtype=torch.float16)\n torch.cuda.synchronize()\n t=time.time()\n for _ in range(10): y=x@x\n torch.cuda.synchronize()\n gt=time.time()-t\n x2=torch.randn(2048,2048,dtype=torch.float16)\n t=time.time()\n for _ in range(10): y=x2@x2\n ct=time.time()-t\n print(f\"GPU:{gt:.3f}s CPU:{ct:.3f}s Ratio:{ct/gt:.1f}x\")\n"\n'
|
||||
with sftp.open('/tmp/gtest.sh', 'w') as f:
|
||||
f.write(ts)
|
||||
sftp.close()
|
||||
chan = c.get_transport().open_session()
|
||||
chan.settimeout(60)
|
||||
chan.exec_command('/bin/bash /tmp/gtest.sh')
|
||||
o = b""
|
||||
while True:
|
||||
try:
|
||||
ch = chan.recv(65536)
|
||||
if not ch: break
|
||||
o += ch
|
||||
except: break
|
||||
chan.close()
|
||||
print(o.decode(errors='replace'))
|
||||
|
||||
c.close()
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BC-250 v3 amdgpu kernel module build script - Phase 1: Download & Prepare
|
||||
"""
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
KERNEL_VER = "6.19.6"
|
||||
KERNEL_FULL = "6.19.6-2-cachyos"
|
||||
BUILD_DIR = "/home/fabian/kernel-build"
|
||||
SRC_DIR = f"{BUILD_DIR}/linux-{KERNEL_VER}"
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, desc="", timeout=600):
|
||||
if desc:
|
||||
print(f"\n=== {desc} ===")
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out:
|
||||
lines = out.split('\n')
|
||||
if len(lines) > 60:
|
||||
print('\n'.join(lines[:25]))
|
||||
print(f" ... ({len(lines)-50} lines omitted) ...")
|
||||
print('\n'.join(lines[-25:]))
|
||||
else:
|
||||
print(out)
|
||||
if err and rc != 0:
|
||||
print(f"STDERR: {err[:500]}")
|
||||
if rc != 0:
|
||||
print(f"EXIT CODE: {rc}")
|
||||
return out, rc
|
||||
|
||||
# Step 1: Download kernel source if not present
|
||||
out, rc = run(f"test -d {SRC_DIR} && echo EXISTS || echo MISSING")
|
||||
if "EXISTS" in out:
|
||||
print(f"Kernel source already at {SRC_DIR}")
|
||||
else:
|
||||
run(f"mkdir -p {BUILD_DIR}", "Creating build directory")
|
||||
run(f"cd {BUILD_DIR} && curl -LO https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-{KERNEL_VER}.tar.xz",
|
||||
f"Downloading linux-{KERNEL_VER}.tar.xz", timeout=600)
|
||||
run(f"cd {BUILD_DIR} && tar xf linux-{KERNEL_VER}.tar.xz",
|
||||
"Extracting kernel source", timeout=300)
|
||||
run(f"rm -f {BUILD_DIR}/linux-{KERNEL_VER}.tar.xz")
|
||||
|
||||
# Step 2: Prepare build environment
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/.config {SRC_DIR}/", "Copying .config")
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/Module.symvers {SRC_DIR}/", "Copying Module.symvers")
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/localversion.10-pkgrel {SRC_DIR}/", "Copying localversion pkgrel")
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/localversion.20-pkgname {SRC_DIR}/", "Copying localversion pkgname")
|
||||
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 olddefconfig 2>&1 | tail -5", "olddefconfig", timeout=120)
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 modules_prepare 2>&1 | tail -10", "modules_prepare", timeout=120)
|
||||
|
||||
# Verify
|
||||
print("\n=== Source files ===")
|
||||
AMDGPU = f"{SRC_DIR}/drivers/gpu/drm/amd/amdgpu"
|
||||
for f in ["gfx_v10_0.c", "gmc_v10_0.c", "amdgpu_gmc.c"]:
|
||||
out, _ = run(f"wc -l {AMDGPU}/{f}")
|
||||
print(f" {f}: {out.split()[0]} lines")
|
||||
|
||||
print("\n=== Phase 1 complete: source downloaded and prepared ===")
|
||||
ssh.close()
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 2: Upload patches, apply them, build and install the amdgpu module."""
|
||||
import paramiko
|
||||
import sys
|
||||
import os
|
||||
|
||||
KERNEL_VER = "6.19.6"
|
||||
KERNEL_FULL = "6.19.6-2-cachyos"
|
||||
SRC_DIR = f"/home/fabian/kernel-build/linux-{KERNEL_VER}"
|
||||
AMDGPU_DIR = f"{SRC_DIR}/drivers/gpu/drm/amd/amdgpu"
|
||||
MODULE_DIR = f"/usr/lib/modules/{KERNEL_FULL}/kernel/drivers/gpu/drm/amd/amdgpu"
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
sftp = ssh.open_sftp()
|
||||
|
||||
def run(cmd, desc="", timeout=600):
|
||||
if desc:
|
||||
print(f"\n=== {desc} ===")
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out:
|
||||
lines = out.split('\n')
|
||||
if len(lines) > 80:
|
||||
print('\n'.join(lines[:30]))
|
||||
print(f" ... ({len(lines)-60} lines omitted) ...")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(out)
|
||||
if err and rc != 0:
|
||||
print(f"STDERR: {err[:1000]}")
|
||||
if rc != 0:
|
||||
print(f"EXIT CODE: {rc}")
|
||||
return out, rc
|
||||
|
||||
# Upload patch scripts
|
||||
desktop = r'C:\Users\fabia\Desktop'
|
||||
for fname in ['patch1_gfxoff.py', 'patch2_gmc.py', 'patch3_amdgpu_gmc.py']:
|
||||
local = os.path.join(desktop, fname)
|
||||
remote = f'/tmp/{fname}'
|
||||
sftp.put(local, remote)
|
||||
print(f"Uploaded {fname}")
|
||||
|
||||
# Apply patches
|
||||
run("python3 /tmp/patch1_gfxoff.py", "Patch 1: gfx_v10_0.c - GFXOFF disable")
|
||||
run("python3 /tmp/patch2_gmc.py", "Patch 2: gmc_v10_0.c - KIQ bypass + dead-GPU")
|
||||
run("python3 /tmp/patch3_amdgpu_gmc.py", "Patch 3: amdgpu_gmc.c - KIQ bypass + dead-GPU")
|
||||
|
||||
# Verify patches
|
||||
print("\n=== Patch verification: BC-250 markers ===")
|
||||
for f in ["gfx_v10_0.c", "gmc_v10_0.c", "amdgpu_gmc.c"]:
|
||||
out, _ = run(f"grep -c 'BC-250' {AMDGPU_DIR}/{f}")
|
||||
print(f" {f}: {out} BC-250 references")
|
||||
|
||||
# Build the module
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 -j$(nproc) M=drivers/gpu/drm/amd/amdgpu modules 2>&1",
|
||||
"Building amdgpu module (this takes a few minutes)", timeout=900)
|
||||
|
||||
# Check if build succeeded
|
||||
out, rc = run(f"test -f {AMDGPU_DIR}/amdgpu.ko && echo SUCCESS || echo FAILED")
|
||||
if "FAILED" in out:
|
||||
print("\nERROR: Module build failed!")
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 -j1 M=drivers/gpu/drm/amd/amdgpu modules 2>&1 | grep -i error | head -20",
|
||||
"Build errors")
|
||||
sys.exit(1)
|
||||
|
||||
# Strip and compress
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko", "Module size before strip")
|
||||
run(f"strip --strip-debug {AMDGPU_DIR}/amdgpu.ko", "Stripping debug info")
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko", "Module size after strip")
|
||||
run(f"zstd -19 -f {AMDGPU_DIR}/amdgpu.ko", "Compressing with zstd-19", timeout=120)
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko.zst", "Compressed module size")
|
||||
|
||||
# Backup original and install
|
||||
run(f"sudo cp {MODULE_DIR}/amdgpu.ko.zst {MODULE_DIR}/amdgpu.ko.zst.original",
|
||||
"Backing up stock module")
|
||||
run(f"sudo cp {AMDGPU_DIR}/amdgpu.ko.zst {MODULE_DIR}/amdgpu.ko.zst",
|
||||
"Installing v3 patched module")
|
||||
run("sudo depmod -a", "Updating module dependencies")
|
||||
|
||||
# Verify
|
||||
run(f"zstd -d -c {MODULE_DIR}/amdgpu.ko.zst | strings | grep 'BC-250'",
|
||||
"Verifying BC-250 strings in installed module")
|
||||
|
||||
# Clean up
|
||||
run("rm -f /tmp/patch1_gfxoff.py /tmp/patch2_gmc.py /tmp/patch3_amdgpu_gmc.py")
|
||||
|
||||
print("\n" + "="*60)
|
||||
print(" v3 PATCHED MODULE BUILT AND INSTALLED")
|
||||
print(" A reboot is required to load the new module.")
|
||||
print("="*60)
|
||||
|
||||
sftp.close()
|
||||
ssh.close()
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 3: Build v3 amdgpu module from CachyOS kernel source."""
|
||||
import paramiko
|
||||
import sys
|
||||
import os
|
||||
|
||||
KERNEL_FULL = "6.19.6-2-cachyos"
|
||||
BUILD_DIR = "/home/fabian/kernel-build"
|
||||
SRC_DIR = f"{BUILD_DIR}/cachyos-6.19.6-1"
|
||||
AMDGPU_DIR = f"{SRC_DIR}/drivers/gpu/drm/amd/amdgpu"
|
||||
MODULE_DIR = f"/usr/lib/modules/{KERNEL_FULL}/kernel/drivers/gpu/drm/amd/amdgpu"
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
sftp = ssh.open_sftp()
|
||||
|
||||
def run(cmd, desc="", timeout=600):
|
||||
if desc:
|
||||
print(f"\n=== {desc} ===")
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out:
|
||||
lines = out.split('\n')
|
||||
if len(lines) > 80:
|
||||
print('\n'.join(lines[:30]))
|
||||
print(f" ... ({len(lines)-60} lines omitted) ...")
|
||||
print('\n'.join(lines[-30:]))
|
||||
else:
|
||||
print(out)
|
||||
if err and rc != 0:
|
||||
print(f"STDERR: {err[:1000]}")
|
||||
if rc != 0:
|
||||
print(f"EXIT CODE: {rc}")
|
||||
return out, rc
|
||||
|
||||
# Step 1: Setup build environment
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/.config {SRC_DIR}/",
|
||||
"Copying CachyOS .config")
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/Module.symvers {SRC_DIR}/",
|
||||
"Copying Module.symvers")
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/localversion.10-pkgrel {SRC_DIR}/",
|
||||
"Copying localversion.10-pkgrel")
|
||||
run(f"cp /usr/lib/modules/{KERNEL_FULL}/build/localversion.20-pkgname {SRC_DIR}/",
|
||||
"Copying localversion.20-pkgname")
|
||||
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 olddefconfig 2>&1 | tail -5",
|
||||
"Running olddefconfig", timeout=120)
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 modules_prepare 2>&1 | tail -10",
|
||||
"Running modules_prepare", timeout=300)
|
||||
|
||||
# Step 2: Apply patches (upload and run)
|
||||
# Copy patch scripts from vanilla tree patching (they use the same logic)
|
||||
desktop = r'C:\Users\fabia\Desktop'
|
||||
|
||||
# Upload patch scripts - update them for new source dir
|
||||
for fname in ['patch1_gfxoff.py', 'patch2_gmc.py', 'patch3_amdgpu_gmc.py']:
|
||||
local = os.path.join(desktop, fname)
|
||||
with open(local, 'r') as f:
|
||||
content = f.read()
|
||||
# Replace old path with new CachyOS path
|
||||
content = content.replace(
|
||||
'/home/fabian/kernel-build/linux-6.19.6',
|
||||
'/home/fabian/kernel-build/cachyos-6.19.6-1'
|
||||
)
|
||||
with sftp.open(f'/tmp/{fname}', 'w') as f:
|
||||
f.write(content)
|
||||
print(f"Uploaded {fname} (updated path)")
|
||||
|
||||
run("python3 /tmp/patch1_gfxoff.py", "Patch 1: gfx_v10_0.c - GFXOFF disable")
|
||||
run("python3 /tmp/patch2_gmc.py", "Patch 2: gmc_v10_0.c - KIQ bypass + dead-GPU")
|
||||
run("python3 /tmp/patch3_amdgpu_gmc.py", "Patch 3: amdgpu_gmc.c - KIQ bypass + dead-GPU")
|
||||
|
||||
# Verify patches
|
||||
for f in ["gfx_v10_0.c", "gmc_v10_0.c", "amdgpu_gmc.c"]:
|
||||
out, _ = run(f"grep -c 'BC-250' {AMDGPU_DIR}/{f}")
|
||||
print(f" {f}: {out} BC-250 refs")
|
||||
|
||||
# Step 3: Build the module
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 -j$(nproc) M=drivers/gpu/drm/amd/amdgpu modules 2>&1 | tail -40",
|
||||
"Building amdgpu module from CachyOS source", timeout=900)
|
||||
|
||||
# Check build
|
||||
out, rc = run(f"test -f {AMDGPU_DIR}/amdgpu.ko && echo SUCCESS || echo FAILED")
|
||||
if "FAILED" in out:
|
||||
print("\nBuild failed! Checking errors...")
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 -j1 M=drivers/gpu/drm/amd/amdgpu modules 2>&1 | grep -i error | head -20")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 4: Strip, compress, install
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko", "Before strip")
|
||||
run(f"strip --strip-debug {AMDGPU_DIR}/amdgpu.ko", "Stripping")
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko", "After strip")
|
||||
run(f"zstd -19 -f {AMDGPU_DIR}/amdgpu.ko", "Compressing zstd-19", timeout=120)
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko.zst", "Compressed size")
|
||||
|
||||
# Install
|
||||
run(f"sudo cp {AMDGPU_DIR}/amdgpu.ko.zst {MODULE_DIR}/amdgpu.ko.zst",
|
||||
"Installing v3 patched module")
|
||||
run("sudo depmod -a", "depmod -a")
|
||||
|
||||
# Rebuild initramfs
|
||||
run("sudo limine-update 2>&1 | tail -10", "Rebuilding initramfs", timeout=120)
|
||||
|
||||
# Verify
|
||||
run(f"zstd -d -c {MODULE_DIR}/amdgpu.ko.zst | strings | grep 'BC-250'",
|
||||
"BC-250 strings in installed module")
|
||||
|
||||
# Cleanup
|
||||
run("rm -f /tmp/patch1_gfxoff.py /tmp/patch2_gmc.py /tmp/patch3_amdgpu_gmc.py")
|
||||
run(f"rm -f {BUILD_DIR}/cachyos-6.19.6-1.tar.gz")
|
||||
|
||||
print("\n" + "="*60)
|
||||
print(" v3 MODULE BUILT FROM CachyOS SOURCE AND INSTALLED")
|
||||
print(" Reboot required.")
|
||||
print("="*60)
|
||||
|
||||
sftp.close()
|
||||
ssh.close()
|
||||
@@ -0,0 +1,65 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
# Connect to BC250
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
sftp = c.open_sftp()
|
||||
|
||||
# Upload hip_probe.cpp
|
||||
with open(r'c:\Users\fabia\Desktop\VibeROCm\hip_probe.cpp', 'r') as f:
|
||||
probe_src = f.read()
|
||||
with sftp.open('/tmp/hip_probe.cpp', 'w') as f:
|
||||
f.write(probe_src)
|
||||
print("Uploaded hip_probe.cpp")
|
||||
|
||||
# Upload hip_minimal_test.cpp
|
||||
with open(r'c:\Users\fabia\Desktop\VibeROCm\hip_minimal_test.cpp', 'r') as f:
|
||||
minimal_src = f.read()
|
||||
with sftp.open('/tmp/hip_minimal_test.cpp', 'w') as f:
|
||||
f.write(minimal_src)
|
||||
print("Uploaded hip_minimal_test.cpp")
|
||||
|
||||
# Upload hip_vector_add.cpp
|
||||
with open(r'c:\Users\fabia\Desktop\VibeROCm\hip_vector_add.cpp', 'r') as f:
|
||||
vector_src = f.read()
|
||||
with sftp.open('/tmp/hip_vector_add.cpp', 'w') as f:
|
||||
f.write(vector_src)
|
||||
print("Uploaded hip_vector_add.cpp")
|
||||
|
||||
sftp.close()
|
||||
|
||||
# Compile all three
|
||||
def run_cmd(client, cmd):
|
||||
stdin, stdout, stderr = client.exec_command(cmd, timeout=120)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
return out, err, rc
|
||||
|
||||
env = "HSA_OVERRIDE_GFX_VERSION=10.1.0 HIP_VISIBLE_DEVICES=0 HSA_ENABLE_SDMA=0 HSA_TOOLS_LIB= HSA_TOOLS_REPORT_LOAD_FAILURE=0"
|
||||
|
||||
# Compile hip_probe
|
||||
print("\nCompiling hip_probe...")
|
||||
out, err, rc = run_cmd(c, f"bash -c 'export {env}; /opt/rocm/bin/hipcc --offload-arch=gfx1010 -o /tmp/hip_probe /tmp/hip_probe.cpp 2>&1'")
|
||||
print(f" Exit code: {rc}")
|
||||
if rc != 0:
|
||||
print(f" Error: {out}{err}")
|
||||
|
||||
# Compile hip_minimal_test
|
||||
print("Compiling hip_minimal_test...")
|
||||
out, err, rc = run_cmd(c, f"bash -c 'export {env}; /opt/rocm/bin/hipcc --offload-arch=gfx1010 -o /tmp/hip_minimal_test /tmp/hip_minimal_test.cpp 2>&1'")
|
||||
print(f" Exit code: {rc}")
|
||||
if rc != 0:
|
||||
print(f" Error: {out}{err}")
|
||||
|
||||
# Compile hip_vector_add
|
||||
print("Compiling hip_vector_add...")
|
||||
out, err, rc = run_cmd(c, f"bash -c 'export {env}; /opt/rocm/bin/hipcc --offload-arch=gfx1010 -o /tmp/hip_vector_add /tmp/hip_vector_add.cpp 2>&1'")
|
||||
print(f" Exit code: {rc}")
|
||||
if rc != 0:
|
||||
print(f" Error: {out}{err}")
|
||||
|
||||
c.close()
|
||||
print("\nAll compilations done.")
|
||||
@@ -0,0 +1,42 @@
|
||||
import paramiko
|
||||
|
||||
HIP_CODE = '''#include <hip/hip_runtime.h>
|
||||
#include <stdio.h>
|
||||
__global__ void vectorAdd(float *a, float *b, float *c, int n) {
|
||||
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i < n) c[i] = a[i] + b[i];
|
||||
}
|
||||
int main() {
|
||||
const int N = 1024;
|
||||
size_t sz = N * sizeof(float);
|
||||
float *h_a = (float*)malloc(sz), *h_b = (float*)malloc(sz), *h_c = (float*)malloc(sz);
|
||||
float *d_a, *d_b, *d_c;
|
||||
for (int i = 0; i < N; i++) { h_a[i] = i; h_b[i] = i * 2; }
|
||||
hipMalloc(&d_a, sz); hipMalloc(&d_b, sz); hipMalloc(&d_c, sz);
|
||||
hipMemcpy(d_a, h_a, sz, hipMemcpyHostToDevice);
|
||||
hipMemcpy(d_b, h_b, sz, hipMemcpyHostToDevice);
|
||||
vectorAdd<<<(N+255)/256, 256>>>(d_a, d_b, d_c, N);
|
||||
hipMemcpy(h_c, d_c, sz, hipMemcpyDeviceToHost);
|
||||
hipDeviceSynchronize();
|
||||
hipError_t err = hipGetLastError();
|
||||
if (err != hipSuccess) { printf("HIP ERROR: %s\\n", hipGetErrorString(err)); return 1; }
|
||||
int ok = 1;
|
||||
for (int i = 0; i < N; i++) {
|
||||
if (h_c[i] != h_a[i] + h_b[i]) { ok = 0; printf("MISMATCH at %d\\n", i); break; }
|
||||
}
|
||||
if (ok) printf("HIP COMPUTE TEST PASSED: %d elements verified\\n", N);
|
||||
hipFree(d_a); hipFree(d_b); hipFree(d_c);
|
||||
free(h_a); free(h_b); free(h_c);
|
||||
return 0;
|
||||
}
|
||||
'''
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
sftp = c.open_sftp()
|
||||
with sftp.open('/tmp/hip_test.cpp', 'w') as f:
|
||||
f.write(HIP_CODE)
|
||||
sftp.close()
|
||||
c.close()
|
||||
print('HIP test file uploaded via SFTP')
|
||||
@@ -0,0 +1,416 @@
|
||||
{
|
||||
"last_node_id": 9,
|
||||
"last_link_id": 9,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "UnetLoaderGGUF",
|
||||
"pos": [
|
||||
100,
|
||||
100
|
||||
],
|
||||
"size": [
|
||||
300,
|
||||
80
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "MODEL",
|
||||
"type": "MODEL",
|
||||
"links": [
|
||||
1
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "UnetLoaderGGUF"
|
||||
},
|
||||
"widgets_values": [
|
||||
"z_image_turbo-Q5_K_S.gguf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPLoaderGGUF",
|
||||
"pos": [
|
||||
100,
|
||||
250
|
||||
],
|
||||
"size": [
|
||||
300,
|
||||
80
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CLIP",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
2
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CLIPLoaderGGUF"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-4B.i1-Q5_K_S.gguf",
|
||||
"qwen_image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "VAELoader",
|
||||
"pos": [
|
||||
100,
|
||||
400
|
||||
],
|
||||
"size": [
|
||||
300,
|
||||
60
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "VAE",
|
||||
"type": "VAE",
|
||||
"links": [
|
||||
3
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VAELoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"ae.safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "CLIPTextEncode",
|
||||
"pos": [
|
||||
500,
|
||||
250
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
120
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": 2
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CONDITIONING",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
4
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CLIPTextEncode"
|
||||
},
|
||||
"widgets_values": [
|
||||
"A red fox in a snowy forest, photorealistic, highly detailed"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "EmptyLatentImage",
|
||||
"pos": [
|
||||
500,
|
||||
450
|
||||
],
|
||||
"size": [
|
||||
300,
|
||||
110
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "LATENT",
|
||||
"type": "LATENT",
|
||||
"links": [
|
||||
5
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "EmptyLatentImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
512,
|
||||
512,
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "KSampler",
|
||||
"pos": [
|
||||
950,
|
||||
100
|
||||
],
|
||||
"size": [
|
||||
320,
|
||||
474
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MODEL",
|
||||
"link": 1
|
||||
},
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"link": 4
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"link": 8
|
||||
},
|
||||
{
|
||||
"name": "latent_image",
|
||||
"type": "LATENT",
|
||||
"link": 5
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "LATENT",
|
||||
"type": "LATENT",
|
||||
"links": [
|
||||
6
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "KSampler"
|
||||
},
|
||||
"widgets_values": [
|
||||
42,
|
||||
"fixed",
|
||||
8,
|
||||
1.0,
|
||||
"euler",
|
||||
"simple",
|
||||
1.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "VAEDecode",
|
||||
"pos": [
|
||||
1350,
|
||||
100
|
||||
],
|
||||
"size": [
|
||||
210,
|
||||
50
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "samples",
|
||||
"type": "LATENT",
|
||||
"link": 6
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 3
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
7
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VAEDecode"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "SaveImage",
|
||||
"pos": [
|
||||
1350,
|
||||
250
|
||||
],
|
||||
"size": [
|
||||
320,
|
||||
270
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 7
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SaveImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
"ZImageTurbo"
|
||||
]
|
||||
}
|
||||
,
|
||||
{
|
||||
"id": 9,
|
||||
"type": "CLIPTextEncode",
|
||||
"pos": [
|
||||
500,
|
||||
420
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
120
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": 9
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CONDITIONING",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
8
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CLIPTextEncode"
|
||||
},
|
||||
"widgets_values": [
|
||||
""
|
||||
],
|
||||
"title": "Negative Prompt"
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
6,
|
||||
0,
|
||||
"MODEL"
|
||||
],
|
||||
[
|
||||
2,
|
||||
2,
|
||||
0,
|
||||
4,
|
||||
0,
|
||||
"CLIP"
|
||||
],
|
||||
[
|
||||
3,
|
||||
3,
|
||||
0,
|
||||
7,
|
||||
1,
|
||||
"VAE"
|
||||
],
|
||||
[
|
||||
4,
|
||||
4,
|
||||
0,
|
||||
6,
|
||||
1,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
5,
|
||||
5,
|
||||
0,
|
||||
6,
|
||||
3,
|
||||
"LATENT"
|
||||
],
|
||||
[
|
||||
6,
|
||||
6,
|
||||
0,
|
||||
7,
|
||||
0,
|
||||
"LATENT"
|
||||
],
|
||||
[
|
||||
7,
|
||||
7,
|
||||
0,
|
||||
8,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
8,
|
||||
9,
|
||||
0,
|
||||
6,
|
||||
2,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
9,
|
||||
2,
|
||||
0,
|
||||
9,
|
||||
0,
|
||||
"CLIP"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Patch 1: gfx_v10_0.c - Disable GFXOFF for Cyan Skillfish (IP 10.1.3)"""
|
||||
import sys
|
||||
|
||||
AMDGPU = "/home/fabian/kernel-build/linux-6.19.6/drivers/gpu/drm/amd/amdgpu"
|
||||
filepath = f"{AMDGPU}/gfx_v10_0.c"
|
||||
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
if 'BC-250' in content:
|
||||
print("Already patched, skipping.")
|
||||
sys.exit(0)
|
||||
|
||||
# Find gfx_v10_0_check_gfxoff_flag function
|
||||
func_start = content.find('static void gfx_v10_0_check_gfxoff_flag')
|
||||
if func_start == -1:
|
||||
print("ERROR: gfx_v10_0_check_gfxoff_flag not found")
|
||||
sys.exit(1)
|
||||
|
||||
# Find the 'default:' case in the switch inside this function
|
||||
# Search within a reasonable range from function start
|
||||
func_region_end = func_start + 2000
|
||||
default_pos = content.find('\tdefault:', func_start, func_region_end)
|
||||
if default_pos == -1:
|
||||
# Try with spaces instead of tabs
|
||||
default_pos = content.find('default:', func_start, func_region_end)
|
||||
if default_pos == -1:
|
||||
print("ERROR: default case not found in gfx_v10_0_check_gfxoff_flag")
|
||||
print("Region:", content[func_start:func_start+500])
|
||||
sys.exit(1)
|
||||
|
||||
# Insert our case BEFORE the default case
|
||||
new_case = (
|
||||
'\t/* ===== BC-250 v3 PATCH START ===== */\n'
|
||||
'\tcase IP_VERSION(10, 1, 3):\n'
|
||||
'\t\t/*\n'
|
||||
'\t\t * BC-250 / Cyan Skillfish (gfx1013): GFXOFF causes the GPU to\n'
|
||||
'\t\t * enter a power-saving state from which it cannot reliably wake.\n'
|
||||
'\t\t * Unconditionally disable GFXOFF to prevent GPU hangs.\n'
|
||||
'\t\t */\n'
|
||||
'\t\tadev->pm.pp_feature &= ~PP_GFXOFF_MASK;\n'
|
||||
'\t\tdev_info(adev->dev,\n'
|
||||
'\t\t\t "BC-250: GFXOFF disabled to prevent GPU power-state hangs\\n");\n'
|
||||
'\t\tbreak;\n'
|
||||
'\t/* ===== BC-250 v3 PATCH END ===== */\n'
|
||||
)
|
||||
|
||||
content = content[:default_pos] + new_case + content[default_pos:]
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("Patch 1 applied: gfx_v10_0.c - GFXOFF disable for Cyan Skillfish")
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Patch 2: gmc_v10_0.c - KIQ bypass + Dead-GPU detection (5 sub-patches)"""
|
||||
import sys
|
||||
|
||||
AMDGPU = "/home/fabian/kernel-build/linux-6.19.6/drivers/gpu/drm/amd/amdgpu"
|
||||
filepath = f"{AMDGPU}/gmc_v10_0.c"
|
||||
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
if 'BC-250' in content:
|
||||
print("Already patched, skipping.")
|
||||
sys.exit(0)
|
||||
|
||||
# ========================================
|
||||
# Patch 2a: KIQ bypass in gmc_v10_0_flush_gpu_tlb
|
||||
# Insert BEFORE the KIQ check: if (adev->gfx.kiq[0].ring.sched.ready
|
||||
# ========================================
|
||||
flush_func = content.find('gmc_v10_0_flush_gpu_tlb(struct amdgpu_device')
|
||||
if flush_func == -1:
|
||||
print("ERROR: gmc_v10_0_flush_gpu_tlb not found")
|
||||
sys.exit(1)
|
||||
|
||||
# Find the KIQ readiness check
|
||||
kiq_check = content.find('adev->gfx.kiq[0].ring.sched.ready', flush_func)
|
||||
if kiq_check == -1:
|
||||
# Try alternate patterns
|
||||
kiq_check = content.find('kiq[0].ring.sched.ready', flush_func)
|
||||
if kiq_check == -1:
|
||||
print("ERROR: KIQ readiness check not found in flush_gpu_tlb")
|
||||
sys.exit(1)
|
||||
|
||||
# Go back to the 'if' statement start
|
||||
if_start = content.rfind('if (', flush_func, kiq_check)
|
||||
if if_start == -1:
|
||||
if_start = kiq_check
|
||||
|
||||
# Find start of line
|
||||
line_start = content.rfind('\n', 0, if_start) + 1
|
||||
|
||||
bypass = (
|
||||
'\t/* ===== BC-250 v2 PATCH: KIQ bypass ===== */\n'
|
||||
'\t/* BC-250 / Cyan Skillfish (gfx1013): KIQ ring TLB flush hangs.\n'
|
||||
'\t * Skip to direct MMIO register path. Widen to all gfx10.1.x.\n'
|
||||
'\t */\n'
|
||||
'\t{\n'
|
||||
'\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);\n'
|
||||
'\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0)))\n'
|
||||
'\t\t\tgoto use_mmio;\n'
|
||||
'\t}\n'
|
||||
'\t/* ===== BC-250 v2 PATCH END ===== */\n'
|
||||
)
|
||||
|
||||
content = content[:line_start] + bypass + content[line_start:]
|
||||
|
||||
# ========================================
|
||||
# Now add the 'use_mmio:' label before the MMIO path
|
||||
# Find "hub_ip = (vmhub ==" which starts the MMIO code path
|
||||
# ========================================
|
||||
hub_ip_assign = content.find('hub_ip = (vmhub ==', flush_func)
|
||||
if hub_ip_assign == -1:
|
||||
# In newer kernels it might be different
|
||||
hub_ip_assign = content.find('hub_ip =', flush_func)
|
||||
if hub_ip_assign == -1:
|
||||
print("ERROR: hub_ip assignment not found")
|
||||
sys.exit(1)
|
||||
|
||||
# Check if there's already a use_mmio label
|
||||
hub_line_start = content.rfind('\n', 0, hub_ip_assign) + 1
|
||||
preceding_text = content[hub_line_start - 50:hub_line_start].strip()
|
||||
if 'use_mmio' not in preceding_text:
|
||||
# Find the comment before the MMIO path to place label after it
|
||||
mmio_comment = content.rfind('/*', flush_func, hub_ip_assign)
|
||||
comment_block_start = content.rfind('\n', 0, mmio_comment) + 1 if mmio_comment > flush_func else hub_line_start
|
||||
|
||||
# Insert use_mmio label
|
||||
content = content[:hub_line_start] + 'use_mmio:\n' + content[hub_line_start:]
|
||||
|
||||
# ========================================
|
||||
# Patch 2b: Pre-spinlock health check after hub_ip assignment
|
||||
# ========================================
|
||||
# Re-find hub_ip
|
||||
hub_ip_assign = content.find('hub_ip = (vmhub ==', flush_func)
|
||||
if hub_ip_assign == -1:
|
||||
hub_ip_assign = content.find('hub_ip =', flush_func)
|
||||
hub_line_end = content.find('\n', hub_ip_assign)
|
||||
|
||||
# Check for second line (MMHUB case) - the ternary might be split across 2 lines
|
||||
next_line = content[hub_line_end+1:hub_line_end+100]
|
||||
if next_line.strip().startswith(':') or next_line.strip().startswith('?'):
|
||||
hub_line_end = content.find('\n', hub_line_end + 1)
|
||||
|
||||
health_check = (
|
||||
'\n'
|
||||
'\t/* ===== BC-250 v3 PATCH: Pre-spinlock health check ===== */\n'
|
||||
'\t{\n'
|
||||
'\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);\n'
|
||||
'\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) &&\n'
|
||||
'\t\t (gc_ver < IP_VERSION(10, 2, 0))) {\n'
|
||||
'\t\t\ttmp = RREG32_RLC_NO_KIQ(ack, hub_ip);\n'
|
||||
'\t\t\tif (tmp == 0xFFFFFFFF) {\n'
|
||||
'\t\t\t\tdev_err_ratelimited(adev->dev,\n'
|
||||
'\t\t\t\t\t"BC-250: GPU unreachable (MMIO 0xFFFFFFFF), "\n'
|
||||
'\t\t\t\t\t"skipping TLB flush vmid=%u hub=%u\\n",\n'
|
||||
'\t\t\t\t\tvmid, vmhub);\n'
|
||||
'\t\t\t\treturn;\n'
|
||||
'\t\t\t}\n'
|
||||
'\t\t}\n'
|
||||
'\t}\n'
|
||||
'\t/* ===== BC-250 v3 PATCH END ===== */\n'
|
||||
)
|
||||
content = content[:hub_line_end] + health_check + content[hub_line_end:]
|
||||
|
||||
# ========================================
|
||||
# Patch 2c: In-spinlock semaphore dead-GPU check
|
||||
# Find: "if (tmp & 0x1)" inside the sem acquire loop
|
||||
# ========================================
|
||||
sem_marker = content.find('semaphore acq', flush_func)
|
||||
if sem_marker == -1:
|
||||
sem_marker = content.find('a read return value of 1 means semaphore', flush_func)
|
||||
|
||||
if sem_marker != -1:
|
||||
tmp_check = content.find('if (tmp & 0x1)', sem_marker)
|
||||
if tmp_check != -1:
|
||||
tmp_line_start = content.rfind('\n', 0, tmp_check) + 1
|
||||
sem_dead = (
|
||||
'\t\t\t\t/* ===== BC-250 v3 PATCH: sem dead-GPU check ===== */\n'
|
||||
'\t\t\t\tif (tmp == 0xFFFFFFFF) {\n'
|
||||
'\t\t\t\t\tdev_err_ratelimited(adev->dev,\n'
|
||||
'\t\t\t\t\t\t"BC-250: GPU died during sem acquire (0xFFFFFFFF)\\n");\n'
|
||||
'\t\t\t\t\tspin_unlock(&adev->gmc.invalidate_lock);\n'
|
||||
'\t\t\t\t\treturn;\n'
|
||||
'\t\t\t\t}\n'
|
||||
'\t\t\t\t/* ===== BC-250 v3 PATCH END ===== */\n'
|
||||
)
|
||||
content = content[:tmp_line_start] + sem_dead + content[tmp_line_start:]
|
||||
else:
|
||||
print("WARNING: 'if (tmp & 0x1)' not found after semaphore comment")
|
||||
else:
|
||||
print("WARNING: semaphore comment not found - skipping sem dead-GPU check")
|
||||
|
||||
# ========================================
|
||||
# Patch 2d: ACK-wait loop dead-GPU check
|
||||
# Find: "Wait for ACK with a delay" or "tmp &= 1 << vmid"
|
||||
# ========================================
|
||||
ack_comment = content.find('Wait for ACK with a delay', flush_func)
|
||||
if ack_comment != -1:
|
||||
tmp_mask = content.find('tmp &= 1 << vmid', ack_comment)
|
||||
if tmp_mask != -1:
|
||||
mask_line_start = content.rfind('\n', 0, tmp_mask) + 1
|
||||
ack_dead = (
|
||||
'\t\t\t\t/* ===== BC-250 v3 PATCH: ACK-wait dead-GPU check ===== */\n'
|
||||
'\t\t\t\tif (tmp == 0xFFFFFFFF) {\n'
|
||||
'\t\t\t\t\tdev_err_ratelimited(adev->dev,\n'
|
||||
'\t\t\t\t\t\t"BC-250: GPU died during TLB flush ACK wait (0xFFFFFFFF)\\n");\n'
|
||||
'\t\t\t\t\tif (use_semaphore)\n'
|
||||
'\t\t\t\t\t\tWREG32_RLC_NO_KIQ(sem, 0, hub_ip);\n'
|
||||
'\t\t\t\t\tspin_unlock(&adev->gmc.invalidate_lock);\n'
|
||||
'\t\t\t\t\treturn;\n'
|
||||
'\t\t\t\t}\n'
|
||||
'\t\t\t\t/* ===== BC-250 v3 PATCH END ===== */\n'
|
||||
)
|
||||
content = content[:mask_line_start] + ack_dead + content[mask_line_start:]
|
||||
else:
|
||||
print("WARNING: 'tmp &= 1 << vmid' not found")
|
||||
else:
|
||||
print("WARNING: 'Wait for ACK with a delay' comment not found")
|
||||
|
||||
# ========================================
|
||||
# Patch 2e: gmc_v10_0_hw_init - PASID KIQ disable
|
||||
# Replace: adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
|
||||
# ========================================
|
||||
hw_init = content.find('static int gmc_v10_0_hw_init')
|
||||
if hw_init == -1:
|
||||
print("ERROR: gmc_v10_0_hw_init not found")
|
||||
sys.exit(1)
|
||||
|
||||
pasid_kiq = content.find('flush_pasid_uses_kiq', hw_init)
|
||||
if pasid_kiq == -1:
|
||||
print("ERROR: flush_pasid_uses_kiq not found in hw_init")
|
||||
sys.exit(1)
|
||||
|
||||
# Get the full line
|
||||
pasid_line_start = content.rfind('\n', 0, pasid_kiq) + 1
|
||||
pasid_line_end = content.find('\n', pasid_kiq)
|
||||
|
||||
new_block = (
|
||||
'\t/* ===== BC-250 v2 PATCH: Disable KIQ-based PASID flush ===== */\n'
|
||||
'\t{\n'
|
||||
'\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);\n'
|
||||
'\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0)))\n'
|
||||
'\t\t\tadev->gmc.flush_pasid_uses_kiq = false;\n'
|
||||
'\t\telse\n'
|
||||
'\t\t\tadev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;\n'
|
||||
'\t}\n'
|
||||
'\t/* ===== BC-250 v2 PATCH END ===== */'
|
||||
)
|
||||
|
||||
content = content[:pasid_line_start] + new_block + content[pasid_line_end:]
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("Patch 2 applied: gmc_v10_0.c - 5 sub-patches (KIQ bypass + dead-GPU)")
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Patch 3: amdgpu_gmc.c - KIQ bypass + Dead-GPU detection (2 sub-patches)"""
|
||||
import sys
|
||||
|
||||
AMDGPU = "/home/fabian/kernel-build/linux-6.19.6/drivers/gpu/drm/amd/amdgpu"
|
||||
filepath = f"{AMDGPU}/amdgpu_gmc.c"
|
||||
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
if 'BC-250' in content:
|
||||
print("Already patched, skipping.")
|
||||
sys.exit(0)
|
||||
|
||||
# ========================================
|
||||
# Patch 3a: KIQ bypass in amdgpu_gmc_flush_gpu_tlb_pasid
|
||||
# Insert AFTER down_read_trylock block, BEFORE KIQ ring code
|
||||
# ========================================
|
||||
func_start = content.find('int amdgpu_gmc_flush_gpu_tlb_pasid')
|
||||
if func_start == -1:
|
||||
print("ERROR: amdgpu_gmc_flush_gpu_tlb_pasid not found")
|
||||
sys.exit(1)
|
||||
|
||||
# Find the trylock check
|
||||
trylock = content.find('down_read_trylock', func_start)
|
||||
if trylock == -1:
|
||||
print("ERROR: down_read_trylock not found")
|
||||
sys.exit(1)
|
||||
|
||||
# Find "return 0;" after trylock
|
||||
return_0 = content.find('return 0;', trylock)
|
||||
end_line = content.find('\n', return_0) + 1
|
||||
|
||||
bypass = (
|
||||
'\n'
|
||||
'\t/* ===== BC-250 v2 PATCH: KIQ bypass for PASID flush ===== */\n'
|
||||
'\t{\n'
|
||||
'\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);\n'
|
||||
'\t\tpr_warn_once("amdgpu: flush_gpu_tlb_pasid called, GC_HWIP=0x%08x "\n'
|
||||
'\t\t\t "(10.1.3=0x%08x) kiq_flag=%d\\n",\n'
|
||||
'\t\t\t gc_ver, IP_VERSION(10, 1, 3),\n'
|
||||
'\t\t\t adev->gmc.flush_pasid_uses_kiq);\n'
|
||||
'\n'
|
||||
'\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0))) {\n'
|
||||
'\t\t\tpr_warn_once("amdgpu: BC-250 KIQ bypass active "\n'
|
||||
'\t\t\t\t "(gc_ver=0x%08x)\\n", gc_ver);\n'
|
||||
'\t\t\tadev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,\n'
|
||||
'\t\t\t\t\t\t\t\t flush_type, all_hub,\n'
|
||||
'\t\t\t\t\t\t\t\t inst);\n'
|
||||
'\t\t\tr = 0;\n'
|
||||
'\t\t\tgoto error_unlock_reset;\n'
|
||||
'\t\t}\n'
|
||||
'\t}\n'
|
||||
'\t/* ===== BC-250 v2 PATCH END ===== */\n'
|
||||
)
|
||||
|
||||
content = content[:end_line] + bypass + content[end_line:]
|
||||
|
||||
# ========================================
|
||||
# Patch 3b: KIQ bypass + dead-GPU in amdgpu_gmc_fw_reg_write_reg_wait
|
||||
# Insert BEFORE the KIQ ring submission code
|
||||
# ========================================
|
||||
func2_start = content.find('void amdgpu_gmc_fw_reg_write_reg_wait')
|
||||
if func2_start == -1:
|
||||
print("ERROR: amdgpu_gmc_fw_reg_write_reg_wait not found")
|
||||
sys.exit(1)
|
||||
|
||||
# Find the first operational code after variable declarations
|
||||
# Look for spin_lock_irqsave or ring->sched.ready
|
||||
spinlock = content.find('spin_lock_irqsave', func2_start)
|
||||
if spinlock == -1:
|
||||
# Try ring->sched.ready
|
||||
spinlock = content.find('ring->sched.ready', func2_start)
|
||||
if spinlock == -1:
|
||||
# Try any substantive code line
|
||||
spinlock = content.find('if (', func2_start + 200)
|
||||
if spinlock == -1:
|
||||
print("ERROR: Could not find KIQ code in fw_reg_write_reg_wait")
|
||||
sys.exit(1)
|
||||
|
||||
spinlock_line_start = content.rfind('\n', 0, spinlock) + 1
|
||||
|
||||
bypass2 = (
|
||||
'\t/* ===== BC-250 v2+v3 PATCH: KIQ bypass + dead-GPU detection ===== */\n'
|
||||
'\t{\n'
|
||||
'\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);\n'
|
||||
'\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0))) {\n'
|
||||
'\t\t\tuint32_t tmp;\n'
|
||||
'\n'
|
||||
'\t\t\tpr_warn_once("amdgpu: BC-250 KIQ bypass active in "\n'
|
||||
'\t\t\t\t "fw_reg_write_reg_wait (gc=0x%08x)\\n", gc_ver);\n'
|
||||
'\n'
|
||||
'\t\t\t/* v3: Health-check read before writing */\n'
|
||||
'\t\t\ttmp = RREG32_NO_KIQ(reg1);\n'
|
||||
'\t\t\tif (tmp == 0xFFFFFFFF) {\n'
|
||||
'\t\t\t\tdev_err_ratelimited(adev->dev,\n'
|
||||
'\t\t\t\t\t"BC-250: GPU unreachable in fw_reg_write_reg_wait "\n'
|
||||
'\t\t\t\t\t"(reg1=0x%x returned 0xFFFFFFFF), skipping\\n", reg1);\n'
|
||||
'\t\t\t\treturn;\n'
|
||||
'\t\t\t}\n'
|
||||
'\n'
|
||||
'\t\t\tWREG32_NO_KIQ(reg0, ref);\n'
|
||||
'\t\t\tfor (cnt = 0; cnt < adev->usec_timeout; cnt++) {\n'
|
||||
'\t\t\t\ttmp = RREG32_NO_KIQ(reg1);\n'
|
||||
'\t\t\t\t/* v3: Dead-GPU detection in polling loop */\n'
|
||||
'\t\t\t\tif (tmp == 0xFFFFFFFF) {\n'
|
||||
'\t\t\t\t\tdev_err_ratelimited(adev->dev,\n'
|
||||
'\t\t\t\t\t\t"BC-250: GPU died during reg_write_reg_wait "\n'
|
||||
'\t\t\t\t\t\t"(0xFFFFFFFF at reg1=0x%x)\\n", reg1);\n'
|
||||
'\t\t\t\t\treturn;\n'
|
||||
'\t\t\t\t}\n'
|
||||
'\t\t\t\tif ((tmp & mask) == (ref & mask))\n'
|
||||
'\t\t\t\t\treturn;\n'
|
||||
'\t\t\t\tudelay(1);\n'
|
||||
'\t\t\t}\n'
|
||||
'\t\t\tdev_warn(adev->dev, "BC-250: MMIO reg write/wait timeout "\n'
|
||||
'\t\t\t\t "reg0=0x%x reg1=0x%x\\n", reg0, reg1);\n'
|
||||
'\t\t\treturn;\n'
|
||||
'\t\t}\n'
|
||||
'\t}\n'
|
||||
'\t/* ===== BC-250 v2+v3 PATCH END ===== */\n'
|
||||
)
|
||||
|
||||
content = content[:spinlock_line_start] + bypass2 + content[spinlock_line_start:]
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("Patch 3 applied: amdgpu_gmc.c - 2 sub-patches (KIQ bypass + dead-GPU)")
|
||||
@@ -0,0 +1,589 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BC-250 v3 amdgpu kernel module build script.
|
||||
Downloads kernel source, applies v3 patches, builds the module.
|
||||
Runs entirely over SSH on the BC250.
|
||||
"""
|
||||
import paramiko
|
||||
import time
|
||||
import sys
|
||||
|
||||
KERNEL_VER = "6.19.6"
|
||||
KERNEL_FULL = "6.19.6-2-cachyos"
|
||||
BUILD_DIR = "/home/fabian/kernel-build"
|
||||
SRC_DIR = f"{BUILD_DIR}/linux-{KERNEL_VER}"
|
||||
AMDGPU_DIR = f"{SRC_DIR}/drivers/gpu/drm/amd/amdgpu"
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
sftp = ssh.open_sftp()
|
||||
|
||||
def run(cmd, desc="", timeout=300):
|
||||
if desc:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {desc}")
|
||||
print(f"{'='*60}")
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out:
|
||||
# Truncate very long output
|
||||
lines = out.split('\n')
|
||||
if len(lines) > 50:
|
||||
print('\n'.join(lines[:20]))
|
||||
print(f" ... ({len(lines)-40} lines omitted) ...")
|
||||
print('\n'.join(lines[-20:]))
|
||||
else:
|
||||
print(out)
|
||||
if err and rc != 0:
|
||||
print(f"STDERR: {err[:500]}")
|
||||
if rc != 0:
|
||||
print(f"EXIT CODE: {rc}")
|
||||
return out, rc
|
||||
|
||||
# ============================================================
|
||||
# Step 1: Download kernel source
|
||||
# ============================================================
|
||||
out, rc = run(f"test -d {SRC_DIR} && echo EXISTS || echo MISSING")
|
||||
if "EXISTS" in out:
|
||||
print(f"\nKernel source already exists at {SRC_DIR}")
|
||||
else:
|
||||
run(f"mkdir -p {BUILD_DIR}", "Creating build directory")
|
||||
|
||||
# Download kernel source tarball
|
||||
tarball_url = f"https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-{KERNEL_VER}.tar.xz"
|
||||
run(f"cd {BUILD_DIR} && curl -LO {tarball_url}",
|
||||
f"Downloading linux-{KERNEL_VER}.tar.xz from kernel.org", timeout=600)
|
||||
|
||||
# Extract
|
||||
run(f"cd {BUILD_DIR} && tar xf linux-{KERNEL_VER}.tar.xz",
|
||||
"Extracting kernel source", timeout=300)
|
||||
|
||||
# Clean up tarball
|
||||
run(f"rm {BUILD_DIR}/linux-{KERNEL_VER}.tar.xz")
|
||||
|
||||
# ============================================================
|
||||
# Step 2: Prepare build environment
|
||||
# ============================================================
|
||||
run(f"""cd {SRC_DIR} && \\
|
||||
cp /usr/lib/modules/{KERNEL_FULL}/build/.config . && \\
|
||||
cp /usr/lib/modules/{KERNEL_FULL}/build/Module.symvers . && \\
|
||||
cp /usr/lib/modules/{KERNEL_FULL}/build/localversion.10-pkgrel . && \\
|
||||
cp /usr/lib/modules/{KERNEL_FULL}/build/localversion.20-pkgname . && \\
|
||||
echo 'Build files copied'""",
|
||||
"Copying kernel config, symvers, and localversion files")
|
||||
|
||||
# Prepare the kernel tree (generate required headers)
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 olddefconfig 2>&1 | tail -5",
|
||||
"Running olddefconfig", timeout=120)
|
||||
run(f"cd {SRC_DIR} && make LLVM=1 modules_prepare 2>&1 | tail -10",
|
||||
"Preparing modules build", timeout=120)
|
||||
|
||||
# ============================================================
|
||||
# Step 3: Verify the source files exist and find patch targets
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print(" Verifying source files")
|
||||
print("="*60)
|
||||
|
||||
for f in ["gfx_v10_0.c", "gmc_v10_0.c", "amdgpu_gmc.c"]:
|
||||
out, rc = run(f"wc -l {AMDGPU_DIR}/{f}")
|
||||
print(f" {f}: {out.split()[0]} lines")
|
||||
|
||||
# ============================================================
|
||||
# Step 4: Apply v3 patches
|
||||
# ============================================================
|
||||
|
||||
# --- Patch 1: gfx_v10_0.c - Disable GFXOFF for Cyan Skillfish ---
|
||||
print("\n" + "="*60)
|
||||
print(" Patch 1: gfx_v10_0.c — GFXOFF Disable")
|
||||
print("="*60)
|
||||
|
||||
# Find the exact function and add our case
|
||||
# Look for the switch statement in gfx_v10_0_check_gfxoff_flag
|
||||
out, rc = run(f"grep -n 'case IP_VERSION(10, 1, 10)' {AMDGPU_DIR}/gfx_v10_0.c")
|
||||
if rc != 0:
|
||||
print("ERROR: Could not find IP_VERSION(10,1,10) in gfx_v10_0.c!")
|
||||
sys.exit(1)
|
||||
|
||||
# Check if already patched
|
||||
out, rc = run(f"grep -c 'BC-250' {AMDGPU_DIR}/gfx_v10_0.c")
|
||||
if out.strip() != "0":
|
||||
print("Already patched, skipping.")
|
||||
else:
|
||||
# The patch: add a case for IP_VERSION(10, 1, 3) after the existing default: break;
|
||||
# We need to find the closing "default:" case in gfx_v10_0_check_gfxoff_flag
|
||||
# and insert our case before it
|
||||
|
||||
patch1_script = r"""
|
||||
import re
|
||||
|
||||
filepath = '""" + AMDGPU_DIR + r"""/gfx_v10_0.c'
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find the pattern: after the IP_VERSION(10,1,10) case block, before 'default:'
|
||||
# in gfx_v10_0_check_gfxoff_flag function
|
||||
old_pattern = '\tdefault:\n\t\tbreak;\n\t}\n}'
|
||||
# Find it specifically near gfx_v10_0_check_gfxoff_flag
|
||||
func_start = content.find('static void gfx_v10_0_check_gfxoff_flag')
|
||||
if func_start == -1:
|
||||
print("ERROR: function not found")
|
||||
exit(1)
|
||||
|
||||
# Find the 'default: break; } }' within this function
|
||||
func_region = content[func_start:func_start+2000]
|
||||
default_pos = func_region.find('\tdefault:\n\t\tbreak;\n\t}\n}')
|
||||
if default_pos == -1:
|
||||
# Try different whitespace
|
||||
default_pos = func_region.find('default:\n')
|
||||
if default_pos == -1:
|
||||
print("ERROR: default case not found")
|
||||
print("Function region:\n" + func_region[:500])
|
||||
exit(1)
|
||||
# Get a bit more context
|
||||
context = func_region[default_pos-100:default_pos+100]
|
||||
print(f"Found default at offset {default_pos}, context:\n{context}")
|
||||
exit(1)
|
||||
|
||||
insert_pos = func_start + default_pos
|
||||
|
||||
new_case = '''\t/* ===== BC-250 v3 PATCH START ===== */
|
||||
\tcase IP_VERSION(10, 1, 3):
|
||||
\t\t/*
|
||||
\t\t * BC-250 / Cyan Skillfish (gfx1013): GFXOFF causes the GPU to
|
||||
\t\t * enter a power-saving state from which it cannot reliably wake.
|
||||
\t\t * Unconditionally disable GFXOFF to prevent GPU hangs.
|
||||
\t\t */
|
||||
\t\tadev->pm.pp_feature &= ~PP_GFXOFF_MASK;
|
||||
\t\tdev_info(adev->dev,
|
||||
\t\t\t "BC-250: GFXOFF disabled to prevent GPU power-state hangs\\n");
|
||||
\t\tbreak;
|
||||
\t/* ===== BC-250 v3 PATCH END ===== */
|
||||
'''
|
||||
|
||||
content = content[:insert_pos] + new_case + content[insert_pos:]
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("Patch 1 applied successfully")
|
||||
"""
|
||||
|
||||
# Write patch script to remote
|
||||
with sftp.open('/tmp/patch1.py', 'w') as f:
|
||||
f.write(patch1_script)
|
||||
run("python3 /tmp/patch1.py", "Applying Patch 1")
|
||||
run(f"grep -A15 'gfx_v10_0_check_gfxoff_flag' {AMDGPU_DIR}/gfx_v10_0.c | head -30",
|
||||
"Verify Patch 1")
|
||||
|
||||
# --- Patch 2: gmc_v10_0.c - KIQ bypass + Dead-GPU detection ---
|
||||
print("\n" + "="*60)
|
||||
print(" Patch 2: gmc_v10_0.c — KIQ bypass + Dead-GPU")
|
||||
print("="*60)
|
||||
|
||||
out, rc = run(f"grep -c 'BC-250' {AMDGPU_DIR}/gmc_v10_0.c")
|
||||
if out.strip() != "0":
|
||||
print("Already patched, skipping.")
|
||||
else:
|
||||
patch2_script = r"""
|
||||
filepath = '""" + AMDGPU_DIR + r"""/gmc_v10_0.c'
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# ===== Patch 2a: KIQ bypass in gmc_v10_0_flush_gpu_tlb =====
|
||||
# Find the line: if (adev->gfx.kiq[0].ring.sched.ready && !adev->enable_mes &&
|
||||
# Insert KIQ bypass BEFORE it
|
||||
|
||||
kiq_check = 'if (adev->gfx.kiq[0].ring.sched.ready && !adev->enable_mes &&'
|
||||
pos = content.find(kiq_check)
|
||||
if pos == -1:
|
||||
# Try alternate: might use kiq_inst or different formatting
|
||||
kiq_check = 'if (adev->gfx.kiq['
|
||||
pos = content.find(kiq_check)
|
||||
if pos == -1:
|
||||
print("ERROR: Could not find KIQ check in gmc_v10_0_flush_gpu_tlb")
|
||||
exit(1)
|
||||
|
||||
# Make sure we're in gmc_v10_0_flush_gpu_tlb
|
||||
func_start = content.rfind('gmc_v10_0_flush_gpu_tlb', 0, pos)
|
||||
if func_start == -1:
|
||||
print("ERROR: Not in gmc_v10_0_flush_gpu_tlb?")
|
||||
exit(1)
|
||||
|
||||
# Find the start of the line (go back to newline)
|
||||
line_start = content.rfind('\n', 0, pos) + 1
|
||||
# Get indentation
|
||||
indent = ''
|
||||
for ch in content[line_start:pos]:
|
||||
if ch in ' \t':
|
||||
indent += ch
|
||||
else:
|
||||
break
|
||||
|
||||
kiq_bypass = indent + """/* ===== BC-250 v2 PATCH: KIQ bypass ===== */
|
||||
""" + indent + """/* BC-250 / Cyan Skillfish (gfx1013): KIQ ring TLB flush hangs this GPU.
|
||||
""" + indent + """ * Skip to direct MMIO register path. Widen to all gfx10.1.x for safety.
|
||||
""" + indent + """ */
|
||||
""" + indent + """{
|
||||
""" + indent + """\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
|
||||
""" + indent + """\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0)))
|
||||
""" + indent + """\t\tgoto use_mmio;
|
||||
""" + indent + """}
|
||||
""" + indent + """/* ===== BC-250 v2 PATCH END ===== */
|
||||
"""
|
||||
|
||||
content = content[:line_start] + kiq_bypass + content[line_start:]
|
||||
|
||||
# ===== Patch 2b: Pre-spinlock health check before MMIO =====
|
||||
# Find: "hub_ip = (vmhub == AMDGPU_GFXHUB(0))" in same function
|
||||
# This is the start of the MMIO path ("use_mmio:" label or the code after it)
|
||||
|
||||
use_mmio_label = content.find('use_mmio:', content.find('gmc_v10_0_flush_gpu_tlb'))
|
||||
if use_mmio_label == -1:
|
||||
# The label might not exist yet - we need to find where the MMIO path starts
|
||||
# In vanilla kernel it should be after "/* This path is needed before KIQ/MES/GFXOFF"
|
||||
mmio_comment = content.find('This path is needed before KIQ')
|
||||
if mmio_comment == -1:
|
||||
print("WARNING: Could not find MMIO path - label may already exist from bypass")
|
||||
|
||||
# Find "hub_ip = (vmhub == AMDGPU_GFXHUB(0))" after our inserted code
|
||||
hub_ip_line = 'hub_ip = (vmhub == AMDGPU_GFXHUB(0))'
|
||||
hub_pos = content.find(hub_ip_line, content.find('gmc_v10_0_flush_gpu_tlb'))
|
||||
if hub_pos == -1:
|
||||
print("ERROR: Could not find hub_ip assignment")
|
||||
exit(1)
|
||||
|
||||
# If there's no use_mmio label, add one before the hub_ip line
|
||||
if content.find('use_mmio:', content.find('gmc_v10_0_flush_gpu_tlb')) == -1:
|
||||
# Find the line start before hub_ip
|
||||
hub_line_start = content.rfind('\n', 0, hub_pos) + 1
|
||||
content = content[:hub_line_start] + "use_mmio:\n" + content[hub_line_start:]
|
||||
|
||||
# Re-find hub_ip position after possible label insertion
|
||||
hub_pos = content.find(hub_ip_line, content.find('gmc_v10_0_flush_gpu_tlb'))
|
||||
hub_line_end = content.find('\n', hub_pos)
|
||||
|
||||
# Insert health check AFTER hub_ip assignment line
|
||||
health_check = """
|
||||
|
||||
\t/* ===== BC-250 v3 PATCH: Pre-spinlock health check ===== */
|
||||
\t/*
|
||||
\t * BC-250: GPU health check before entering the spinlock-protected
|
||||
\t * MMIO section. On this SoC the internal PCIe fabric has NO completion
|
||||
\t * timeout - readl() on an unresponsive GPU hangs CPU indefinitely.
|
||||
\t */
|
||||
\t{
|
||||
\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
|
||||
|
||||
\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) &&
|
||||
\t\t (gc_ver < IP_VERSION(10, 2, 0))) {
|
||||
\t\t\ttmp = RREG32_RLC_NO_KIQ(ack, hub_ip);
|
||||
\t\t\tif (tmp == 0xFFFFFFFF) {
|
||||
\t\t\t\tdev_err_ratelimited(adev->dev,
|
||||
\t\t\t\t\t"BC-250: GPU unreachable (MMIO returned 0xFFFFFFFF), "
|
||||
\t\t\t\t\t"skipping TLB flush vmid=%u hub=%u\\n",
|
||||
\t\t\t\t\tvmid, vmhub);
|
||||
\t\t\t\treturn;
|
||||
\t\t\t}
|
||||
\t\t}
|
||||
\t}
|
||||
\t/* ===== BC-250 v3 PATCH END ===== */
|
||||
"""
|
||||
content = content[:hub_line_end] + health_check + content[hub_line_end:]
|
||||
|
||||
# ===== Patch 2c: In-spinlock semaphore dead-GPU check =====
|
||||
# Find the semaphore acquire loop: "if (tmp & 0x1)" inside the TLB flush function
|
||||
# We need to add 0xFFFFFFFF check right after the RREG32 in the sem loop
|
||||
|
||||
# Find "a read return value of 1 means semaphore" comment (unique marker)
|
||||
sem_comment = content.find('a read return value of 1 means semaphore', content.find('gmc_v10_0_flush_gpu_tlb'))
|
||||
if sem_comment == -1:
|
||||
sem_comment = content.find('semaphore acq', content.find('gmc_v10_0_flush_gpu_tlb'))
|
||||
|
||||
if sem_comment != -1:
|
||||
# Find "if (tmp & 0x1)" after this comment
|
||||
tmp_check = content.find('if (tmp & 0x1)', sem_comment)
|
||||
if tmp_check != -1:
|
||||
# Insert dead-GPU check before "if (tmp & 0x1)"
|
||||
tmp_line_start = content.rfind('\n', 0, tmp_check) + 1
|
||||
sem_dead_check = """\t\t\t\t/* ===== BC-250 v3 PATCH: In-spinlock sem dead-GPU check ===== */
|
||||
\t\t\t\tif (tmp == 0xFFFFFFFF) {
|
||||
\t\t\t\t\tdev_err_ratelimited(adev->dev,
|
||||
\t\t\t\t\t\t"BC-250: GPU died during sem acquire (0xFFFFFFFF)\\n");
|
||||
\t\t\t\t\tspin_unlock(&adev->gmc.invalidate_lock);
|
||||
\t\t\t\t\treturn;
|
||||
\t\t\t\t}
|
||||
\t\t\t\t/* ===== BC-250 v3 PATCH END ===== */
|
||||
"""
|
||||
content = content[:tmp_line_start] + sem_dead_check + content[tmp_line_start:]
|
||||
else:
|
||||
print("WARNING: Could not find semaphore comment - skipping sem dead-GPU check")
|
||||
|
||||
# ===== Patch 2d: ACK-wait loop dead-GPU check =====
|
||||
# Find the ACK wait loop: "Wait for ACK with a delay" comment
|
||||
ack_comment = content.find('Wait for ACK with a delay', content.find('gmc_v10_0_flush_gpu_tlb'))
|
||||
if ack_comment != -1:
|
||||
# Find "tmp &= 1 << vmid;" after this - that's inside the ACK loop
|
||||
tmp_mask = content.find('tmp &= 1 << vmid', ack_comment)
|
||||
if tmp_mask != -1:
|
||||
tmp_mask_line_start = content.rfind('\n', 0, tmp_mask) + 1
|
||||
ack_dead_check = """\t\t\t\t/* ===== BC-250 v3 PATCH: ACK-wait dead-GPU check ===== */
|
||||
\t\t\t\tif (tmp == 0xFFFFFFFF) {
|
||||
\t\t\t\t\tdev_err_ratelimited(adev->dev,
|
||||
\t\t\t\t\t\t"BC-250: GPU died during TLB flush ACK wait (0xFFFFFFFF)\\n");
|
||||
\t\t\t\t\tif (use_semaphore)
|
||||
\t\t\t\t\t\tWREG32_RLC_NO_KIQ(sem, 0, hub_ip);
|
||||
\t\t\t\t\tspin_unlock(&adev->gmc.invalidate_lock);
|
||||
\t\t\t\t\treturn;
|
||||
\t\t\t\t}
|
||||
\t\t\t\t/* ===== BC-250 v3 PATCH END ===== */
|
||||
"""
|
||||
content = content[:tmp_mask_line_start] + ack_dead_check + content[tmp_mask_line_start:]
|
||||
else:
|
||||
print("WARNING: Could not find ACK wait comment")
|
||||
|
||||
# ===== Patch 2e: gmc_v10_0_hw_init - PASID KIQ disable =====
|
||||
# Find gmc_v10_0_hw_init function, specifically the line:
|
||||
# adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
|
||||
hw_init_func = content.find('static int gmc_v10_0_hw_init')
|
||||
if hw_init_func == -1:
|
||||
print("ERROR: Could not find gmc_v10_0_hw_init")
|
||||
exit(1)
|
||||
|
||||
pasid_kiq = content.find('flush_pasid_uses_kiq', hw_init_func)
|
||||
if pasid_kiq == -1:
|
||||
print("ERROR: Could not find flush_pasid_uses_kiq in hw_init")
|
||||
exit(1)
|
||||
|
||||
# Find the full line
|
||||
pasid_line_start = content.rfind('\n', 0, pasid_kiq) + 1
|
||||
pasid_line_end = content.find('\n', pasid_kiq)
|
||||
old_line = content[pasid_line_start:pasid_line_end]
|
||||
|
||||
new_block = """\t/* ===== BC-250 v2 PATCH: Disable KIQ-based PASID flush ===== */
|
||||
\t{
|
||||
\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
|
||||
\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0)))
|
||||
\t\t\tadev->gmc.flush_pasid_uses_kiq = false;
|
||||
\t\telse
|
||||
\t\t\tadev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
|
||||
\t}
|
||||
\t/* ===== BC-250 v2 PATCH END ===== */"""
|
||||
|
||||
content = content[:pasid_line_start] + new_block + content[pasid_line_end:]
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("Patch 2 applied successfully (5 sub-patches to gmc_v10_0.c)")
|
||||
"""
|
||||
|
||||
with sftp.open('/tmp/patch2.py', 'w') as f:
|
||||
f.write(patch2_script)
|
||||
run("python3 /tmp/patch2.py", "Applying Patch 2")
|
||||
run(f"grep -c 'BC-250' {AMDGPU_DIR}/gmc_v10_0.c", "Count BC-250 markers in gmc_v10_0.c")
|
||||
|
||||
# --- Patch 3: amdgpu_gmc.c - KIQ bypass + Dead-GPU detection ---
|
||||
print("\n" + "="*60)
|
||||
print(" Patch 3: amdgpu_gmc.c — KIQ bypass + Dead-GPU")
|
||||
print("="*60)
|
||||
|
||||
out, rc = run(f"grep -c 'BC-250' {AMDGPU_DIR}/amdgpu_gmc.c")
|
||||
if out.strip() != "0":
|
||||
print("Already patched, skipping.")
|
||||
else:
|
||||
patch3_script = r"""
|
||||
filepath = '""" + AMDGPU_DIR + r"""/amdgpu_gmc.c'
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# ===== Patch 3a: KIQ bypass in amdgpu_gmc_flush_gpu_tlb_pasid =====
|
||||
# Find down_read_trylock check, then insert bypass after it
|
||||
|
||||
func_start = content.find('int amdgpu_gmc_flush_gpu_tlb_pasid')
|
||||
if func_start == -1:
|
||||
print("ERROR: amdgpu_gmc_flush_gpu_tlb_pasid not found")
|
||||
exit(1)
|
||||
|
||||
# Find the trylock return 0 line
|
||||
trylock = content.find('down_read_trylock', func_start)
|
||||
if trylock == -1:
|
||||
print("ERROR: down_read_trylock not found")
|
||||
exit(1)
|
||||
|
||||
# Find the end of the if block (return 0;)
|
||||
return_0 = content.find('return 0;', trylock)
|
||||
end_of_block = content.find('\n', return_0) + 1
|
||||
|
||||
bypass_code = """
|
||||
\t/* ===== BC-250 v2 PATCH: KIQ bypass for PASID flush ===== */
|
||||
\t{
|
||||
\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
|
||||
\t\tpr_warn_once("amdgpu: flush_gpu_tlb_pasid called, GC_HWIP=0x%08x "
|
||||
\t\t\t "(10.1.3=0x%08x) kiq_flag=%d\\n",
|
||||
\t\t\t gc_ver, IP_VERSION(10, 1, 3),
|
||||
\t\t\t adev->gmc.flush_pasid_uses_kiq);
|
||||
|
||||
\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0))) {
|
||||
\t\t\tpr_warn_once("amdgpu: BC-250 KIQ bypass active "
|
||||
\t\t\t\t "(gc_ver=0x%08x)\\n", gc_ver);
|
||||
\t\t\tadev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
|
||||
\t\t\t\t\t\t\t\t flush_type, all_hub,
|
||||
\t\t\t\t\t\t\t\t inst);
|
||||
\t\t\tr = 0;
|
||||
\t\t\tgoto error_unlock_reset;
|
||||
\t\t}
|
||||
\t}
|
||||
\t/* ===== BC-250 v2 PATCH END ===== */
|
||||
"""
|
||||
|
||||
content = content[:end_of_block] + bypass_code + content[end_of_block:]
|
||||
|
||||
# ===== Patch 3b: KIQ bypass + dead-GPU in amdgpu_gmc_fw_reg_write_reg_wait =====
|
||||
func2_start = content.find('void amdgpu_gmc_fw_reg_write_reg_wait')
|
||||
if func2_start == -1:
|
||||
print("ERROR: amdgpu_gmc_fw_reg_write_reg_wait not found")
|
||||
exit(1)
|
||||
|
||||
# Find the first KIQ-related code after the function signature
|
||||
# Look for "ring->sched.ready" or "spin_lock"
|
||||
# We need to insert BEFORE the KIQ ring submission code
|
||||
# Find the first operational line after variable declarations
|
||||
|
||||
# Look for spin_lock_irqsave which starts the KIQ path
|
||||
spinlock = content.find('spin_lock_irqsave', func2_start)
|
||||
if spinlock == -1:
|
||||
# Try another marker
|
||||
spinlock = content.find('ring->sched.ready', func2_start)
|
||||
if spinlock == -1:
|
||||
print("ERROR: Could not find KIQ code in fw_reg_write_reg_wait")
|
||||
exit(1)
|
||||
|
||||
spinlock_line_start = content.rfind('\n', 0, spinlock) + 1
|
||||
|
||||
bypass2_code = """\t/* ===== BC-250 v2+v3 PATCH: KIQ bypass + dead-GPU detection ===== */
|
||||
\t{
|
||||
\t\tuint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
|
||||
\t\tif ((gc_ver >= IP_VERSION(10, 1, 0)) && (gc_ver < IP_VERSION(10, 2, 0))) {
|
||||
\t\t\tuint32_t tmp;
|
||||
|
||||
\t\t\tpr_warn_once("amdgpu: BC-250 KIQ bypass active in "
|
||||
\t\t\t\t "fw_reg_write_reg_wait (gc=0x%08x)\\n", gc_ver);
|
||||
|
||||
\t\t\t/* v3: Health-check read before writing */
|
||||
\t\t\ttmp = RREG32_NO_KIQ(reg1);
|
||||
\t\t\tif (tmp == 0xFFFFFFFF) {
|
||||
\t\t\t\tdev_err_ratelimited(adev->dev,
|
||||
\t\t\t\t\t"BC-250: GPU unreachable in fw_reg_write_reg_wait "
|
||||
\t\t\t\t\t"(reg1=0x%x returned 0xFFFFFFFF), skipping\\n", reg1);
|
||||
\t\t\t\treturn;
|
||||
\t\t\t}
|
||||
|
||||
\t\t\tWREG32_NO_KIQ(reg0, ref);
|
||||
\t\t\tfor (cnt = 0; cnt < adev->usec_timeout; cnt++) {
|
||||
\t\t\t\ttmp = RREG32_NO_KIQ(reg1);
|
||||
\t\t\t\t/* v3: Dead-GPU detection in polling loop */
|
||||
\t\t\t\tif (tmp == 0xFFFFFFFF) {
|
||||
\t\t\t\t\tdev_err_ratelimited(adev->dev,
|
||||
\t\t\t\t\t\t"BC-250: GPU died during reg_write_reg_wait "
|
||||
\t\t\t\t\t\t"(0xFFFFFFFF at reg1=0x%x)\\n", reg1);
|
||||
\t\t\t\t\treturn;
|
||||
\t\t\t\t}
|
||||
\t\t\t\tif ((tmp & mask) == (ref & mask))
|
||||
\t\t\t\t\treturn;
|
||||
\t\t\t\tudelay(1);
|
||||
\t\t\t}
|
||||
\t\t\tdev_warn(adev->dev, "BC-250: MMIO reg write/wait timeout "
|
||||
\t\t\t\t "reg0=0x%x reg1=0x%x\\n", reg0, reg1);
|
||||
\t\t\treturn;
|
||||
\t\t}
|
||||
\t}
|
||||
\t/* ===== BC-250 v2+v3 PATCH END ===== */
|
||||
"""
|
||||
|
||||
content = content[:spinlock_line_start] + bypass2_code + content[spinlock_line_start:]
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("Patch 3 applied successfully (2 sub-patches to amdgpu_gmc.c)")
|
||||
"""
|
||||
|
||||
with sftp.open('/tmp/patch3.py', 'w') as f:
|
||||
f.write(patch3_script)
|
||||
run("python3 /tmp/patch3.py", "Applying Patch 3")
|
||||
run(f"grep -c 'BC-250' {AMDGPU_DIR}/amdgpu_gmc.c", "Count BC-250 markers in amdgpu_gmc.c")
|
||||
|
||||
# ============================================================
|
||||
# Step 5: Verify all patches
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print(" Patch summary — BC-250 markers in all files")
|
||||
print("="*60)
|
||||
for f in ["gfx_v10_0.c", "gmc_v10_0.c", "amdgpu_gmc.c"]:
|
||||
run(f"grep -n 'BC-250' {AMDGPU_DIR}/{f}")
|
||||
|
||||
# ============================================================
|
||||
# Step 6: Build the module
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print(" Building amdgpu module (LLVM=1)")
|
||||
print(" This may take several minutes...")
|
||||
print("="*60)
|
||||
|
||||
# Build using nohup + log file for long build
|
||||
run(f"cd {SRC_DIR} && nohup make LLVM=1 -j$(nproc) M=drivers/gpu/drm/amd/amdgpu modules > /tmp/amdgpu-build.log 2>&1; echo BUILD_EXIT=$?",
|
||||
"Building amdgpu module", timeout=900)
|
||||
|
||||
# Check build result
|
||||
run("tail -30 /tmp/amdgpu-build.log", "Build log (last 30 lines)")
|
||||
|
||||
# Check if module was built
|
||||
out, rc = run(f"ls -la {AMDGPU_DIR}/amdgpu.ko 2>&1")
|
||||
if rc != 0:
|
||||
print("\nERROR: Module build failed!")
|
||||
run("grep -i error /tmp/amdgpu-build.log | head -20", "Build errors")
|
||||
sys.exit(1)
|
||||
|
||||
# ============================================================
|
||||
# Step 7: Strip, compress, and install
|
||||
# ============================================================
|
||||
print("\n" + "="*60)
|
||||
print(" Stripping and compressing module")
|
||||
print("="*60)
|
||||
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko", "Module size before strip")
|
||||
run(f"strip --strip-debug {AMDGPU_DIR}/amdgpu.ko")
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko", "Module size after strip")
|
||||
run(f"zstd -19 -f {AMDGPU_DIR}/amdgpu.ko", "Compressing with zstd-19", timeout=120)
|
||||
run(f"ls -lh {AMDGPU_DIR}/amdgpu.ko.zst", "Compressed module size")
|
||||
|
||||
# Backup original module
|
||||
MODULE_DIR = f"/usr/lib/modules/{KERNEL_FULL}/kernel/drivers/gpu/drm/amd/amdgpu"
|
||||
run(f"sudo cp {MODULE_DIR}/amdgpu.ko.zst {MODULE_DIR}/amdgpu.ko.zst.original 2>/dev/null; echo done",
|
||||
"Backing up original module")
|
||||
|
||||
# Install new module
|
||||
run(f"sudo cp {AMDGPU_DIR}/amdgpu.ko.zst {MODULE_DIR}/amdgpu.ko.zst",
|
||||
"Installing v3 patched module")
|
||||
|
||||
# Update module dependencies
|
||||
run("sudo depmod -a", "Updating module dependencies")
|
||||
|
||||
# Verify installed module has BC-250 strings
|
||||
run(f"zstd -d -c {MODULE_DIR}/amdgpu.ko.zst | strings | grep 'BC-250'",
|
||||
"Verifying BC-250 strings in installed module")
|
||||
|
||||
print("\n" + "="*60)
|
||||
print(" BUILD AND INSTALL COMPLETE")
|
||||
print(" Reboot required to load the v3 patched module.")
|
||||
print("="*60)
|
||||
|
||||
# Clean up temp files
|
||||
run("rm -f /tmp/patch1.py /tmp/patch2.py /tmp/patch3.py")
|
||||
|
||||
sftp.close()
|
||||
ssh.close()
|
||||
@@ -0,0 +1,98 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, desc=""):
|
||||
if desc:
|
||||
print(f"\n=== {desc} ===")
|
||||
print(f"$ {cmd}")
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out.strip():
|
||||
print(out.strip())
|
||||
if err.strip():
|
||||
print(f"STDERR: {err.strip()}")
|
||||
if rc != 0:
|
||||
print(f"EXIT CODE: {rc}")
|
||||
return out.strip(), err.strip(), rc
|
||||
|
||||
# 1. Append ROCm env vars to fish config
|
||||
fish_env_block = """
|
||||
# === ROCm / HIP Configuration for AMD BC-250 (v3) ===
|
||||
# GPU target override (gfx1013 -> gfx1010 compatible)
|
||||
set -gx HSA_OVERRIDE_GFX_VERSION 10.1.0
|
||||
|
||||
# Device selection
|
||||
set -gx HIP_VISIBLE_DEVICES 0
|
||||
|
||||
# ROCm path
|
||||
set -gx ROCM_PATH /opt/rocm
|
||||
set -gx PATH /opt/rocm/bin $PATH
|
||||
|
||||
# CRITICAL: Disable SDMA engine (HW bugs on gfx1013)
|
||||
set -gx HSA_ENABLE_SDMA 0
|
||||
|
||||
# Disable profiling tools (stability)
|
||||
set -gx HSA_TOOLS_LIB ""
|
||||
set -gx HSA_TOOLS_REPORT_LOAD_FAILURE 0
|
||||
|
||||
# NOTE: Do NOT set HIP_LAUNCH_BLOCKING=1 or GPU_MAX_HW_QUEUES=1
|
||||
# These severely hurt performance and are unnecessary with v3 kernel patches.
|
||||
"""
|
||||
|
||||
# Check if already configured
|
||||
out, _, _ = run("cat ~/.config/fish/config.fish")
|
||||
if "HSA_OVERRIDE_GFX_VERSION" in out:
|
||||
print("\nFish config already has ROCm vars, skipping.")
|
||||
else:
|
||||
# Write the block to a temp file and append
|
||||
run(f"cat >> ~/.config/fish/config.fish << 'FISHEOF'{fish_env_block}FISHEOF",
|
||||
"Appending ROCm env vars to fish config")
|
||||
run("cat ~/.config/fish/config.fish", "Verify fish config")
|
||||
|
||||
# 2. Create /etc/modprobe.d/amdgpu.conf
|
||||
amdgpu_conf = """# AMD BC-250 (Cyan Skillfish / gfx1013) - ROCm Stability Parameters
|
||||
# noretry=0 - Allow page fault retry (critical for shared memory / APU)
|
||||
# gpu_recovery=1 - Enable GPU recovery on timeout
|
||||
# sched_hw_submission=2 - Limit concurrent HW submissions (prevent queue overload)
|
||||
# ppfeaturemask=0xfff73ef7 - Disable GFXOFF (bit 15), SCLK_DEEP_SLEEP (bit 3),
|
||||
# and ULV (bit 8) to prevent unrecoverable power states.
|
||||
options amdgpu noretry=0 gpu_recovery=1 sched_hw_submission=2 ppfeaturemask=0xfff73ef7
|
||||
"""
|
||||
|
||||
out, _, _ = run("cat /etc/modprobe.d/amdgpu.conf 2>/dev/null || echo 'NOT_FOUND'")
|
||||
if "NOT_FOUND" in out or "ppfeaturemask" not in out:
|
||||
run(f"sudo tee /etc/modprobe.d/amdgpu.conf << 'MODEOF'{amdgpu_conf}MODEOF",
|
||||
"Creating /etc/modprobe.d/amdgpu.conf")
|
||||
run("cat /etc/modprobe.d/amdgpu.conf", "Verify amdgpu.conf")
|
||||
else:
|
||||
print("\namdgpu.conf already configured, skipping.")
|
||||
|
||||
# 3. Update Limine boot parameters
|
||||
run("cat /etc/default/limine", "Current Limine config")
|
||||
|
||||
# Read current config
|
||||
out, _, _ = run("cat /etc/default/limine")
|
||||
if "amdgpu.gpu_recovery" in out:
|
||||
print("\nLimine already has amdgpu boot params, skipping.")
|
||||
else:
|
||||
# We need to add amdgpu params to the KERNEL_CMDLINE
|
||||
# The current line likely looks like:
|
||||
# KERNEL_CMDLINE[default]="quiet nowatchdog splash rw rootflags=subvol=/@ root=UUID=..."
|
||||
# We need to add params before rootflags or at end of quoted string
|
||||
|
||||
# Use sed to insert amdgpu params before rootflags
|
||||
sed_cmd = r"""sudo sed -i 's|rootflags=subvol=/@|amdgpu.gpu_recovery=1 amdgpu.noretry=0 amdgpu.dc=0 amdgpu.lockup_timeout=120000 amdgpu.ppfeaturemask=0xfff73ef7 rootflags=subvol=/@|' /etc/default/limine"""
|
||||
run(sed_cmd, "Adding amdgpu boot params to Limine config")
|
||||
run("cat /etc/default/limine", "Verify Limine config")
|
||||
|
||||
# Apply limine update
|
||||
run("sudo limine-update", "Applying Limine update")
|
||||
|
||||
print("\n=== Configuration complete ===")
|
||||
ssh.close()
|
||||
@@ -0,0 +1,77 @@
|
||||
import paramiko
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
sftp = ssh.open_sftp()
|
||||
|
||||
def run(cmd, desc=""):
|
||||
if desc:
|
||||
print(f"\n=== {desc} ===")
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
out = stdout.read().decode().strip()
|
||||
err = stderr.read().decode().strip()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
if out: print(out)
|
||||
if err: print(f"STDERR: {err}")
|
||||
return out, rc
|
||||
|
||||
# 1. Fix fish config - read current, append ROCm block, write back
|
||||
print("=== Updating fish config ===")
|
||||
current = sftp.open('/home/fabian/.config/fish/config.fish', 'r').read().decode()
|
||||
|
||||
rocm_block = """
|
||||
# === ROCm / HIP Configuration for AMD BC-250 (v3) ===
|
||||
# GPU target override (gfx1013 -> gfx1010 compatible)
|
||||
set -gx HSA_OVERRIDE_GFX_VERSION 10.1.0
|
||||
|
||||
# Device selection
|
||||
set -gx HIP_VISIBLE_DEVICES 0
|
||||
|
||||
# ROCm path
|
||||
set -gx ROCM_PATH /opt/rocm
|
||||
set -gx PATH /opt/rocm/bin $PATH
|
||||
|
||||
# CRITICAL: Disable SDMA engine (HW bugs on gfx1013)
|
||||
set -gx HSA_ENABLE_SDMA 0
|
||||
|
||||
# Disable profiling tools (stability)
|
||||
set -gx HSA_TOOLS_LIB ""
|
||||
set -gx HSA_TOOLS_REPORT_LOAD_FAILURE 0
|
||||
|
||||
# NOTE: Do NOT set HIP_LAUNCH_BLOCKING=1 or GPU_MAX_HW_QUEUES=1
|
||||
# These severely hurt performance and are unnecessary with v3 kernel patches.
|
||||
"""
|
||||
|
||||
if "HSA_OVERRIDE_GFX_VERSION" not in current:
|
||||
with sftp.open('/home/fabian/.config/fish/config.fish', 'w') as f:
|
||||
f.write(current + rocm_block)
|
||||
print("ROCm env vars appended to fish config")
|
||||
else:
|
||||
print("Already configured, skipping")
|
||||
|
||||
run("cat ~/.config/fish/config.fish", "Verify fish config")
|
||||
|
||||
# 2. Fix amdgpu.conf - write to /tmp first, then sudo mv
|
||||
print("\n=== Creating /etc/modprobe.d/amdgpu.conf ===")
|
||||
amdgpu_conf = """# AMD BC-250 (Cyan Skillfish / gfx1013) - ROCm Stability Parameters
|
||||
# noretry=0 - Allow page fault retry (critical for shared memory / APU)
|
||||
# gpu_recovery=1 - Enable GPU recovery on timeout
|
||||
# sched_hw_submission=2 - Limit concurrent HW submissions (prevent queue overload)
|
||||
# ppfeaturemask=0xfff73ef7 - Disable GFXOFF (bit 15), SCLK_DEEP_SLEEP (bit 3),
|
||||
# and ULV (bit 8) to prevent unrecoverable power states.
|
||||
options amdgpu noretry=0 gpu_recovery=1 sched_hw_submission=2 ppfeaturemask=0xfff73ef7
|
||||
"""
|
||||
|
||||
with sftp.open('/tmp/amdgpu.conf', 'w') as f:
|
||||
f.write(amdgpu_conf)
|
||||
|
||||
run("sudo cp /tmp/amdgpu.conf /etc/modprobe.d/amdgpu.conf && rm /tmp/amdgpu.conf")
|
||||
run("cat /etc/modprobe.d/amdgpu.conf", "Verify amdgpu.conf")
|
||||
|
||||
# 3. Verify Limine config (already done by previous script)
|
||||
run("cat /etc/default/limine", "Verify Limine config")
|
||||
|
||||
print("\n=== All configuration complete ===")
|
||||
sftp.close()
|
||||
ssh.close()
|
||||
@@ -0,0 +1,122 @@
|
||||
import paramiko
|
||||
import sys
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
|
||||
|
||||
def run(cmd, timeout=30):
|
||||
stdin, stdout, stderr = c.exec_command(f"bash -c '{cmd}'", timeout=timeout)
|
||||
out = stdout.read().decode()
|
||||
err = stderr.read().decode()
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
return out.strip(), err.strip(), rc
|
||||
|
||||
def section(title):
|
||||
print(f"\n{'='*50}")
|
||||
print(f" {title}")
|
||||
print(f"{'='*50}")
|
||||
|
||||
# Test 1: Module status
|
||||
section("Test 1: amdgpu Module")
|
||||
out, _, _ = run("lsmod | grep amdgpu | head -3")
|
||||
print(out)
|
||||
|
||||
# Test 2: v3 Module messages
|
||||
section("Test 2: v3 Module (GFXOFF, BC-250)")
|
||||
out, _, _ = run('sudo dmesg | grep -E "BC-250|GFXOFF|out-of-tree" | head -5')
|
||||
print(out)
|
||||
|
||||
# Test 3: KIQ errors
|
||||
section("Test 3: KIQ Fence Timeouts")
|
||||
out, _, _ = run('sudo dmesg | grep -ci "timeout waiting for kiq fence" 2>/dev/null || echo 0')
|
||||
print(f"KIQ timeout count: {out}")
|
||||
|
||||
# Test 4: GPU dead events
|
||||
section("Test 4: GPU Unreachable/Dead Events")
|
||||
out, _, _ = run('sudo dmesg | grep -ci "GPU unreachable\\|GPU died" 2>/dev/null || echo 0')
|
||||
print(f"GPU dead count: {out}")
|
||||
|
||||
# Test 5: ppfeaturemask
|
||||
section("Test 5: ppfeaturemask")
|
||||
out, _, _ = run("cat /sys/module/amdgpu/parameters/ppfeaturemask")
|
||||
print(f"ppfeaturemask: {out}")
|
||||
|
||||
# Test 6: Devices
|
||||
section("Test 6: DRM/KFD Devices")
|
||||
out, _, _ = run("ls -la /dev/dri/ 2>&1; echo; ls -la /dev/kfd 2>&1")
|
||||
print(out)
|
||||
|
||||
# Test 7: GPU clocks
|
||||
section("Test 7: GPU Clock Levels")
|
||||
out, _, _ = run("cat /sys/class/drm/card0/device/pp_dpm_sclk 2>&1")
|
||||
print(out)
|
||||
|
||||
# Test 8: Governor
|
||||
section("Test 8: Cyan Skillfish Governor")
|
||||
out, _, _ = run("systemctl is-active cyan-skillfish-governor.service 2>&1")
|
||||
print(f"Governor: {out}")
|
||||
|
||||
# Test 9: rocminfo
|
||||
section("Test 9: rocminfo")
|
||||
out, _, _ = run('export HSA_OVERRIDE_GFX_VERSION=10.1.0 HIP_VISIBLE_DEVICES=0 HSA_ENABLE_SDMA=0 HSA_TOOLS_LIB= HSA_TOOLS_REPORT_LOAD_FAILURE=0; rocminfo 2>&1 | grep -E "Name:|Marketing Name:|Compute Unit:|gfx|Done"', timeout=60)
|
||||
print(out)
|
||||
|
||||
# Test 10: hip_probe
|
||||
section("Test 10: hip_probe (Dani's diagnostic)")
|
||||
out, _, rc = run('export HSA_OVERRIDE_GFX_VERSION=10.1.0 HIP_VISIBLE_DEVICES=0 HSA_ENABLE_SDMA=0 HSA_TOOLS_LIB= HSA_TOOLS_REPORT_LOAD_FAILURE=0; /tmp/hip_probe 2>&1', timeout=60)
|
||||
print(out)
|
||||
print(f"Exit code: {rc}")
|
||||
|
||||
# Check KIQ after hip_probe
|
||||
import time
|
||||
time.sleep(3)
|
||||
out, _, _ = run('sudo dmesg | tail -20 | grep -ci "timeout waiting for kiq fence" 2>/dev/null || echo 0')
|
||||
print(f"\nKIQ errors after hip_probe: {out}")
|
||||
|
||||
# Test 11: hip_minimal_test
|
||||
section("Test 11: hip_minimal_test (kernel compute)")
|
||||
out, _, rc = run('export HSA_OVERRIDE_GFX_VERSION=10.1.0 HIP_VISIBLE_DEVICES=0 HSA_ENABLE_SDMA=0 HSA_TOOLS_LIB= HSA_TOOLS_REPORT_LOAD_FAILURE=0; /tmp/hip_minimal_test 2>&1', timeout=60)
|
||||
print(out)
|
||||
print(f"Exit code: {rc}")
|
||||
|
||||
time.sleep(3)
|
||||
out, _, _ = run('sudo dmesg | tail -20 | grep -ci "timeout waiting for kiq fence" 2>/dev/null || echo 0')
|
||||
print(f"\nKIQ errors after hip_minimal_test: {out}")
|
||||
|
||||
# Test 12: hip_vector_add (managed memory + timing)
|
||||
section("Test 12: hip_vector_add (managed memory, sin²+cos²)")
|
||||
out, _, rc = run('export HSA_OVERRIDE_GFX_VERSION=10.1.0 HIP_VISIBLE_DEVICES=0 HSA_ENABLE_SDMA=0 HSA_TOOLS_LIB= HSA_TOOLS_REPORT_LOAD_FAILURE=0; /tmp/hip_vector_add 2>&1', timeout=60)
|
||||
print(out)
|
||||
print(f"Exit code: {rc}")
|
||||
|
||||
time.sleep(3)
|
||||
out, _, _ = run('sudo dmesg | tail -20 | grep -ci "timeout waiting for kiq fence" 2>/dev/null || echo 0')
|
||||
print(f"\nKIQ errors after hip_vector_add: {out}")
|
||||
|
||||
# Test 13: Sequential stress (3 rounds)
|
||||
section("Test 13: Sequential GPU Stress (3 rounds)")
|
||||
for i in range(1, 4):
|
||||
out, _, rc = run('export HSA_OVERRIDE_GFX_VERSION=10.1.0 HIP_VISIBLE_DEVICES=0 HSA_ENABLE_SDMA=0 HSA_TOOLS_LIB= HSA_TOOLS_REPORT_LOAD_FAILURE=0; /tmp/hip_minimal_test 2>&1 | tail -3', timeout=60)
|
||||
status = "PASS" if rc == 0 else "FAIL"
|
||||
print(f"Round {i}: {status} (rc={rc}) — {out.split(chr(10))[-1]}")
|
||||
time.sleep(2)
|
||||
|
||||
# Final KIQ check
|
||||
time.sleep(5)
|
||||
section("Final: Post-Stress KIQ Check")
|
||||
out, _, _ = run('sudo dmesg | grep -ci "timeout waiting for kiq fence" 2>/dev/null || echo 0')
|
||||
print(f"Total KIQ timeout count: {out}")
|
||||
|
||||
out, _, _ = run('sudo dmesg | grep -ci "GPU unreachable\\|GPU died" 2>/dev/null || echo 0')
|
||||
print(f"Total GPU dead count: {out}")
|
||||
|
||||
# Test 14: rocminfo AFTER all GPU tests
|
||||
section("Test 14: rocminfo After Stress (previously would hang)")
|
||||
out, _, rc = run('export HSA_OVERRIDE_GFX_VERSION=10.1.0 HIP_VISIBLE_DEVICES=0 HSA_ENABLE_SDMA=0 HSA_TOOLS_LIB= HSA_TOOLS_REPORT_LOAD_FAILURE=0; timeout 30 rocminfo 2>&1 | grep "Done"', timeout=60)
|
||||
print(f"rocminfo: {out} (rc={rc})")
|
||||
|
||||
print("\n" + "="*50)
|
||||
print(" ALL VERIFICATION COMPLETE")
|
||||
print("="*50)
|
||||
c.close()
|
||||
Reference in New Issue
Block a user