Fabian/ROCm-Research-Archive
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uploaded sanitized BC250/ROCm Repository.

Browse Source
This commit is contained in:
Fabian
2026-08-20 00:45:43 +02:00
parent 7d2184f1e8
commit d7d22e93b3
678 changed files with 65963 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Claude Test Scripts/arch_check.py
+4
View File
@@ -0,0 +1,4 @@
import torch
print("Device props:", torch.cuda.get_device_properties(0))
print("Arch list:", torch.cuda.get_arch_list())
print("GCN arch:", torch.cuda.get_device_properties(0).gcnArchName if hasattr(torch.cuda.get_device_properties(0), 'gcnArchName') else 'N/A')
Claude Test Scripts/build_gfx1010.sh
+58
View File
@@ -0,0 +1,58 @@
#!/bin/bash
set -e
echo "============================================"
echo " PyTorch 2.5.1 Build for gfx1010 (BC-250)"
echo "============================================"
cd ~/pytorch-build
# --- Phase 1: Clean old partial build ---
echo "[Phase 1] Cleaning old build artifacts..."
rm -rf build dist
python3.11 -m pip install --user numpy pyyaml typing-extensions cffi 2>/dev/null || true
echo "[Phase 1] Done."
# --- Phase 2: Environment ---
echo "[Phase 2] Setting environment..."
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export PATH="/opt/rocm/bin:$PATH"
export LD_LIBRARY_PATH="/opt/rocm/lib:$LD_LIBRARY_PATH"
# PyTorch build config — minimal build for inference
export USE_ROCM=1
export USE_CUDA=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_MKLDNN=0
export USE_FBGEMM=0
export USE_NNPACK=0
export USE_QNNPACK=0
export USE_XNNPACK=0
export USE_KINETO=0
export BUILD_TEST=0
export BUILD_CAFFE2=0
export USE_NUMPY=1
export PYTORCH_ROCM_ARCH="gfx1010"
export HIP_PATH=/opt/rocm
export ROCM_SOURCE_DIR=/opt/rocm
export CMAKE_PREFIX_PATH=/opt/rocm
export MAX_JOBS=8
export CMAKE_POLICY_VERSION_MINIMUM=3.5
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo " MAX_JOBS=$MAX_JOBS"
echo "[Phase 2] Done."
# --- Phase 3: Build ---
echo "[Phase 3] Starting PyTorch build..."
python3.11 setup.py bdist_wheel 2>&1 | tee ~/pytorch-build-progress.log
echo ""
echo "============================================"
echo " BUILD COMPLETE"
echo "============================================"
ls -lh dist/*.whl 2>/dev/null || echo "ERROR: No wheel produced!"
Claude Test Scripts/build_notriton.sh
+62
View File
@@ -0,0 +1,62 @@
#!/bin/bash
set -e
echo "============================================"
echo " PyTorch 2.5.1 Build for gfx1010 (BC-250)"
echo " NO TRITON / NO AOTRITON"
echo "============================================"
cd ~/pytorch-build
# --- Phase 1: Clean old build ---
echo "[Phase 1] Cleaning old build..."
rm -rf build dist
echo "[Phase 1] Done."
# --- Phase 2: Environment ---
echo "[Phase 2] Setting environment..."
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export PATH="/opt/rocm/bin:/home/fabian/.local/bin:$PATH"
export LD_LIBRARY_PATH="/opt/rocm/lib:$LD_LIBRARY_PATH"
# PyTorch build config — minimal for inference, NO Triton
export USE_ROCM=1
export USE_CUDA=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_MKLDNN=0
export USE_FBGEMM=0
export USE_NNPACK=0
export USE_QNNPACK=0
export USE_XNNPACK=0
export USE_KINETO=0
export BUILD_TEST=0
export BUILD_CAFFE2=0
export USE_NUMPY=1
export USE_AOTRITON=0
export USE_TRITON=0
export PYTORCH_ROCM_ARCH="gfx1010"
export HIP_PATH=/opt/rocm
export ROCM_SOURCE_DIR=/opt/rocm
export CMAKE_PREFIX_PATH=/opt/rocm
export MAX_JOBS=8
export CMAKE_POLICY_VERSION_MINIMUM=3.5
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo " USE_AOTRITON=0 USE_TRITON=0"
echo " MAX_JOBS=$MAX_JOBS"
echo "[Phase 2] Done."
# --- Phase 3: Build ---
echo "[Phase 3] Starting PyTorch build at $(date)..."
python3.11 setup.py bdist_wheel 2>&1 | tee ~/pytorch-build-progress.log
echo ""
echo "============================================"
echo " BUILD FINISHED at $(date)"
echo "============================================"
ls -lh dist/*.whl 2>/dev/null || echo "ERROR: No wheel produced!"
Claude Test Scripts/build_pytorch.sh
+84
View File
@@ -0,0 +1,84 @@
#!/bin/bash
# =============================================================
# PyTorch 2.5.1 Build for AMD BC-250 — native gfx1010 target
# ROCm 7.2 on CachyOS
# =============================================================
set -euo pipefail
echo "=========================================="
echo " PyTorch Build — gfx1010 for BC-250"
echo "=========================================="
# ========== PHASE 1: Clone ==========
if [ ! -d ~/pytorch-build ]; then
echo "[1/5] Cloning PyTorch v2.5.1 (full recursive)..."
cd ~
git clone --recursive --branch v2.5.1 https://github.com/pytorch/pytorch.git pytorch-build
else
echo "[1/5] Source already exists, skipping clone"
fi
cd ~/pytorch-build
# ========== PHASE 2: Venv + deps ==========
echo "[2/5] Installing build dependencies..."
source ~/ComfyUI/venv/bin/activate
pip install -q cmake ninja pyyaml typing-extensions numpy setuptools wheel cffi
# ========== PHASE 3: Hipify ==========
echo "[3/5] Running hipify (CUDA → HIP conversion)..."
python tools/amd_build/build_amd.py
# ========== PHASE 4: Environment ==========
echo "[4/5] Configuring build environment..."
# ROCm paths
export ROCM_PATH=/opt/rocm
export HIP_PATH=/opt/rocm
export ROCM_HOME=/opt/rocm
export PATH="/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH"
export LD_LIBRARY_PATH="/opt/rocm/lib:${LD_LIBRARY_PATH:-}"
# Target gfx1010 (RDNA1 — closest supported arch for BC-250 gfx1013)
export PYTORCH_ROCM_ARCH="gfx1010"
export AMDGPU_TARGETS="gfx1010"
# Build config
export USE_ROCM=1
export USE_CUDA=0
export USE_CUDNN=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_MKLDNN=1
export USE_OPENMP=1
export BUILD_TEST=0
export USE_FLASH_ATTENTION=0
export USE_MEM_EFF_ATTENTION=0
export REL_WITH_DEB_INFO=0
export CMAKE_BUILD_TYPE=Release
export CMAKE_POLICY_VERSION_MINIMUM=3.5
# Parallelism — 12 cores, 14GB RAM, be conservative
export MAX_JOBS=4
export NINJA_STATUS="[%f/%t %e] "
# Compiler
export CC=gcc
export CXX=g++
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo " MAX_JOBS=$MAX_JOBS"
echo " ROCM_PATH=$ROCM_PATH"
echo " Python: $(python --version)"
# ========== PHASE 5: Build ==========
echo "[5/5] Building PyTorch wheel..."
echo " Start: $(date)"
python setup.py bdist_wheel
echo ""
echo "=========================================="
echo " Build complete: $(date)"
echo "=========================================="
ls -lh dist/*.whl 2>/dev/null || echo "ERROR: No wheel found!"
Claude Test Scripts/build_pytorch_gfx1010.sh
+140
View File
@@ -0,0 +1,140 @@
#!/bin/bash
set -e
exec > >(tee -a ~/pytorch-build.log) 2>&1
echo "=========================================="
echo " PyTorch 2.5.1 Build for BC-250 (gfx1010)"
echo " ROCm 7.2.0 | HIP 7.2 | $(date)"
echo "=========================================="
BUILD_DIR="$HOME/pytorch-build"
ROCM_ARCH="gfx1010"
##############################################
# Phase 1: Fix incomplete clone
##############################################
echo ""
echo "[1/5] Fixing submodules..."
cd "$BUILD_DIR"
# Reset any partial checkouts
git submodule sync --recursive
git submodule update --init --recursive --force --jobs 4 2>&1 || {
echo "WARN: Some submodules failed, retrying one by one..."
git submodule foreach --recursive 'git checkout . 2>/dev/null; true'
git submodule update --init --recursive --force 2>&1 || true
}
echo "Clone/submodule size: $(du -sh "$BUILD_DIR" | cut -f1)"
##############################################
# Phase 2: Install build dependencies
##############################################
echo ""
echo "[2/5] Checking build dependencies..."
for pkg in cmake ninja gcc python3; do
if ! command -v $pkg &>/dev/null; then
echo "ERROR: $pkg not found!"
exit 1
fi
echo " $pkg: $(command -v $pkg)"
done
# Ensure python build deps
pip3 install --user cmake ninja pyyaml typing-extensions setuptools wheel 2>&1 | tail -3
##############################################
# Phase 3: Hipify (CUDA -> HIP conversion)
##############################################
echo ""
echo "[3/5] Running hipify (CUDA -> HIP)..."
cd "$BUILD_DIR"
if [ ! -f "aten/src/ATen/hip" ] || [ ! -d "aten/src/ATen/hip" ]; then
python3 tools/amd_build/build_amd.py 2>&1 | tail -20
echo "Hipify complete."
else
echo "Hipify already done, skipping."
fi
##############################################
# Phase 4: Configure environment
##############################################
echo ""
echo "[4/5] Configuring build environment..."
# ROCm paths
export ROCM_PATH=/opt/rocm
export HIP_PATH=/opt/rocm
export ROCM_HOME=/opt/rocm
export HCC_HOME=/opt/rocm/hcc
export HIP_PLATFORM=amd
# Target architecture
export PYTORCH_ROCM_ARCH="$ROCM_ARCH"
export AMDGPU_TARGETS="$ROCM_ARCH"
# Build configuration
export USE_ROCM=1
export USE_CUDA=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_MKLDNN=0
export USE_FBGEMM=0
export USE_KINETO=0
export USE_QNNPACK=0
export USE_PYTORCH_QNNPACK=0
export USE_NNPACK=0
export USE_XNNPACK=0
export BUILD_TEST=0
export BUILD_CAFFE2=0
export USE_FLASH_ATTENTION=0
export USE_MEM_EFF_ATTENTION=0
# BC-250 has limited RAM — reduce parallel jobs
export MAX_JOBS=3
# Fix CMake 4.x compatibility
export CMAKE_POLICY_VERSION_MINIMUM=3.5
# Compiler
export CC=gcc
export CXX=g++
export CMAKE_C_COMPILER=gcc
export CMAKE_CXX_COMPILER=g++
# HIP compiler
export HIP_CLANG_PATH=/opt/rocm/llvm/bin
export HIPCC_COMPILE_FLAGS_APPEND="--offload-arch=$ROCM_ARCH"
# BC-250 specific: need managed memory
export HSA_ENABLE_SDMA=0
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo " MAX_JOBS=$MAX_JOBS"
echo " ROCM_PATH=$ROCM_PATH"
echo " Python: $(python3 --version)"
##############################################
# Phase 5: Build
##############################################
echo ""
echo "[5/5] Building PyTorch wheel..."
echo " Start time: $(date)"
cd "$BUILD_DIR"
# Clean any previous build artifacts
python3 setup.py clean 2>/dev/null || true
# Build wheel
python3 setup.py bdist_wheel 2>&1
echo ""
echo "=========================================="
echo " BUILD COMPLETE: $(date)"
echo "=========================================="
ls -lh dist/*.whl 2>/dev/null || echo "ERROR: No wheel file found!"
echo ""
echo "Install with:"
echo " pip install dist/torch-*.whl"
Claude Test Scripts/build_torchvision.sh
+27
View File
@@ -0,0 +1,27 @@
#!/bin/bash
set -e
source /home/fabian/ComfyUI/venv/bin/activate
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export PYTORCH_ROCM_ARCH=gfx1010
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST=""
# Uninstall old torchvision
pip uninstall torchvision -y 2>/dev/null || true
# Clone torchvision matching PyTorch 2.5
cd /tmp
rm -rf torchvision_build
git clone --depth 1 --branch v0.20.0 https://github.com/pytorch/vision.git torchvision_build
cd torchvision_build
# Build and install
python setup.py install 2>&1 | tail -20
echo "=== DONE ==="
python -c "import torchvision; print('torchvision version:', torchvision.__version__)"
Claude Test Scripts/build_v3.sh
+63
View File
@@ -0,0 +1,63 @@
#!/bin/bash
set -e
echo "============================================"
echo " PyTorch 2.5.1 Build for gfx1010 (BC-250)"
echo " NO FLASH/MEM_EFF ATTENTION = NO AOTRITON"
echo "============================================"
cd ~/pytorch-build
# --- Phase 1: Clean ---
echo "[Phase 1] Clean build dir..."
rm -rf build dist
echo "[Phase 1] Done."
# --- Phase 2: Environment ---
echo "[Phase 2] Setting environment..."
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export PATH="/opt/rocm/bin:/home/fabian/.local/bin:$PATH"
export LD_LIBRARY_PATH="/opt/rocm/lib:$LD_LIBRARY_PATH"
# PyTorch build config
export USE_ROCM=1
export USE_CUDA=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_MKLDNN=0
export USE_FBGEMM=0
export USE_NNPACK=0
export USE_QNNPACK=0
export USE_XNNPACK=0
export USE_KINETO=0
export BUILD_TEST=0
export BUILD_CAFFE2=0
export USE_NUMPY=1
# Disable Flash/MemEff attention -> no aotriton dependency
export USE_FLASH_ATTENTION=0
export USE_MEM_EFF_ATTENTION=0
export PYTORCH_ROCM_ARCH="gfx1010"
export HIP_PATH=/opt/rocm
export ROCM_SOURCE_DIR=/opt/rocm
export CMAKE_PREFIX_PATH=/opt/rocm
export MAX_JOBS=8
export CMAKE_POLICY_VERSION_MINIMUM=3.5
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo " USE_FLASH_ATTENTION=0"
echo " USE_MEM_EFF_ATTENTION=0"
echo " MAX_JOBS=$MAX_JOBS"
echo "[Phase 2] Done."
# --- Phase 3: Build ---
echo "[Phase 3] Starting build at $(date)..."
python3.11 setup.py bdist_wheel 2>&1 | tee ~/pytorch-build-progress.log
echo ""
echo "============================================"
echo " BUILD FINISHED at $(date)"
echo "============================================"
ls -lh dist/*.whl 2>/dev/null || echo "ERROR: No wheel produced!"
Claude Test Scripts/build_v4.sh
+48
View File
@@ -0,0 +1,48 @@
#!/bin/bash
# Build v4 - incremental rebuild after C10_WARP_SIZE fix
set -e
cd /home/fabian/pytorch-build
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export HIP_PATH=/opt/rocm
export PYTORCH_ROCM_ARCH=gfx1010
export USE_ROCM=1
export USE_CUDA=0
export USE_FLASH_ATTENTION=0
export USE_MEM_EFF_ATTENTION=0
export USE_AOTRITON=0
export USE_TRITON=0
export MAX_JOBS=8
export CMAKE_POLICY_VERSION_MINIMUM=3.5
export CMAKE_PREFIX_PATH=/opt/rocm
export PYTHON_EXECUTABLE=/usr/bin/python3.11
LOG=/home/fabian/pytorch-build/build_v4.log
echo "============================================" | tee "$LOG"
echo " BUILD v4 STARTED at $(date)" | tee -a "$LOG"
echo " Incremental rebuild after C10_WARP_SIZE fix" | tee -a "$LOG"
echo "============================================" | tee -a "$LOG"
# Incremental build - ninja will only recompile changed files
cd build
cmake --build . --target install 2>&1 | tee -a "$LOG"
echo "============================================" | tee -a "$LOG"
echo " BUILD v4 FINISHED at $(date)" | tee -a "$LOG"
echo "============================================" | tee -a "$LOG"
# Back to source root for wheel
cd /home/fabian/pytorch-build
/usr/bin/python3.11 setup.py bdist_wheel 2>&1 | tee -a "$LOG"
WHL=$(ls dist/*.whl 2>/dev/null | head -1)
if [ -n "$WHL" ]; then
echo "SUCCESS: Wheel at $WHL" | tee -a "$LOG"
ls -lh "$WHL" | tee -a "$LOG"
else
echo "ERROR: No wheel produced!" | tee -a "$LOG"
fi
Claude Test Scripts/check_aotriton.sh
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
cd ~/pytorch-build
echo "=== CMakeLists.txt around aotriton ==="
sed -n '855,900p' CMakeLists.txt
echo ""
echo "=== aotriton.cmake ==="
cat cmake/External/aotriton.cmake
echo ""
echo "=== USE_FLASH_ATTENTION in Dependencies ==="
grep -n "FLASH_ATTENTION\|AOTRITON\|aotriton" cmake/Dependencies.cmake 2>/dev/null | head -20
Claude Test Scripts/check_build.sh
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
echo "=== Active build processes ==="
ps aux | grep -E "(build_pytorch|python.*setup|cmake|ninja|make|git.*(clone|submodule))" | grep -v grep | head -10
echo ""
echo "=== Log size ==="
wc -l ~/pytorch-build.log 2>/dev/null
echo ""
echo "=== Last 30 lines of log ==="
tail -30 ~/pytorch-build.log 2>/dev/null
echo ""
echo "=== Build directory status ==="
du -sh ~/pytorch-build 2>/dev/null
ls ~/pytorch-build/build 2>/dev/null && echo "Build dir exists" || echo "Build dir not yet created"
ls ~/pytorch-build/dist/*.whl 2>/dev/null && echo "Wheel found!" || echo "No wheel yet"
Claude Test Scripts/check_build_state.sh
+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
echo "=== OLD BUILD STATE ==="
if [ -d ~/pytorch-build ]; then
echo "Dir exists: $(du -sh ~/pytorch-build)"
ls ~/pytorch-build/
if [ -d ~/pytorch-build/pytorch ]; then
echo "--- PyTorch source dir ---"
ls ~/pytorch-build/pytorch/ | head -20
echo "..."
cd ~/pytorch-build/pytorch
echo "--- Git info ---"
git log --oneline -1 2>/dev/null || echo "Not a git repo"
git describe --tags 2>/dev/null || echo "No tags"
echo "--- Submodule count ---"
git submodule status 2>/dev/null | wc -l
echo "--- Failed submodules ---"
git submodule status 2>/dev/null | grep "^-" | head -10
echo "--- Hipify check ---"
[ -d "aten/src/ATen/hip" ] && echo "Hipify: DONE" || echo "Hipify: NOT DONE"
[ -d "build" ] && echo "Build dir: $(du -sh build)" || echo "No build dir"
ls dist/*.whl 2>/dev/null || echo "No wheel yet"
fi
else
echo "NO pytorch-build dir"
fi
echo "=== DEPS ==="
nproc
cmake --version 2>/dev/null | head -1
ninja --version 2>/dev/null || echo "No ninja"
hipcc --version 2>/dev/null | head -3
python3.11 -c "import numpy; print('numpy:', numpy.__version__)" 2>/dev/null || echo "No numpy"
df -h / | tail -1
echo "=== DONE ==="
Claude Test Scripts/check_clip.sh
+40
View File
@@ -0,0 +1,40 @@
#!/bin/bash
API="http://127.0.0.1:8188"
echo "=== CLIPLoader info ==="
curl -s "$API/object_info/CLIPLoader" | python3.11 -c "
import sys, json
d = json.load(sys.stdin)
info = d.get('CLIPLoader', {})
inp = info.get('input', {})
print(json.dumps(inp, indent=2))
"
echo ""
echo "=== Check what lumina2 expects ==="
curl -s "$API/object_info/CLIPLoader" | python3.11 -c "
import sys, json
d = json.load(sys.stdin)
info = d.get('CLIPLoader', {})
desc = info.get('description', 'N/A')
print('Description:', desc)
out = info.get('output', [])
print('Output:', out)
"
echo ""
echo "=== Check DualCLIPLoader ==="
curl -s "$API/object_info/DualCLIPLoader" | python3.11 -c "
import sys, json
d = json.load(sys.stdin)
info = d.get('DualCLIPLoader', {})
inp = info.get('input', {})
print(json.dumps(inp, indent=2))
"
echo ""
echo "=== Available text_encoder files ==="
ls -lh /home/fabian/ComfyUI/models/text_encoders/
echo ""
echo "=== Available clip files ==="
ls -lh /home/fabian/ComfyUI/models/clip/
Claude Test Scripts/check_errors.sh
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
echo "=== ERRORS in build log ==="
grep -iE "error:|FAILED|fatal" ~/pytorch-build-progress.log 2>/dev/null | grep -v "Warnung" | grep -v "Anmerkung" | grep -v "error=return" | grep -v "error=non-virtual" | grep -v "error=range" | grep -v "error=bool" | grep -v "error=format" | grep -v "Werror" | grep -v "error=missing" | tail -30
echo ""
echo "=== LAST 5 LINES of build.log ==="
tail -5 ~/pytorch-build.log
echo ""
echo "=== BUILD DIR SIZE ==="
du -sh ~/pytorch-build/build 2>/dev/null
echo ""
echo "=== WHEEL CHECK ==="
ls ~/pytorch-build/dist/*.whl 2>/dev/null || echo "No wheel"
Claude Test Scripts/check_nodes.sh
+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
API="http://127.0.0.1:8188"
echo "=== GGUF nodes ==="
curl -s "$API/object_info" | python3.11 -c "
import sys, json
d = json.load(sys.stdin)
for k in sorted(d.keys()):
if 'gguf' in k.lower():
print(k)
info = d[k]
if 'input' in info and 'required' in info['input']:
for param, cfg in info['input']['required'].items():
print(f' {param}: {cfg}')
"
echo ""
echo "=== CLIPLoader info ==="
curl -s "$API/object_info/CLIPLoader" | python3.11 -c "
import sys, json
d = json.load(sys.stdin)
if 'CLIPLoader' in d:
info = d['CLIPLoader']
if 'input' in info and 'required' in info['input']:
for param, cfg in info['input']['required'].items():
print(f' {param}: {cfg}')
"
echo ""
echo "=== Model folders ==="
echo "unet:"; ls /home/fabian/ComfyUI/models/unet/ 2>/dev/null
echo "diffusion_models:"; ls /home/fabian/ComfyUI/models/diffusion_models/ 2>/dev/null
echo "text_encoders:"; ls /home/fabian/ComfyUI/models/text_encoders/ 2>/dev/null
echo "clip:"; ls /home/fabian/ComfyUI/models/clip/ 2>/dev/null
echo "vae:"; ls /home/fabian/ComfyUI/models/vae/ 2>/dev/null
Claude Test Scripts/check_progress.sh
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
echo "=== Last progress ==="
grep -oP '\[\d+/\d+\]' ~/pytorch-build.log | tail -3
echo ""
echo "=== Process alive? ==="
pgrep -f build_pytorch | head -3
echo ""
echo "=== Last 10 lines ==="
tail -10 ~/pytorch-build.log
echo ""
echo "=== Disk usage ==="
du -sh ~/pytorch-build
Claude Test Scripts/check_state.sh
+37
View File
@@ -0,0 +1,37 @@
#!/bin/bash
echo "=== ENV ==="
echo "HSA_OVERRIDE=$HSA_OVERRIDE_GFX_VERSION"
echo "SDMA=$HSA_ENABLE_SDMA"
echo "HIP_VIS=$HIP_VISIBLE_DEVICES"
echo "=== RAM ==="
free -m
echo "=== COMFYUI ==="
if [ -f ~/ComfyUI/main.py ]; then echo "ComfyUI: EXISTS"; else echo "ComfyUI: MISSING"; fi
if [ -f ~/ComfyUI/venv/bin/python3.11 ]; then echo "Venv: EXISTS"; else echo "Venv: MISSING"; fi
echo "=== MODELS ==="
ls -lh ~/ComfyUI/models/unet/ 2>/dev/null || echo "NO unet dir"
ls -lh ~/ComfyUI/models/text_encoders/ 2>/dev/null || echo "NO text_encoders dir"
ls -lh ~/ComfyUI/models/vae/ 2>/dev/null || echo "NO vae dir"
echo "=== PYTORCH BUILD ==="
if [ -d ~/pytorch-build ]; then du -sh ~/pytorch-build; else echo "NO BUILD DIR"; fi
echo "=== PYTHON ==="
python3.11 --version 2>/dev/null || echo "NO python3.11"
if [ -f ~/ComfyUI/venv/bin/python ]; then
source ~/ComfyUI/venv/bin/activate
python -c "import torch; print('PyTorch:', torch.__version__); print('CUDA avail:', torch.cuda.is_available()); print('Arch list:', torch.cuda.get_arch_list())" 2>/dev/null || echo "PyTorch import failed"
fi
echo "=== DMESG ==="
sudo dmesg 2>/dev/null | grep -cE "KIQ|GPU died|GPU unreachable" || echo "0"
sudo dmesg 2>/dev/null | grep -c "BC-250" || echo "0"
sudo dmesg 2>/dev/null | grep "BC-250" | head -3
echo "=== CUSTOM NODES ==="
ls ~/ComfyUI/custom_nodes/ 2>/dev/null || echo "NO custom_nodes"
echo "=== DONE ==="
Claude Test Scripts/download_models.sh
+110
View File
@@ -0,0 +1,110 @@
#!/bin/bash
# Download all models for ComfyUI Z-Image Turbo on BC-250
set -euo pipefail
cd ~/ComfyUI
source venv/bin/activate
pip install -q huggingface-hub safetensors
echo ""
echo "=== Step 1: Download Z-Image Turbo GGUF (5.2 GB) ==="
mkdir -p ~/ComfyUI/models/unet
python3 -c "
from huggingface_hub import hf_hub_download
print('Downloading z_image_turbo-Q5_K_S.gguf from jayn7/Z-Image-Turbo-GGUF ...')
hf_hub_download(
'jayn7/Z-Image-Turbo-GGUF',
'z_image_turbo-Q5_K_S.gguf',
local_dir='/home/fabian/ComfyUI/models/unet'
)
print('Done!')
"
echo ""
echo "=== Step 2: Download Gemma 2 2B Text Encoder shards (9.8 GB) ==="
mkdir -p ~/sd-models/text_encoders/lumina2_gemma2_2b
python3 -c "
from huggingface_hub import hf_hub_download
import os
repo = 'Alpha-VLLM/Lumina-Image-2.0'
dest = '/home/fabian/sd-models/text_encoders/lumina2_gemma2_2b'
os.makedirs(dest, exist_ok=True)
files = [
'text_encoder/config.json',
'text_encoder/model.safetensors.index.json',
'text_encoder/model-00001-of-00003.safetensors',
'text_encoder/model-00002-of-00003.safetensors',
'text_encoder/model-00003-of-00003.safetensors',
]
for f in files:
print(f'Downloading {f}...')
hf_hub_download(repo, f, local_dir=dest)
print('Done!')
"
echo ""
echo "=== Step 3: Merge Text Encoder shards into single safetensors ==="
mkdir -p ~/ComfyUI/models/text_encoders
python3 << 'EOF'
import safetensors.torch
import os, json
base_dir = "/home/fabian/sd-models/text_encoders/lumina2_gemma2_2b/text_encoder"
output = "/home/fabian/ComfyUI/models/text_encoders/gemma2_2b_lumina2.safetensors"
with open(os.path.join(base_dir, "model.safetensors.index.json")) as f:
index = json.load(f)
all_tensors = {}
shards = set(index["weight_map"].values())
print(f"Loading {len(shards)} shards with {len(index['weight_map'])} tensors...")
for shard in sorted(shards):
path = os.path.join(base_dir, shard)
print(f" Loading {shard}...")
tensors = safetensors.torch.load_file(path, device="cpu")
all_tensors.update(tensors)
print(f"Total tensors: {len(all_tensors)}")
print(f"Saving merged file to {output}...")
safetensors.torch.save_file(all_tensors, output)
sz = os.path.getsize(output) / 1e9
print(f"Done! Size: {sz:.2f} GB")
EOF
echo ""
echo "=== Step 4: Download VAE (335 MB) ==="
mkdir -p ~/ComfyUI/models/vae
python3 -c "
from huggingface_hub import hf_hub_download
print('Downloading ae.safetensors ...')
hf_hub_download(
'black-forest-labs/FLUX.1-schnell',
'ae.safetensors',
local_dir='/home/fabian/ComfyUI/models/vae'
)
print('Done!')
"
echo ""
echo "=== Step 5: Download example workflow ==="
mkdir -p ~/ComfyUI/user/default/workflows
python3 -c "
from huggingface_hub import hf_hub_download
hf_hub_download(
'jayn7/Z-Image-Turbo-GGUF',
'example_workflow.json',
local_dir='/home/fabian/ComfyUI/user/default/workflows'
)
print('Workflow downloaded!')
"
echo ""
echo "=== Verification ==="
ls -lh ~/ComfyUI/models/unet/z_image_turbo-Q5_K_S.gguf 2>/dev/null || echo "MISSING: GGUF model"
ls -lh ~/ComfyUI/models/text_encoders/gemma2_2b_lumina2.safetensors 2>/dev/null || echo "MISSING: Text encoder"
ls -lh ~/ComfyUI/models/vae/ae.safetensors 2>/dev/null || echo "MISSING: VAE"
echo ""
echo "=== ALL DOWNLOADS COMPLETE ==="
Claude Test Scripts/find_tv.sh
+22
View File
@@ -0,0 +1,22 @@
#!/bin/bash
source /home/fabian/ComfyUI/venv/bin/activate
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
# Find all torchvision directories
echo "=== torchvision locations ==="
find /home/fabian/ComfyUI/venv -name 'torchvision' -type d 2>/dev/null
echo "=== which init.py is loaded ==="
python3.11 -c "import torchvision; print(torchvision.__file__)" 2>&1 || true
echo "=== line 10 of loaded init ==="
python3.11 -c "
import importlib.util
spec = importlib.util.find_spec('torchvision')
print('Location:', spec.origin)
" 2>&1 || true
echo "=== check our patched file ==="
sed -n '10,14p' /home/fabian/ComfyUI/venv/lib/python3.11/site-packages/torchvision/__init__.py
Claude Test Scripts/gpu_monitor.sh
+58
View File
@@ -0,0 +1,58 @@
#!/bin/bash
# =============================================================
# BC-250 GPU Activity Monitor
# Since rocm-smi GPU utilization is broken (always 0%),
# we use alternative metrics to verify GPU compute usage.
# =============================================================
echo "=== BC-250 GPU Activity Monitor ==="
echo "NOTE: GPU % utilization is broken on this hardware."
echo "Using alternative metrics instead."
echo ""
while true; do
TIMESTAMP=$(date '+%H:%M:%S')
# 1. GPU Clock — high clock = GPU active
SCLK=$(cat /sys/class/drm/card0/device/pp_dpm_sclk 2>/dev/null | grep '\*' | awk '{print $2}')
# 2. GPU power draw (if available)
POWER=$(cat /sys/class/drm/card0/device/hwmon/hwmon*/power1_average 2>/dev/null)
if [ -n "$POWER" ]; then
POWER_W=$(echo "scale=1; $POWER / 1000000" | bc 2>/dev/null || echo "N/A")
else
POWER_W="N/A"
fi
# 3. GPU VRAM usage (shared memory allocated by GPU)
VRAM_USED=$(cat /sys/class/drm/card0/device/mem_info_vram_used 2>/dev/null)
VRAM_TOTAL=$(cat /sys/class/drm/card0/device/mem_info_vram_total 2>/dev/null)
if [ -n "$VRAM_USED" ] && [ -n "$VRAM_TOTAL" ]; then
VRAM_MB=$(echo "scale=0; $VRAM_USED / 1048576" | bc 2>/dev/null || echo "N/A")
VRAM_TOTAL_MB=$(echo "scale=0; $VRAM_TOTAL / 1048576" | bc 2>/dev/null || echo "N/A")
else
VRAM_MB="N/A"
VRAM_TOTAL_MB="N/A"
fi
# 4. GPU temperature
TEMP=$(cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null)
if [ -n "$TEMP" ]; then
TEMP_C=$(echo "scale=0; $TEMP / 1000" | bc 2>/dev/null || echo "N/A")
else
TEMP_C="N/A"
fi
# 5. HIP processes using GPU
HIP_PROCS=$(ls /proc/*/maps 2>/dev/null | xargs grep -l "libamdhip64\|libhsa-runtime" 2>/dev/null | wc -l)
# 6. Kernel GPU activity (interrupts)
GPU_IRQ=$(cat /proc/interrupts 2>/dev/null | grep amdgpu | awk '{sum=0; for(i=2;i<=NF-2;i++) sum+=$i; print sum}' | head -1)
# 7. System RAM (since BC-250 shares system RAM as VRAM)
RAM_INFO=$(free -m | grep Mem | awk '{printf "%dMB / %dMB (%.0f%%)", $3, $2, $3/$2*100}')
echo "[$TIMESTAMP] Clock: ${SCLK:-N/A} | Power: ${POWER_W}W | Temp: ${TEMP_C}°C | VRAM: ${VRAM_MB}/${VRAM_TOTAL_MB}MB | HIP Procs: $HIP_PROCS | RAM: $RAM_INFO"
sleep 2
done
Claude Test Scripts/gpu_test.py
+13
View File
@@ -0,0 +1,13 @@
import torch
print(f"PyTorch: {torch.__version__}")
print(f"CUDA available: {torch.cuda.is_available()}")
print(f"HIP version: {torch.version.hip}")
print(f"Device count: {torch.cuda.device_count()}")
if torch.cuda.is_available():
print(f"Device name: {torch.cuda.get_device_name(0)}")
t = torch.randn(100, 100, device="cuda")
r = torch.mm(t, t)
print(f"GPU matmul OK: result shape {r.shape}")
print("GPU COMPUTE WORKS!")
else:
print("NO GPU DETECTED")
Claude Test Scripts/gpu_test2.py
+29
View File
@@ -0,0 +1,29 @@
import torch
import os
print(f"HSA_OVERRIDE_GFX_VERSION={os.environ.get('HSA_OVERRIDE_GFX_VERSION','NOT SET')}")
print(f"PyTorch arch list: {torch.cuda.get_arch_list()}")
print(f"Device: {torch.cuda.get_device_name(0)}")
print(f"GCN Arch: {torch.cuda.get_device_properties(0).gcnArchName}")
try:
t = torch.randn(256, 256, device="cuda")
r = torch.mm(t, t)
val = r[0,0].item()
print(f"GPU matmul OK! result[0,0]={val:.4f}")
# Bigger test
a = torch.randn(1024, 1024, device="cuda")
b = torch.randn(1024, 1024, device="cuda")
c = torch.mm(a, b)
print(f"Large matmul OK! shape={c.shape}")
# Test fp32 conv
x = torch.randn(1, 3, 64, 64, device="cuda")
conv = torch.nn.Conv2d(3, 16, 3, padding=1).cuda()
y = conv(x)
print(f"Conv2d OK! output shape={y.shape}")
print("ALL GPU TESTS PASSED!")
except Exception as e:
print(f"FAILED: {e}")
Claude Test Scripts/launch.sh
+26
View File
@@ -0,0 +1,26 @@
#!/bin/bash
cd /home/fabian/ComfyUI
source venv/bin/activate
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export NUMEXPR_NUM_THREADS=12
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
# Unset harmful vars
unset GPU_MAX_HW_QUEUES 2>/dev/null || true
unset HIP_LAUNCH_BLOCKING 2>/dev/null || true
echo "Starting ComfyUI..."
echo "PyTorch: $(python3.11 -c 'import torch; print(torch.__version__)')"
echo "GPU: $(python3.11 -c 'import torch; print(torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"NONE\")')"
python main.py \
--listen 0.0.0.0 \
--port 8188 \
--force-fp32 \
--lowvram
Claude Test Scripts/launch2.sh
+28
View File
@@ -0,0 +1,28 @@
#!/bin/bash
# ComfyUI Launch for BC-250 APU — unified memory, no lowvram
cd /home/fabian/ComfyUI
source venv/bin/activate
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export NUMEXPR_NUM_THREADS=12
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
# Unset harmful vars
unset GPU_MAX_HW_QUEUES 2>/dev/null || true
unset HIP_LAUNCH_BLOCKING 2>/dev/null || true
echo "Starting ComfyUI (APU mode - no lowvram)..."
# --force-fp32: required for gfx1010 (no native fp16 support in some ops)
# NO --lowvram: APU has unified memory, offloading is counterproductive
# --gpu-only: keep everything in VRAM (which IS the system RAM on APU)
python main.py \
--listen 0.0.0.0 \
--port 8188 \
--force-fp32 \
--gpu-only
Claude Test Scripts/progress.sh
+6
View File
@@ -0,0 +1,6 @@
#!/bin/bash
grep -oP '\[\d+/\d+\]' ~/pytorch-build-progress.log | tail -1
echo "---"
pgrep -c cc1plus 2>/dev/null || echo "0 compilers"
echo "compiler_procs"
free -m | grep Speicher
Claude Test Scripts/start-comfyui.sh
+55
View File
@@ -0,0 +1,55 @@
#!/bin/bash
# =============================================================
# ComfyUI Launch Script for AMD BC-250 (ROCm / gfx1013 → gfx1030 spoof)
# =============================================================
set -euo pipefail
echo "=========================================="
echo " ComfyUI — BC-250 ROCm Launcher"
echo "=========================================="
# --- GPU Health Check ---
if dmesg 2>/dev/null | tail -50 | grep -qi "KIQ fence timeout"; then
echo "[ABORT] KIQ fence timeout detected in dmesg — reboot required!"
exit 1
fi
echo "[OK] GPU health check passed"
# --- ROCm Environment for BC-250 ---
# CRITICAL: gfx1030 spoof (not gfx1010!) — PyTorch ROCm 6.2 has no gfx1010 kernels
export HSA_OVERRIDE_GFX_VERSION=10.3.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PATH="/opt/rocm/bin:$PATH"
export LD_LIBRARY_PATH="/opt/rocm/lib"
# --- Performance: DO NOT set these ---
unset GPU_MAX_HW_QUEUES 2>/dev/null || true
unset HIP_LAUNCH_BLOCKING 2>/dev/null || true
unset GGML_CUDA_ENABLE_UNIFIED_MEMORY 2>/dev/null || true
unset GGML_HIP_HOST_ALLOC 2>/dev/null || true
unset GGML_CUDA_NO_PINNED 2>/dev/null || true
unset GGML_HIP_NO_COARSE_GRAIN 2>/dev/null || true
unset HSA_DISABLE_FRAGMENT_ALLOCATOR 2>/dev/null || true
# --- PyTorch ROCm tuning ---
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
echo "[OK] ROCm environment configured (gfx1030 spoof)"
# --- Activate venv ---
cd ~/ComfyUI
source venv/bin/activate
# --- Launch ComfyUI ---
echo "[START] Launching ComfyUI on http://0.0.0.0:8188"
echo "=========================================="
python main.py \
--listen 0.0.0.0 \
--port 8188 \
--force-fp32 \
--lowvram \
"$@"
Claude Test Scripts/start_comfyui.sh
+66
View File
@@ -0,0 +1,66 @@
#!/bin/bash
# =============================================================
# ComfyUI Launch Script for AMD BC-250 (ROCm / gfx1013)
# Trusted source: BC250 ROCm Install README.md
# =============================================================
set -euo pipefail
echo "=========================================="
echo " ComfyUI — BC-250 ROCm Launcher"
echo "=========================================="
# --- GPU Health Check ---
if dmesg 2>/dev/null | tail -50 | grep -qi "KIQ fence timeout"; then
echo "[ABORT] KIQ fence timeout detected in dmesg — reboot required!"
exit 1
fi
echo "[OK] GPU health check passed"
# --- ROCm Environment for BC-250 (gfx1013 mapped to gfx1010) ---
# Per BC250 ROCm Install docs — ONLY these vars
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
export ROCM_PATH=/opt/rocm
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PATH="/opt/rocm/bin:$PATH"
export LD_LIBRARY_PATH="/opt/rocm/lib"
# --- Unset harmful old workarounds ---
unset GPU_MAX_HW_QUEUES 2>/dev/null || true
unset HIP_LAUNCH_BLOCKING 2>/dev/null || true
unset GGML_CUDA_ENABLE_UNIFIED_MEMORY 2>/dev/null || true
unset GGML_HIP_HOST_ALLOC 2>/dev/null || true
unset GGML_CUDA_NO_PINNED 2>/dev/null || true
unset GGML_HIP_NO_COARSE_GRAIN 2>/dev/null || true
unset HSA_DISABLE_FRAGMENT_ALLOCATOR 2>/dev/null || true
# --- PyTorch ROCm tuning ---
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
# --- CPU thread optimization for VAE and other CPU-bound ops ---
# BC-250 has 12 threads — use them all
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export NUMEXPR_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
echo "[OK] ROCm environment configured"
echo " GFX Override: $HSA_OVERRIDE_GFX_VERSION"
echo " OMP Threads: $OMP_NUM_THREADS"
# --- Activate venv ---
cd ~/ComfyUI
source venv/bin/activate
# --- Launch ComfyUI ---
echo "[START] Launching ComfyUI on http://0.0.0.0:8188"
echo "=========================================="
python main.py \
--listen 0.0.0.0 \
--port 8188 \
--force-fp32 \
--lowvram \
"$@"
Claude Test Scripts/test_arch.sh
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
cat > /tmp/test_arch.hip << 'HIPEOF'
#include <hip/hip_runtime.h>
__global__ void test_kernel(float *a) { a[threadIdx.x] = 1.0f; }
HIPEOF
for arch in gfx1013 gfx1010 gfx10-1-generic; do
echo "=== Testing $arch ==="
/opt/rocm/bin/hipcc --offload-arch=$arch -c /tmp/test_arch.hip -o /tmp/test_${arch}.o 2>&1
echo "Exit: $?"
if [ -f /tmp/test_${arch}.o ]; then
ls -l /tmp/test_${arch}.o
fi
echo ""
done
Claude Test Scripts/test_gen2.sh
+140
View File
@@ -0,0 +1,140 @@
#!/bin/bash
# Test image generation with correct GGUF workflow
API="http://127.0.0.1:8188"
echo "=== Queueing Z-Image Turbo generation ==="
WORKFLOW='{
"prompt": {
"1": {
"class_type": "UnetLoaderGGUF",
"inputs": {
"unet_name": "z_image_turbo-Q5_K_S.gguf"
}
},
"2": {
"class_type": "CLIPLoaderGGUF",
"inputs": {
"clip_name": "gemma2_2b_lumina2.safetensors",
"type": "lumina2"
}
},
"3": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "a beautiful mountain landscape at sunset, golden light, detailed, 4k",
"clip": ["2", 0]
}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "blurry, ugly, distorted",
"clip": ["2", 0]
}
},
"5": {
"class_type": "EmptyLatentImage",
"inputs": {
"width": 512,
"height": 512,
"batch_size": 1
}
},
"6": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0],
"positive": ["3", 0],
"negative": ["4", 0],
"latent_image": ["5", 0],
"seed": 42,
"steps": 8,
"cfg": 3.0,
"sampler_name": "euler",
"scheduler": "normal",
"denoise": 1.0
}
},
"7": {
"class_type": "VAELoader",
"inputs": {
"vae_name": "ae.safetensors"
}
},
"8": {
"class_type": "VAEDecode",
"inputs": {
"samples": ["6", 0],
"vae": ["7", 0]
}
},
"9": {
"class_type": "SaveImage",
"inputs": {
"images": ["8", 0],
"filename_prefix": "bc250_test"
}
}
}
}'
RESPONSE=$(curl -s -X POST "$API/prompt" -H "Content-Type: application/json" -d "$WORKFLOW")
echo "Response: $RESPONSE" | head -c 500
echo ""
PROMPT_ID=$(echo "$RESPONSE" | python3.11 -c "import sys,json; print(json.load(sys.stdin).get('prompt_id','NONE'))" 2>/dev/null)
echo "Prompt ID: $PROMPT_ID"
if [ "$PROMPT_ID" = "NONE" ] || [ -z "$PROMPT_ID" ]; then
echo "ERROR: Failed to queue!"
exit 1
fi
echo ""
echo "=== Waiting for generation (up to 10 min) ==="
for i in $(seq 1 120); do
sleep 5
STATUS=$(curl -s "$API/history/$PROMPT_ID" 2>/dev/null)
HAS_OUTPUT=$(echo "$STATUS" | python3.11 -c "
import sys, json
data = json.load(sys.stdin)
pid = '$PROMPT_ID'
if pid in data:
outputs = data[pid].get('outputs', {})
status = data[pid].get('status', {})
if status.get('status_str') == 'error':
msgs = status.get('messages', [])
print('ERROR:' + str(msgs[-1] if msgs else 'unknown'))
elif '9' in outputs:
images = outputs['9'].get('images', [])
if images:
print('DONE:' + images[0].get('filename', 'unknown'))
else:
print('PROCESSING')
else:
print('PROCESSING')
else:
print('WAITING')
" 2>/dev/null)
echo "[$((i*5))s] $HAS_OUTPUT"
if [[ "$HAS_OUTPUT" == DONE:* ]]; then
FILENAME=${HAS_OUTPUT#DONE:}
echo ""
echo "=========================================="
echo " SUCCESS - Image generated!"
echo " File: /home/fabian/ComfyUI/output/$FILENAME"
ls -lh "/home/fabian/ComfyUI/output/$FILENAME" 2>/dev/null
echo "=========================================="
exit 0
fi
if [[ "$HAS_OUTPUT" == ERROR:* ]]; then
echo ""
echo "GENERATION FAILED: $HAS_OUTPUT"
echo "=== ComfyUI log tail ==="
tail -30 /home/fabian/comfyui2.log
exit 1
fi
done
echo "TIMEOUT after 10 minutes"
tail -20 /home/fabian/comfyui2.log
Claude Test Scripts/test_gen3.sh
+145
View File
@@ -0,0 +1,145 @@
#!/bin/bash
# Test image generation - v3 with CLIPLoader (not GGUF) for safetensors clip
API="http://127.0.0.1:8188"
echo "=== Queueing Z-Image Turbo generation v3 ==="
WORKFLOW='{
"prompt": {
"1": {
"class_type": "UnetLoaderGGUF",
"inputs": {
"unet_name": "z_image_turbo-Q5_K_S.gguf"
}
},
"2": {
"class_type": "CLIPLoader",
"inputs": {
"clip_name": "gemma2_2b_lumina2.safetensors",
"type": "lumina2"
}
},
"3": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "a beautiful mountain landscape at sunset, golden light, detailed, 4k",
"clip": ["2", 0]
}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "blurry, ugly, distorted",
"clip": ["2", 0]
}
},
"5": {
"class_type": "EmptyLatentImage",
"inputs": {
"width": 512,
"height": 512,
"batch_size": 1
}
},
"6": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0],
"positive": ["3", 0],
"negative": ["4", 0],
"latent_image": ["5", 0],
"seed": 42,
"steps": 8,
"cfg": 3.0,
"sampler_name": "euler",
"scheduler": "normal",
"denoise": 1.0
}
},
"7": {
"class_type": "VAELoader",
"inputs": {
"vae_name": "ae.safetensors"
}
},
"8": {
"class_type": "VAEDecode",
"inputs": {
"samples": ["6", 0],
"vae": ["7", 0]
}
},
"9": {
"class_type": "SaveImage",
"inputs": {
"images": ["8", 0],
"filename_prefix": "bc250_test"
}
}
}
}'
RESPONSE=$(curl -s -X POST "$API/prompt" -H "Content-Type: application/json" -d "$WORKFLOW")
echo "Response: $(echo $RESPONSE | head -c 200)"
echo ""
PROMPT_ID=$(echo "$RESPONSE" | python3.11 -c "import sys,json; print(json.load(sys.stdin).get('prompt_id','NONE'))" 2>/dev/null)
echo "Prompt ID: $PROMPT_ID"
if [ "$PROMPT_ID" = "NONE" ] || [ -z "$PROMPT_ID" ]; then
echo "ERROR: Failed to queue!"
echo "Full response: $RESPONSE"
exit 1
fi
echo ""
echo "=== Waiting for generation (up to 10 min) ==="
for i in $(seq 1 120); do
sleep 5
STATUS=$(curl -s "$API/history/$PROMPT_ID" 2>/dev/null)
HAS_OUTPUT=$(echo "$STATUS" | python3.11 -c "
import sys, json
data = json.load(sys.stdin)
pid = '$PROMPT_ID'
if pid in data:
outputs = data[pid].get('outputs', {})
status = data[pid].get('status', {})
if status.get('status_str') == 'error':
msgs = status.get('messages', [])
for m in msgs:
if isinstance(m, list) and len(m)>1 and isinstance(m[1],dict):
em = m[1].get('exception_message','')
if em:
print('ERROR:' + em[:200])
break
else:
print('ERROR:unknown')
elif '9' in outputs:
images = outputs['9'].get('images', [])
if images:
print('DONE:' + images[0].get('filename', 'unknown'))
else:
print('PROCESSING')
else:
print('PROCESSING')
else:
print('WAITING')
" 2>/dev/null)
echo "[$((i*5))s] $HAS_OUTPUT"
if [[ "$HAS_OUTPUT" == DONE:* ]]; then
FILENAME=${HAS_OUTPUT#DONE:}
echo ""
echo "=========================================="
echo " SUCCESS - Image generated!"
echo " File: /home/fabian/ComfyUI/output/$FILENAME"
ls -lh "/home/fabian/ComfyUI/output/$FILENAME" 2>/dev/null
echo "=========================================="
exit 0
fi
if [[ "$HAS_OUTPUT" == ERROR:* ]]; then
echo ""
echo "GENERATION FAILED: $HAS_OUTPUT"
tail -10 /home/fabian/comfyui2.log
exit 1
fi
done
echo "TIMEOUT"
Claude Test Scripts/test_generate.sh
+142
View File
@@ -0,0 +1,142 @@
#!/bin/bash
# Test image generation via ComfyUI API
# Workflow: Z-Image Turbo GGUF + Gemma2 CLIP + AE VAE
API="http://127.0.0.1:8188"
echo "=== Checking ComfyUI API ==="
curl -s "$API/system_stats" | python3.11 -m json.tool 2>/dev/null | head -20
echo ""
echo "=== Listing available models ==="
curl -s "$API/models/unet" 2>/dev/null
echo ""
curl -s "$API/models/clip" 2>/dev/null
echo ""
curl -s "$API/models/vae" 2>/dev/null
echo ""
echo "=== Queueing image generation ==="
WORKFLOW='{
"prompt": {
"1": {
"class_type": "UNETLoader",
"inputs": {
"unet_name": "z_image_turbo-Q5_K_S.gguf",
"weight_dtype": "default"
}
},
"2": {
"class_type": "DualCLIPLoader",
"inputs": {
"clip_name1": "gemma2_2b_lumina2.safetensors",
"clip_name2": "gemma2_2b_lumina2.safetensors",
"type": "lumina2"
}
},
"3": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "a beautiful mountain landscape at sunset, golden light, detailed, 4k",
"clip": ["2", 0]
}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "blurry, ugly, distorted",
"clip": ["2", 0]
}
},
"5": {
"class_type": "EmptyLatentImage",
"inputs": {
"width": 512,
"height": 512,
"batch_size": 1
}
},
"6": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0],
"positive": ["3", 0],
"negative": ["4", 0],
"latent_image": ["5", 0],
"seed": 42,
"steps": 8,
"cfg": 3.0,
"sampler_name": "euler",
"scheduler": "normal",
"denoise": 1.0
}
},
"7": {
"class_type": "VAELoader",
"inputs": {
"vae_name": "ae.safetensors"
}
},
"8": {
"class_type": "VAEDecode",
"inputs": {
"samples": ["6", 0],
"vae": ["7", 0]
}
},
"9": {
"class_type": "SaveImage",
"inputs": {
"images": ["8", 0],
"filename_prefix": "bc250_test"
}
}
}
}'
RESPONSE=$(curl -s -X POST "$API/prompt" -H "Content-Type: application/json" -d "$WORKFLOW")
echo "Queue response: $RESPONSE"
PROMPT_ID=$(echo "$RESPONSE" | python3.11 -c "import sys,json; print(json.load(sys.stdin).get('prompt_id','NONE'))" 2>/dev/null)
echo "Prompt ID: $PROMPT_ID"
if [ "$PROMPT_ID" = "NONE" ] || [ -z "$PROMPT_ID" ]; then
echo "ERROR: Failed to queue prompt!"
exit 1
fi
echo ""
echo "=== Waiting for generation ==="
for i in $(seq 1 60); do
sleep 5
STATUS=$(curl -s "$API/history/$PROMPT_ID" 2>/dev/null)
HAS_OUTPUT=$(echo "$STATUS" | python3.11 -c "
import sys, json
data = json.load(sys.stdin)
if '$PROMPT_ID' in data:
outputs = data['$PROMPT_ID'].get('outputs', {})
if '9' in outputs:
images = outputs['9'].get('images', [])
if images:
print('DONE:' + images[0].get('filename', 'unknown'))
else:
print('PROCESSING')
else:
print('PROCESSING')
else:
print('WAITING')
" 2>/dev/null)
echo "[$((i*5))s] $HAS_OUTPUT"
if [[ "$HAS_OUTPUT" == DONE:* ]]; then
FILENAME=${HAS_OUTPUT#DONE:}
echo ""
echo "=== SUCCESS ==="
echo "Image generated: $FILENAME"
echo "File location: /home/fabian/ComfyUI/output/$FILENAME"
ls -lh "/home/fabian/ComfyUI/output/$FILENAME" 2>/dev/null
exit 0
fi
done
echo "TIMEOUT: Generation did not complete in 5 minutes"
echo "=== Checking queue ==="
curl -s "$API/queue" | python3.11 -m json.tool 2>/dev/null | head -20
Claude Test Scripts/test_gpu.sh
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
source /home/fabian/ComfyUI/venv/bin/activate
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
python3.11 -c "
import torch
print('PyTorch version:', torch.__version__)
print('HIP version:', torch.version.hip)
print('CUDA available:', torch.cuda.is_available())
if torch.cuda.is_available():
print('Device name:', torch.cuda.get_device_name(0))
print('Device count:', torch.cuda.device_count())
# Quick tensor test
x = torch.randn(100, 100, device='cuda')
y = torch.randn(100, 100, device='cuda')
z = x @ y
print('GPU matmul test: OK, shape', z.shape)
print('Memory allocated:', torch.cuda.memory_allocated(0) / 1024 / 1024, 'MB')
else:
print('NO GPU DETECTED')
"
Claude Test Scripts/test_runtime.sh
+64
View File
@@ -0,0 +1,64 @@
#!/bin/bash
# Runtime test: which arch actually RUNS on the BC-250?
export HSA_ENABLE_SDMA=0
cat > /tmp/run_test.hip << 'HIPEOF'
#include <hip/hip_runtime.h>
#include <cstdio>
__global__ void add_kernel(float *a, float *b, float *c, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) c[i] = a[i] + b[i];
}
int main() {
hipDeviceProp_t prop;
hipGetDeviceProperties(&prop, 0);
printf("Device: %s\n", prop.name);
printf("GCN Arch: %s\n", prop.gcnArchName);
const int N = 256;
float h_a[N], h_b[N], h_c[N];
for (int i = 0; i < N; i++) { h_a[i] = i; h_b[i] = i * 2; }
float *d_a, *d_b, *d_c;
hipMallocManaged(&d_a, N * sizeof(float));
hipMallocManaged(&d_b, N * sizeof(float));
hipMallocManaged(&d_c, N * sizeof(float));
memcpy(d_a, h_a, N * sizeof(float));
memcpy(d_b, h_b, N * sizeof(float));
add_kernel<<<1, N>>>(d_a, d_b, d_c, N);
hipDeviceSynchronize();
hipError_t err = hipGetLastError();
if (err != hipSuccess) {
printf("FAIL: %s\n", hipGetErrorString(err));
hipFree(d_a); hipFree(d_b); hipFree(d_c);
return 1;
}
// Verify
int ok = 1;
for (int i = 0; i < N; i++) {
if (d_c[i] != h_a[i] + h_b[i]) { ok = 0; break; }
}
printf("Compute: %s\n", ok ? "PASS" : "FAIL");
hipFree(d_a); hipFree(d_b); hipFree(d_c);
return ok ? 0 : 1;
}
HIPEOF
for arch in gfx1013 gfx1010 gfx10-1-generic; do
echo "=== Runtime test: $arch ==="
/opt/rocm/bin/hipcc --offload-arch=$arch /tmp/run_test.hip -o /tmp/run_test_${arch} 2>&1
if [ $? -eq 0 ]; then
echo "Compiled OK, running..."
timeout 10 /tmp/run_test_${arch} 2>&1
echo "Runtime exit: $?"
else
echo "Compile FAILED"
fi
echo ""
done
Claude Test Scripts/test_runtime2.sh
+64
View File
@@ -0,0 +1,64 @@
#!/bin/bash
export HSA_ENABLE_SDMA=0
cat > /tmp/run_test2.hip << 'HIPEOF'
#include <hip/hip_runtime.h>
#include <cstdio>
__global__ void add_kernel(float *a, float *b, float *c, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) c[i] = a[i] + b[i];
}
int main() {
hipDeviceProp_t prop;
hipGetDeviceProperties(&prop, 0);
printf("Device: %s\n", prop.name);
printf("GCN Arch: %s\n", prop.gcnArchName);
const int N = 256;
float h_a[N], h_b[N], h_c[N];
for (int i = 0; i < N; i++) { h_a[i] = (float)i; h_b[i] = (float)(i * 2); }
float *d_a, *d_b, *d_c;
hipMallocManaged(&d_a, N * sizeof(float));
hipMallocManaged(&d_b, N * sizeof(float));
hipMallocManaged(&d_c, N * sizeof(float));
hipMemcpy(d_a, h_a, N * sizeof(float), hipMemcpyHostToDevice);
hipMemcpy(d_b, h_b, N * sizeof(float), hipMemcpyHostToDevice);
add_kernel<<<1, N>>>(d_a, d_b, d_c, N);
hipDeviceSynchronize();
hipError_t err = hipGetLastError();
if (err != hipSuccess) {
printf("KERNEL FAIL: %s\n", hipGetErrorString(err));
hipFree(d_a); hipFree(d_b); hipFree(d_c);
return 1;
}
hipMemcpy(h_c, d_c, N * sizeof(float), hipMemcpyDeviceToHost);
int ok = 1;
for (int i = 0; i < N; i++) {
if (h_c[i] != h_a[i] + h_b[i]) { ok = 0; printf("Mismatch at %d: %f vs %f\n", i, h_c[i], h_a[i]+h_b[i]); break; }
}
printf("Compute: %s\n", ok ? "PASS" : "FAIL");
hipFree(d_a); hipFree(d_b); hipFree(d_c);
return ok ? 0 : 1;
}
HIPEOF
for arch in gfx1013 gfx1010 gfx10-1-generic; do
echo "=== Runtime test: $arch ==="
/opt/rocm/bin/hipcc --offload-arch=$arch /tmp/run_test2.hip -o /tmp/run_test2_${arch} 2>&1
if [ $? -eq 0 ]; then
echo "Compiled OK. Running..."
timeout 15 /tmp/run_test2_${arch} 2>&1
echo "Exit: $?"
else
echo "Compile FAILED"
fi
echo ""
done
Claude Test Scripts/test_tv.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
source /home/fabian/ComfyUI/venv/bin/activate
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export HIP_VISIBLE_DEVICES=0
python3.11 -c "
import torchvision
print('torchvision version:', torchvision.__version__)
from torchvision import transforms
print('transforms OK')
print('ALL GOOD')
"
ComfyUI Scripts/ROCm_BC250_Documentation.md
+2135
View File
File diff suppressed because it is too large Load Diff
ComfyUI Scripts/bc250_bench.py
+123
View File
@@ -0,0 +1,123 @@
"""Clean old output, re-submit, get REAL GPU timing."""
import paramiko, time, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, timeout=60):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# Delete old output
print("Cleaning old output...")
sh('rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
# Truncate log to see fresh output only
sh('truncate -s 0 /tmp/comfyui.log; sleep 1')
# Submit fresh workflow
workflow = {
"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 123, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}
}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(workflow))
print("Submitting fresh workflow (seed=123)...")
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f" {resp[:120]}")
t0 = time.time()
print("\nMonitoring (NORMAL_VRAM = real GPU compute)...")
for i in range(200):
elapsed = int(time.time() - t0)
gpu_temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
sampling = ''
last = ''
for line in log.split('\n'):
s = line.strip()
if '/8' in s and ('it/s' in s or 's/it' in s):
sampling = s
if s and 'FETCH' not in s and 'startup tasks' not in s and 'DEPRECATION' not in s:
last = s
display = sampling if sampling else last[-100:]
print(f" [{elapsed:>4}s] {temp_c}C | {display}")
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
exec_time = ''
for line in log.split('\n'):
if 'Prompt executed in' in line:
exec_time = line.strip()
print(f"\n *** IMAGE GENERATED! ***")
print(f" File: {imgs}")
print(f" {exec_time}")
print(f" Wall time: {elapsed}s")
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['loaded completely', 'loaded partially', '/8', 'Prompt executed']):
print(f" {s}")
break
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 20:
time.sleep(2)
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** IMAGE: {imgs} ***")
else:
print(f"\n Queue empty, no image:")
for line in log.split('\n')[-20:]:
if line.strip(): print(f" {line.strip()}")
break
except: pass
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
if alive == 'N':
print(f"\n CRASHED!")
for line in log.split('\n')[-25:]:
if line.strip(): print(f" {line.strip()}")
break
time.sleep(10)
sftp.close()
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_check.py
+48
View File
@@ -0,0 +1,48 @@
"""Quick check: what's ACTUALLY happening in the ComfyUI log right now?"""
import paramiko
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
def sh(cmd):
chan = c.get_transport().open_session()
chan.settimeout(30)
chan.exec_command(f"/bin/bash -c '{cmd}'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
print("=== FULL LOG (minus ComfyUI-Manager spam) ===")
# Show all lines EXCEPT the registry fetch / manager spam
log = sh("grep -v 'FETCH ComfyRegistry\\|All startup tasks\\|ComfyUI-Manager' /tmp/comfyui.log | tail -50")
print(log)
print("\n=== PROCESS ===")
print(sh("ps aux | grep python3 | grep -v grep"))
print("\n=== GPU sysfs ===")
# Find the actual gpu_busy path
print(sh("find /sys/class/drm/ -name 'gpu_busy_percent' 2>/dev/null"))
print(sh("cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null; cat /sys/class/drm/card1/device/gpu_busy_percent 2>/dev/null"))
print("\n=== rocm-smi ===")
print(sh("rocm-smi 2>/dev/null | head -15"))
print("\n=== OUTPUT DIR ===")
print(sh("ls -la ~/ComfyUI/output/ 2>/dev/null"))
print("\n=== QUEUE ===")
print(sh("curl -s http://127.0.0.1:8188/queue 2>/dev/null"))
print("\n=== Log lines with 'load' or 'sample' or 'error' or '%' ===")
print(sh("grep -iE 'load|sample|error|%|step|Traceback|OOM|killed' /tmp/comfyui.log | tail -30"))
c.close()
ComfyUI Scripts/bc250_check2.py
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""Check full log after sampling for VAE decode status."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd):
_, so, se = ssh.exec_command(cmd, timeout=15)
return so.read().decode()
# Get more log lines - look for everything after the sampling
print("=== FULL LOG (last 60 lines) ===")
print(run("tail -60 /home/fabian/comfyui.log 2>/dev/null"))
# Check output directory
print("\n=== OUTPUT FILES ===")
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null"))
# Queue status
print("=== QUEUE ===")
print(run("curl -s http://localhost:8188/queue 2>/dev/null"))
# History
print("\n=== HISTORY ===")
print(run("curl -s http://localhost:8188/history 2>/dev/null")[:2000])
# Process count and wchan
print("\n=== PROCESS STATE ===")
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
"cat /proc/$PID/wchan 2>/dev/null; echo; "
"ps -L -p $PID -o tid,%cpu,comm --sort=-%cpu 2>/dev/null | head -20'"))
ssh.close()
ComfyUI Scripts/bc250_check_build.py
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Check PyTorch build progress on BC-250."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
if out.strip():
print(out.strip())
if err.strip():
print(f"STDERR: {err.strip()}")
# Build process status
run("pgrep -fa 'setup.py|build_pytorch|cmake|ninja|hipcc|cc1plus' | head -20",
desc="Build processes")
# How long has it been running
run("ps -p 350823 -o etime=,cmd= 2>/dev/null || echo 'Process no longer running'",
desc="Build process uptime")
# Build log tail
run("tail -60 /home/fabian/pytorch_build.log 2>/dev/null || echo 'No log file'",
desc="Build log (last 60 lines)")
# Memory usage
run("free -h", desc="Memory status")
# Check for build completion marker
run("grep 'BUILD_COMPLETE' /home/fabian/pytorch_build.log 2>/dev/null || echo 'Build still in progress'",
desc="Build completion check")
# Check for any errors in log
run("grep -i 'error:\\|fatal:\\|failed' /home/fabian/pytorch_build.log 2>/dev/null | tail -10 || echo 'No errors found'",
desc="Error check")
# Disk space
run("df -h / | tail -1", desc="Disk space")
ssh.close()
ComfyUI Scripts/bc250_check_error.py
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Check the build error from PyTorch CMake on BC-250."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
if out.strip():
print(out.strip())
# Get the full cmake error
run("grep -A5 -B2 'Error\\|error\\|FATAL\\|fatal\\|Could not find' /home/fabian/pytorch_build.log | head -60",
desc="CMake errors in build log")
# Also check the cmake output file if it exists
run("cat /home/fabian/pytorch/build/CMakeFiles/CMakeOutput.log 2>/dev/null | tail -30 || echo 'no output log'",
desc="CMake output log")
run("cat /home/fabian/pytorch/build/CMakeFiles/CMakeError.log 2>/dev/null | tail -50 || echo 'no error log'",
desc="CMake error log")
# Check specifically what's missing
run("grep -i 'not found\\|could not find\\|missing' /home/fabian/pytorch_build.log | head -20",
desc="Missing packages")
# Also check if the process is still running
run("pgrep -fa 'setup.py\\|build_pytorch' || echo 'Build process not running'",
desc="Build process status")
# Check roctracer
run("find /opt/rocm -name 'roctracer*' 2>/dev/null | head -10",
desc="roctracer files")
ssh.close()
ComfyUI Scripts/bc250_check_flags.py
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Check ComfyUI flags and update startup."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=60):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
return out + err
# Kill any leftover
print(run("pkill -9 -f 'python3 main.py' 2>/dev/null; echo killed"))
# Full help output
print("=== ComfyUI --help ===")
help_text = run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'")
# Filter for interesting lines
for line in help_text.split('\n'):
low = line.lower()
if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram', 'memory', 'offload', 'precision']):
print(f" {line.strip()}")
# Also just dump the full thing to see everything
print("\n=== FULL HELP ===")
print(help_text)
ssh.close()
ComfyUI Scripts/bc250_check_state.py
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""Check PyTorch state and start fresh build on BC-250."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=120, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 60:
print(f" ... ({len(lines)} lines, showing last 60)")
print('\n'.join(lines[-60:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-20:] if len(lines) > 20 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# Check if there's a wheel already built
run("ls -lh ~/pytorch/dist/*.whl 2>/dev/null || echo 'No wheels found'",
desc="Check for existing PyTorch wheels")
# Check for any previous build directory
run("ls -la ~/pytorch/build/ 2>/dev/null | head -10 || echo 'No build dir'",
desc="Check build directory")
# Check if pytorch is already installed in venv
run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \"import torch; print(torch.__version__); print(torch.version.hip); print(torch.cuda.is_available())\" 2>&1'",
desc="Check if PyTorch is already installed")
# Check pytorch source integrity
run("ls ~/pytorch/setup.py ~/pytorch/CMakeLists.txt 2>&1",
desc="Verify PyTorch source files")
# Verify ROCm works before build
run("bash -c 'export HSA_OVERRIDE_GFX_VERSION=10.1.0 && /opt/rocm/bin/rocminfo 2>&1 | grep -E \"gfx|Marketing\" | head -5'",
desc="Verify ROCm is working")
# Check venv
run("bash -c 'source ~/comfyui-env/bin/activate && which python3 && python3 --version'",
desc="Verify venv")
ssh.close()
print("\nDone checking state.")
ComfyUI Scripts/bc250_check_status.py
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Quick check on ComfyUI status."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
return stdout.read().decode()
# Log tail
print("=== LOG (last 40 lines) ===")
print(run("tail -40 /home/fabian/comfyui.log 2>/dev/null"))
# Process status
print("=== PROCESS ===")
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" ps -p $PID -o pid,%cpu,%mem,nlwp,stat --no-headers; "
" echo \"LOAD: $(cat /proc/loadavg)\"; "
"else echo DEAD; fi'"))
# rocm-smi
print("=== GPU ===")
print(run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null || echo 'no rocm-smi'"))
# Queue
print("=== QUEUE ===")
print(run("curl -s http://localhost:8188/queue 2>/dev/null || echo 'no connection'"))
# History
print("=== HISTORY ===")
hist = run("curl -s http://localhost:8188/history 2>/dev/null || echo 'no connection'")
import json
try:
h = json.loads(hist)
for pid, info in h.items():
print(f" Prompt: {pid}")
print(f" Status: {info.get('status', {})}")
outputs = info.get('outputs', {})
if outputs:
for nid, nout in outputs.items():
if isinstance(nout, dict):
for key, val in nout.items():
print(f" Output node {nid}/{key}: {str(val)[:200]}")
else:
print(" No outputs")
except:
print(hist[:1000])
# Output directory
print("\n=== OUTPUT FILES ===")
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null"))
ssh.close()
ComfyUI Scripts/bc250_check_threads.py
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Read full ops.py and check torch thread defaults, then fix threading."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
print(out.strip())
if err.strip():
lines = err.strip().split('\n')[-10:]
print(f"STDERR: {chr(10).join(lines)}")
print(f" Exit: {rc}")
return rc, out, err
# Check current torch thread defaults
run("bash -c 'source ~/comfyui-env/bin/activate && "
"export HSA_OVERRIDE_GFX_VERSION=10.1.0 && "
"python -c \""
"import torch; "
"print(f\\\"num_threads={torch.get_num_threads()}\\\"); "
"print(f\\\"num_interop_threads={torch.get_num_interop_threads()}\\\"); "
"import os; "
"print(f\\\"OMP_NUM_THREADS={os.environ.get(\\\\\\\"OMP_NUM_THREADS\\\\\\\", \\\\\\\"not set\\\\\\\")}\\\"); "
"print(f\\\"MKL_NUM_THREADS={os.environ.get(\\\\\\\"MKL_NUM_THREADS\\\\\\\", \\\\\\\"not set\\\\\\\")}\\\"); "
"\"'",
desc="Check default torch thread settings")
# Read forward_ggml_cast_weights (where dequant happens during inference)
run("bash -c 'sed -n \"200,281p\" ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py'",
desc="ops.py lines 200-281 (forward functions)")
# Check __init__.py for any loading/patching
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/__init__.py 2>/dev/null | head -40'",
desc="ComfyUI-GGUF __init__.py")
# Check nodes.py for model loading
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/nodes.py 2>/dev/null'",
desc="ComfyUI-GGUF nodes.py (model loader)")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_cpu_mode.py
+271
View File
@@ -0,0 +1,271 @@
#!/usr/bin/env python3
"""
Switch ComfyUI to --cpu mode so ALL 12 cores are used.
The GPU (Cyan Skillfish gfx1013) hangs during HIP inference ops,
causing the single-core stall. CPU mode with MKL+OpenMP will use all cores.
"""
import paramiko
import json
import time
import textwrap
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=120, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 30:
print(f" ({len(lines)} lines, showing last 30)")
print('\n'.join(lines[-30:]))
else:
print(out.strip())
if err.strip():
for line in err.strip().split('\n')[-5:]:
print(f" STDERR: {line}")
return rc, out, err
# ── Step 1: Kill stuck ComfyUI ──
run("bash -c 'pkill -f \"python3 main.py\" 2>/dev/null; sleep 2; "
"pkill -9 -f \"python3 main.py\" 2>/dev/null; sleep 1; "
"echo \"Killed. Remaining:\"; pgrep -af \"main.py\" || echo none'",
desc="Kill stuck ComfyUI")
# ── Step 2: Write new startup script with --cpu ──
# Key: OMP_NUM_THREADS=12 + MKL_NUM_THREADS=12 + --cpu
# This uses Intel MKL (built into this PyTorch) for matrix ops across all cores
startup_script = textwrap.dedent("""\
#!/bin/bash
# ComfyUI CPU-mode launcher for BC-250
# Forces ALL computation on CPU using 12 cores via MKL + OpenMP
# Threading: use ALL 12 cores
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
export OMP_PROC_BIND=spread
export OMP_PLACES=cores
export GOMP_CPU_AFFINITY="0-11"
# No GPU needed in CPU mode, but keep env for potential future use
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# MKL tuning for multi-core
export MKL_DYNAMIC=FALSE
export MKL_ENABLE_INSTRUCTIONS=AVX2
# Activate venv
source /home/fabian/comfyui-env/bin/activate
cd /home/fabian/ComfyUI
echo "=== BC-250 ComfyUI CPU Mode ==="
echo "Cores: 12, OMP_NUM_THREADS=$OMP_NUM_THREADS, MKL_NUM_THREADS=$MKL_NUM_THREADS"
echo "OMP_PROC_BIND=$OMP_PROC_BIND, OMP_PLACES=$OMP_PLACES"
# --cpu: force ALL ops on CPU (no GPU)
# --disable-auto-launch: don't open browser
exec python3 main.py --listen 0.0.0.0 --port 8188 --cpu --disable-auto-launch
""")
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
f.write(startup_script)
sftp.close()
run("chmod +x /home/fabian/start_comfyui.sh")
print("\n Updated start_comfyui.sh -> --cpu mode, 12 cores, MKL tuning")
# ── Step 3: Update sitecustomize.py to also set MKL_DYNAMIC=FALSE ──
sitecustomize = textwrap.dedent("""\
import os
os.environ.setdefault('OMP_NUM_THREADS', '12')
os.environ.setdefault('MKL_NUM_THREADS', '12')
os.environ.setdefault('MKL_DYNAMIC', 'FALSE')
os.environ.setdefault('OMP_PROC_BIND', 'spread')
os.environ.setdefault('OMP_PLACES', 'cores')
try:
import torch
torch.set_num_threads(12)
torch.set_num_interop_threads(12)
print(f"Threads: intra={torch.get_num_threads()} inter={torch.get_num_interop_threads()}")
except Exception:
pass
""")
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/comfyui-env/lib/python3.14/site-packages/sitecustomize.py', 'w') as f:
f.write(sitecustomize)
sftp.close()
print(" Updated sitecustomize.py with MKL_DYNAMIC=FALSE")
# ── Step 4: Launch ComfyUI in CPU mode ──
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'",
desc="Launch ComfyUI in CPU mode")
time.sleep(8)
rc, out, _ = run("bash -c 'tail -30 /home/fabian/comfyui.log 2>/dev/null'",
desc="Startup log")
# Verify server started
for attempt in range(10):
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null'")
if '200' in out:
print(f"\n Server is UP on port 8188 (attempt {attempt+1})")
break
time.sleep(5)
else:
print("\n WARNING: Server didn't respond after 50s")
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Full log")
ssh.close()
exit(1)
# ── Step 5: Submit workflow with slightly smaller image for faster CPU gen ──
# 768x432 instead of 1024x576 to speed up first test
workflow = {
"prompt": {
"1": {
"class_type": "UnetLoaderGGUF",
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
},
"2": {
"class_type": "CLIPLoaderGGUF",
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
},
"3": {
"class_type": "VAELoader",
"inputs": {"vae_name": "ae.safetensors"}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "A majestic mountain landscape at sunset, golden light on snow peaks, crystal lake reflection, photorealistic, 8k",
"clip": ["2", 0]
}
},
"5": {
"class_type": "CLIPTextEncode",
"inputs": {"text": "", "clip": ["2", 0]}
},
"6": {
"class_type": "EmptyLatentImage",
"inputs": {"width": 768, "height": 432, "batch_size": 1}
},
"7": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0],
"seed": 42,
"steps": 8,
"cfg": 1.0,
"sampler_name": "euler",
"scheduler": "simple",
"positive": ["4", 0],
"negative": ["5", 0],
"latent_image": ["6", 0],
"denoise": 1.0
}
},
"8": {
"class_type": "VAEDecode",
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
},
"9": {
"class_type": "SaveImage",
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_BC250"}
}
}
}
sftp = ssh.open_sftp()
with sftp.open('/tmp/zimage_workflow.json', 'w') as f:
f.write(json.dumps(workflow))
sftp.close()
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
"-H \"Content-Type: application/json\" "
"-d @/tmp/zimage_workflow.json'",
desc="Submit 768x432 workflow")
try:
resp = json.loads(out.strip())
if 'error' in resp:
print(f"\n ERROR: {resp['error']}")
if 'node_errors' in resp:
for nid, e in resp['node_errors'].items():
print(f" Node {nid}: {e}")
ssh.close()
exit(1)
prompt_id = resp.get('prompt_id', 'unknown')
print(f"\n Prompt ID: {prompt_id}")
except Exception as e:
print(f" Parse error: {e}\n Raw: {out[:500]}")
# ── Step 6: Monitor CPU/progress ──
print("\n Monitoring generation (CPU mode, 12 cores)...")
print(" This is a 6B model on CPU — expect several minutes per step")
start_time = time.time()
last_log = ""
for i in range(240): # up to 60 min
time.sleep(15)
elapsed = time.time() - start_time
minutes = int(elapsed // 60)
seconds = int(elapsed % 60)
# CPU usage - check if ALL cores are active
rc, cpu_out, _ = run("bash -c '"
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" echo \"CPU_PCT=$(ps -p $PID -o %cpu= 2>/dev/null)\"; "
" echo \"MEM_PCT=$(ps -p $PID -o %mem= 2>/dev/null)\"; "
" echo \"THREADS=$(ps -p $PID -o nlwp= 2>/dev/null)\"; "
" echo \"LOADAVG=$(cat /proc/loadavg)\"; "
"else echo PROCESS_DEAD; fi'")
# Parse CPU metrics
cpu_pct = "?"
load_avg = "?"
for line in (cpu_out or '').split('\n'):
if line.startswith('CPU_PCT='):
cpu_pct = line.split('=')[1].strip()
if line.startswith('LOADAVG='):
load_avg = line.split('=')[1].strip().split()[0]
# Log tail
rc, log_out, _ = run("bash -c 'tail -3 /home/fabian/comfyui.log 2>/dev/null'")
log_tail = (log_out or '').strip().split('\n')[-1] if log_out else ""
if 'PROCESS_DEAD' in (cpu_out or ''):
print(f"\n [{minutes}m{seconds}s] PROCESS DIED!")
run("bash -c 'tail -50 /home/fabian/comfyui.log'", desc="Death log")
break
# Show progress
print(f" [{minutes}m{seconds}s] CPU={cpu_pct}% Load={load_avg} | {log_tail[:80]}")
if 'Prompt executed in' in (log_out or ''):
print(f"\n IMAGE GENERATED! Total time: {minutes}m{seconds}s")
run("bash -c 'tail -20 /home/fabian/comfyui.log'", desc="Completion log")
run("bash -c 'ls -lah ~/ComfyUI/output/'", desc="Output files")
break
if 'Error' in log_tail or 'Traceback' in (log_out or ''):
print(f"\n ERROR DETECTED!")
run("bash -c 'tail -60 /home/fabian/comfyui.log'", desc="Error log")
break
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_diag.py
+31
View File
@@ -0,0 +1,31 @@
import paramiko, time
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
# 1) Full diagnostic
cmds = {
"LOG_LAST_40": "tail -40 /tmp/comfyui.log 2>/dev/null",
"PROCESS": "ps aux | grep -E 'python|comfy' | grep -v grep",
"GPU": "cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null",
"GPU_TEMP": "cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null",
"CPU_CORES": "mpstat -P ALL 1 1 2>/dev/null | tail -15 || top -bn1 | head -5",
"MEM": "free -m",
"OUTPUT": "ls -la ~/ComfyUI/output/ 2>/dev/null",
"QUEUE": "curl -s http://127.0.0.1:8188/queue 2>/dev/null",
"ROCM_CHECK": "rocm-smi --showuse --showtemp --showpower 2>/dev/null | head -20",
}
for name, cmd in cmds.items():
print(f"\n=== {name} ===")
_, o, e = c.exec_command(cmd)
out = o.read().decode(errors='replace').strip()
err = e.read().decode(errors='replace').strip()
print(out if out else "(empty)")
if err and name not in ("ROCM_CHECK",):
print(f" STDERR: {err}")
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_diag2.py
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Diagnostic: check full log and GPU state."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
return out, err
# Full log (last 80 lines)
out, _ = run("tail -80 /home/fabian/comfyui.log 2>/dev/null")
print("=== FULL LOG (last 80 lines) ===")
print(out)
# Process state
out, _ = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" echo \"PID: $PID\"; "
" echo \"=== PROCESS STATE ===\"; "
" cat /proc/$PID/status | grep -E \"State|Threads|VmRSS|VmSize\"; "
" echo \"=== WCHAN (what syscall is process blocked on) ===\"; "
" cat /proc/$PID/wchan 2>/dev/null; echo; "
" echo \"=== STACK TRACE (kernel) ===\"; "
" sudo cat /proc/$PID/stack 2>/dev/null || echo \"no permission\"; "
" echo \"=== TOP THREADS ===\"; "
" ps -L -p $PID -o tid,%cpu,comm --sort=-%cpu | head -15; "
"fi'")
print(out)
# GPU info
out, _ = run("bash -c 'rocm-smi 2>/dev/null || echo no rocm-smi; "
"echo \"=== dmesg GPU ===\"; "
"dmesg 2>/dev/null | grep -i -E \"amdgpu|error|fault\" | tail -15 || echo no-dmesg'")
print("=== GPU ===")
print(out)
# Memory
out, _ = run("free -h")
print("=== MEMORY ===")
print(out)
ssh.close()
ComfyUI Scripts/bc250_diag3.py
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env python3
"""Check ComfyUI API queue/history for errors."""
import paramiko, json
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
return stdout.read().decode(), stderr.read().decode()
# Queue status
out, _ = run("curl -s http://localhost:8188/queue")
print("=== QUEUE ===")
try:
q = json.loads(out)
print(f"Running: {len(q.get('queue_running', []))}")
print(f"Pending: {len(q.get('queue_pending', []))}")
except:
print(out[:500])
# History
out, _ = run("curl -s http://localhost:8188/history")
print("\n=== HISTORY ===")
try:
h = json.loads(out)
for pid, info in h.items():
print(f"\nPrompt ID: {pid}")
status = info.get('status', {})
print(f" Status: {status}")
outputs = info.get('outputs', {})
for nid, nout in outputs.items():
print(f" Node {nid}: {list(nout.keys()) if isinstance(nout, dict) else nout}")
if not outputs:
print(" NO OUTPUTS")
except:
print(out[:2000])
# Check stderr output (nohup might redirect differently)
out, _ = run("cat /home/fabian/comfyui_err.log 2>/dev/null || echo 'no err log'")
print(f"\n=== STDERR LOG ===\n{out[:2000]}")
# Check full nohup output
out, _ = run("wc -l /home/fabian/comfyui.log 2>/dev/null")
print(f"\n=== LOG LINES: {out.strip()}")
# Check if there are processes actively computing
out, _ = run("bash -c 'top -bn1 | head -20'")
print(f"\n=== TOP ===\n{out}")
ssh.close()
ComfyUI Scripts/bc250_diag_cores.py
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Deep diagnosis: WHY only 1 core? Check OpenMP, threading, GGUF code path."""
import paramiko, json, textwrap
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=60):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
return rc, out, err
def show(label, cmd, timeout=60):
print(f"\n{'='*60}")
print(f" {label}")
print(f"{'='*60}")
rc, out, err = run(cmd, timeout)
if out.strip():
print(out.strip())
if err.strip():
for line in err.strip().split('\n')[-10:]:
print(f"STDERR: {line}")
return out
# 1. Check ComfyUI queue/history - did generation succeed or fail?
show("Queue status", "curl -s http://localhost:8188/queue")
hist_out = show("History", "curl -s http://localhost:8188/history")
try:
h = json.loads(hist_out.strip())
for pid, info in h.items():
status = info.get('status', {})
print(f"\n Prompt {pid}: status={status}")
outputs = info.get('outputs', {})
if outputs:
print(f" Outputs: {json.dumps(outputs, indent=2)[:500]}")
else:
print(" NO OUTPUTS")
except:
pass
# 2. Check if PyTorch has OpenMP
show("PyTorch OpenMP & threading",
"bash -c 'source /home/fabian/comfyui-env/bin/activate && "
"OMP_NUM_THREADS=12 python3 -c \""
"import torch; "
"print(f\\\"OpenMP available: {torch.backends.openmp.is_available()}\\\"); "
"print(f\\\"MKL available: {torch.backends.mkl.is_available()}\\\"); "
"print(f\\\"Num threads: {torch.get_num_threads()}\\\"); "
"print(f\\\"Num interop threads: {torch.get_num_interop_threads()}\\\"); "
"print(f\\\"torch.__config__.show(): \\\"); "
"print(torch.__config__.show()); "
"\"'")
# 3. Check if libomp/libgomp is available
show("OpenMP libraries",
"bash -c 'ldconfig -p 2>/dev/null | grep -i omp; "
"echo ---; "
"pacman -Qs openmp 2>/dev/null; "
"echo ---; "
"pacman -Qs libgomp 2>/dev/null; "
"echo ---; "
"ls -la /usr/lib/libomp* /usr/lib/libgomp* 2>/dev/null || echo none'")
# 4. Check pytorch shared lib dependencies for OpenMP
show("PyTorch .so OpenMP deps",
"bash -c 'ldd /usr/lib/python3.14/site-packages/torch/lib/libtorch_cpu.so 2>/dev/null | grep -i omp'")
# 5. Actual thread test - does a matrix multiply use multiple cores?
show("Matrix multiply CPU benchmark (should use all cores)",
"bash -c 'source /home/fabian/comfyui-env/bin/activate && "
"OMP_NUM_THREADS=12 python3 -c \""
"import torch, time, os; "
"print(f\\\"PID: {os.getpid()}\\\"); "
"torch.set_num_threads(12); "
"print(f\\\"Threads set to: {torch.get_num_threads()}\\\"); "
"a = torch.randn(4096, 4096); "
"b = torch.randn(4096, 4096); "
"# warmup; "
"c = torch.mm(a, b); "
"import subprocess; "
"# Start monitoring in background; "
"start = time.time(); "
"for i in range(5): c = torch.mm(a, b); "
"elapsed = time.time() - start; "
"print(f\\\"5x matmul 4096x4096: {elapsed:.2f}s\\\"); "
"\"'")
# 6. Check what the GGUF dequant code actually does (single-threaded python loop?)
show("GGUF dequant code - is there a Python for-loop?",
"bash -c 'grep -n \"for \" /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py | head -20; "
"echo \"---\"; "
"grep -n \"for \" /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py | head -20; "
"echo \"---\"; "
"grep -n \"def dequantize\" /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py'")
# 7. Check if lowvram is causing sequential layer-by-layer processing
show("ComfyUI lowvram model loading code",
"bash -c 'grep -rn \"lowvram\\|low_vram\\|offload\" /home/fabian/ComfyUI/comfy/model_management.py 2>/dev/null | head -30'")
# 8. Output directory
show("Output files", "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null")
ssh.close()
print("\n\nDONE.")
ComfyUI Scripts/bc250_diag_vae.py
+82
View File
@@ -0,0 +1,82 @@
"""Diagnose and fix VAE hang. Check log, fix threading, restart."""
import paramiko, time, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, timeout=30):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# 1. What's running?
print("=== CURRENT STATE ===")
ps = sh('ps aux | grep main.py | grep -v grep')
print(f"Process: {ps or 'NONE'}")
# Check env of running process
env = sh(r'cat /proc/$(pgrep -f "python3.*main.py" | head -1)/environ 2>/dev/null | tr "\0" "\n" | grep -E "OMP|MKL|COMFYUI|THREAD|OPENBLAS"')
print(f"Env:\n{env}")
# 2. Log tail
print("\n=== LOG TAIL ===")
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n')[-40:]:
s = line.strip()
if s: print(f" {s}")
except Exception as e:
log = ''
print(f" No log: {e}")
# 3. Check launcher
print("\n=== LAUNCHER ===")
try:
with sftp.open('/tmp/run_comfyui.sh', 'r') as f:
print(f.read().decode())
except: print(" No launcher")
# 4. model_management.py patch?
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
mm = f.read().decode()
print(f"SHARED patch: {'YES' if 'COMFYUI_SHARED_MEMORY' in mm else 'NO'}")
# 5. Check torch threads in same env
print("\n=== TORCH THREADS ===")
tcheck = sh('''source ~/comfyui-env/bin/activate
export HSA_OVERRIDE_GFX_VERSION=10.1.0
python3 -c "
import torch, os
print(f'torch.get_num_threads() = {torch.get_num_threads()}')
print(f'OMP_NUM_THREADS = {os.environ.get(chr(34)+'OMP_NUM_THREADS'+chr(34), chr(34)+'NOT SET'+chr(34))}')
"''', timeout=30)
print(tcheck)
# 6. Check sitecustomize
print("\n=== SITECUSTOMIZE ===")
sc = sh('cat ~/comfyui-env/lib/python*/site-packages/sitecustomize.py 2>/dev/null || echo MISSING')
print(sc[:500])
# 7. Key log lines
print("\n=== KEY LOG ENTRIES ===")
for line in log.split('\n'):
s = line.strip()
if any(x in s.lower() for x in ['vram state', 'shared', 'loaded', 'offloaded', 'device:', 'total vram', 'vae', 'thread']):
print(f" {s}")
sftp.close()
c.close()
print("\nDiag done.")
ComfyUI Scripts/bc250_diagnose.py
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Diagnose ComfyUI state on BC-250 — check if stuck or OOM."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
print(out.strip())
if err.strip():
lines = err.strip().split('\n')[-10:]
print(f"STDERR: {chr(10).join(lines)}")
print(f" Exit: {rc}")
return rc, out, err
# Check if ComfyUI process is alive
run("bash -c 'ps aux | grep \"python main.py\" | grep -v grep'",
desc="ComfyUI process status")
# Memory state
run("bash -c 'free -h'", desc="RAM/Swap usage")
# GPU VRAM
run("bash -c 'cat /sys/class/drm/card1/device/mem_info_vram_used 2>/dev/null; "
"echo \"---\"; cat /sys/class/drm/card1/device/mem_info_vram_total 2>/dev/null'",
desc="GPU VRAM usage")
# dmesg for OOM
run("bash -c 'dmesg | tail -20'", desc="Recent kernel messages")
# Last 80 lines of comfyui log
run("bash -c 'tail -80 /home/fabian/comfyui.log 2>/dev/null'", desc="ComfyUI log (last 80)")
# Check if port still listening
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_GONE'", desc="Port 8188")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_diagnose_gpu.py
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""
Diagnose GPU hang: kill stuck ComfyUI, run targeted HIP tests,
check what ops hang on Cyan Skillfish gfx1013->gfx1010.
Single SSH connection, properly closed.
"""
import paramiko
import json
import time
import sys
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for attempt in range(5):
try:
ssh.connect('192.168.178.150', username='fabian',
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
break
except Exception as e:
print(f" SSH attempt {attempt+1}/5: {e}")
time.sleep(10)
else:
print("FATAL: Cannot connect"); sys.exit(1)
def run(cmd, timeout=120):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
return out, err
try:
# 1. Kill stuck ComfyUI
print("=== Kill stuck ComfyUI ===")
out, _ = run("pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; echo killed")
print(f" {out.strip()}")
# 2. Check ComfyUI help for --cpu-vae flag existence
print("\n=== Check if --cpu-vae exists ===")
out, err = run("bash -c 'source ~/comfyui-env/bin/activate && cd ~/ComfyUI && python3 main.py --help 2>&1'")
full_help = out + err
has_cpu_vae = '--cpu-vae' in full_help
print(f" --cpu-vae flag exists: {has_cpu_vae}")
# Print all vram/gpu related flags
for line in full_help.split('\n'):
if any(w in line.lower() for w in ['vram', 'cpu', 'gpu', 'fp16', 'fp32', 'vae', 'force', 'precision']):
print(f" {line.strip()}")
# 3. Check what the CachyOS pytorch-rocm was built for
print("\n=== PyTorch ROCm build info ===")
out, _ = run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \""
"import torch; "
"print(f\\\"PyTorch version: {torch.__version__}\\\"); "
"print(f\\\"CUDA/HIP available: {torch.cuda.is_available()}\\\"); "
"print(f\\\"ROCm version: {torch.version.hip}\\\"); "
"print(f\\\"Device: {torch.cuda.get_device_name(0)}\\\"); "
"print(f\\\"Arch: {torch.cuda.get_device_capability(0)}\\\"); "
"print(f\\\"VRAM free/total: {torch.cuda.mem_get_info()[0]//1048576}/{torch.cuda.mem_get_info()[1]//1048576} MB\\\"); "
"\"' 2>&1")
print(out.strip())
# 4. Targeted GPU op tests - find what hangs
print("\n=== GPU operation tests (timeout 30s each) ===")
tests = [
("Basic matmul fp32",
"a=torch.randn(256,256,device='cuda'); b=a@a; print(f'fp32 matmul: {b.shape} sum={b.sum().item():.1f}')"),
("Basic matmul fp16",
"a=torch.randn(256,256,device='cuda').half(); b=a@a; print(f'fp16 matmul: {b.shape} sum={b.sum().item():.1f}')"),
("Conv2d fp16 (VAE-like)",
"import torch.nn as nn; c=nn.Conv2d(128,128,3,padding=1).half().cuda(); x=torch.randn(1,128,64,64,device='cuda').half(); y=c(x); print(f'conv2d fp16: {y.shape}')"),
("Conv2d fp32 (VAE default)",
"import torch.nn as nn; c=nn.Conv2d(128,128,3,padding=1).cuda(); x=torch.randn(1,128,64,64,device='cuda'); y=c(x); print(f'conv2d fp32: {y.shape}')"),
("GroupNorm fp16",
"import torch.nn as nn; gn=nn.GroupNorm(32,128).half().cuda(); x=torch.randn(1,128,32,32,device='cuda').half(); y=gn(x); print(f'groupnorm fp16: {y.shape}')"),
("GroupNorm fp32",
"import torch.nn as nn; gn=nn.GroupNorm(32,128).cuda(); x=torch.randn(1,128,32,32,device='cuda'); y=gn(x); print(f'groupnorm fp32: {y.shape}')"),
("LayerNorm fp16",
"import torch.nn as nn; ln=nn.LayerNorm(256).half().cuda(); x=torch.randn(1,64,256,device='cuda').half(); y=ln(x); print(f'layernorm fp16: {y.shape}')"),
("Linear fp16 (DiT-like)",
"import torch.nn as nn; l=nn.Linear(1024,1024).half().cuda(); x=torch.randn(1,64,1024,device='cuda').half(); y=l(x); print(f'linear fp16: {y.shape}')"),
("Attention fp16 (scaled_dot_product)",
"q=torch.randn(1,8,64,64,device='cuda').half(); k=q.clone(); v=q.clone(); "
"y=torch.nn.functional.scaled_dot_product_attention(q,k,v); print(f'sdpa fp16: {y.shape}')"),
("Attention fp32 (scaled_dot_product)",
"q=torch.randn(1,8,64,64,device='cuda'); k=q.clone(); v=q.clone(); "
"y=torch.nn.functional.scaled_dot_product_attention(q,k,v); print(f'sdpa fp32: {y.shape}')"),
("Large matmul fp16 (5032x5032)",
"a=torch.randn(2048,2048,device='cuda').half(); b=a@a; print(f'large fp16 matmul: {b.shape} sum={b.sum().item():.1f}')"),
("RoPE-like op (complex multiply)",
"x=torch.randn(1,8,64,64,device='cuda').half(); "
"f=torch.randn(64,32,2,device='cuda').half(); "
"print(f'rope input shapes: x={x.shape} f={f.shape} OK')"),
("torch.compile basic test",
"import torch._dynamo; f=lambda x: x*2+1; cf=torch.compile(f); "
"x=torch.randn(100,device='cuda'); y=cf(x); print(f'compile: {y.shape}')"),
]
for name, code in tests:
print(f"\n Testing: {name}...", end=" ", flush=True)
cmd = (f"bash -c 'timeout 30 bash -c \""
f"source ~/comfyui-env/bin/activate && "
f"HSA_OVERRIDE_GFX_VERSION=10.1.0 HSA_ENABLE_SDMA=0 "
f"python3 -c \\\"import torch; {code}\\\"\" 2>&1 || echo TIMEOUT_OR_ERROR'")
out, err = run(cmd, timeout=40)
result = (out + err).strip()
if 'TIMEOUT_OR_ERROR' in result:
# Get just the error part
lines = result.split('\n')
for l in reversed(lines):
if l.strip() and l.strip() != 'TIMEOUT_OR_ERROR':
print(f"FAILED: {l.strip()[:100]}")
break
else:
print("TIMEOUT (GPU HANG)")
elif result:
last_line = [l for l in result.split('\n') if l.strip()][-1] if result.split('\n') else result
print(f"OK: {last_line.strip()[:100]}")
else:
print("NO OUTPUT (possible hang)")
# 5. Check dmesg for GPU errors after tests
print("\n\n=== dmesg GPU errors (last 20) ===")
out, _ = run("dmesg 2>/dev/null | grep -i -E 'amdgpu|gpu|gfx|error|fault' | tail -20 || echo 'no permission'")
print(out.strip() if out.strip() else " (empty or no permission)")
# 6. Check rocm-smi for GPU health
print("\n=== GPU health after tests ===")
out, _ = run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null")
for line in out.split('\n'):
if any(c in line for c in ['°C', '%', 'Device', 'Node']):
print(f" {line.strip()}")
finally:
ssh.close()
print("\n\nSSH connection closed.")
ComfyUI Scripts/bc250_fix5.py
+165
View File
@@ -0,0 +1,165 @@
import paramiko, time, json, textwrap
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
sftp = c.open_sftp()
def sh(cmd, timeout=60):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# 1) Kill
print("1) Kill")
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
# 2) New launcher: --novram streams weights (proven GPU 136W), no --cpu-vae
print("2) Write launcher")
launcher = textwrap.dedent("""\
#!/bin/bash
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
export MIOPEN_FIND_MODE=3
cd ~/ComfyUI
source ~/comfyui-env/bin/activate
# --novram: weights in system RAM, GPU computes via streaming (337MB buffer fits in 512MB real VRAM)
# --force-fp16: half precision
# NO --cpu-vae: let VAE run on GPU (320MB fits in 512MB VRAM)
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--novram \\
--force-fp16
""")
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
f.write(launcher)
sh('chmod +x /tmp/run_comfyui.sh')
# 3) Start
print("3) Start")
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(3)
pid = sh('pgrep -f "python3.*main.py"')
print(f" PID: {pid}")
# 4) Wait ready
print("4) Wait HTTP", end='', flush=True)
for i in range(90):
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
if '200' in code:
print(f" OK ({i*2}s)")
break
print('.', end='', flush=True)
time.sleep(2)
# Show mode
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['vram state', 'Device:', 'Total VRAM', 'offloading']):
print(f" {s}")
# 5) Submit
print("5) Submit")
wf = {"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 777, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(wf))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f" {resp[:120]}")
# 6) Monitor
print("6) Monitor")
t0 = time.time()
for i in range(200):
el = int(time.time() - t0)
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
tc = int(temp)//1000 if temp.isdigit() else '?'
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
samp = ''
last = ''
for line in log.split('\n'):
s = line.strip()
if '/8' in s and ('it/s' in s or 's/it' in s): samp = s
if s and 'FETCH' not in s and 'startup' not in s and 'DEPRECATION' not in s: last = s
print(f" [{el:>4}s] {tc}C | {(samp or last)[-90:]}")
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
et = ''
for line in log.split('\n'):
if 'Prompt executed' in line: et = line.strip()
print(f"\n *** DONE! *** {imgs}")
print(f" {et}")
print(f" Wall: {el}s")
# Show key log lines
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['loaded', '/8', 'Prompt executed', 'VAE', 'Requested']):
if 'FETCH' not in s:
print(f" {s}")
break
# Queue empty?
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending') and el > 30:
time.sleep(2)
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** DONE: {imgs} ***")
else:
print(f"\n Queue empty, no image:")
for line in log.split('\n')[-20:]:
if line.strip() and 'FETCH' not in line: print(f" {line.strip()}")
break
except: pass
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
print("\n CRASHED!")
for line in log.split('\n')[-25:]:
if line.strip(): print(f" {line.strip()}")
break
time.sleep(10)
sftp.close()
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_now.py
+168
View File
@@ -0,0 +1,168 @@
"""FAST FIX: patch offload devices, restart. No fluff."""
import paramiko, time, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, t=30):
ch = c.get_transport().open_session()
ch.settimeout(t)
ch.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
o = b""
while True:
try:
d = ch.recv(65536)
if not d: break
o += d
except: break
ch.close()
return o.decode(errors='replace').strip()
# 1. KILL
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 1')
print("Killed")
# 2. READ
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
code = f.read().decode()
# 3. PATCH: unet_offload_device - return GPU for SHARED too
# Find the function and add SHARED check
changed = False
# Patch unet_offload_device: "HIGH_VRAM" -> "HIGH_VRAM or SHARED"
if 'def unet_offload_device' in code:
lines = code.split('\n')
for i, line in enumerate(lines):
if 'def unet_offload_device' in line:
# Look at next few lines for the HIGH_VRAM check
for j in range(i, min(i+8, len(lines))):
if 'HIGH_VRAM' in lines[j] and 'SHARED' not in lines[j] and 'unet_offload' not in lines[j]:
old = lines[j]
lines[j] = old.replace('VRAMState.HIGH_VRAM', 'VRAMState.HIGH_VRAM or vram_state == VRAMState.SHARED')
print(f"Patched unet_offload L{j+1}: {lines[j].strip()}")
changed = True
break
break
code = '\n'.join(lines)
# Patch vae_offload_device: "args.gpu_only" -> "args.gpu_only or SHARED"
if 'def vae_offload_device' in code:
lines = code.split('\n')
for i, line in enumerate(lines):
if 'def vae_offload_device' in line:
for j in range(i, min(i+8, len(lines))):
if 'gpu_only' in lines[j] and 'SHARED' not in lines[j]:
old = lines[j]
lines[j] = old.replace('args.gpu_only', '(args.gpu_only or vram_state == VRAMState.SHARED)')
print(f"Patched vae_offload L{j+1}: {lines[j].strip()}")
changed = True
break
break
code = '\n'.join(lines)
# Also patch text_encoder_offload_device if it offloads to CPU
if 'def text_encoder_offload_device' in code:
lines = code.split('\n')
for i, line in enumerate(lines):
if 'def text_encoder_offload_device' in line:
for j in range(i, min(i+8, len(lines))):
if 'gpu_only' in lines[j] and 'SHARED' not in lines[j]:
old = lines[j]
lines[j] = old.replace('args.gpu_only', '(args.gpu_only or vram_state == VRAMState.SHARED)')
print(f"Patched text_enc_offload L{j+1}: {lines[j].strip()}")
changed = True
break
break
code = '\n'.join(lines)
if changed:
sh('cp /home/fabian/ComfyUI/comfy/model_management.py /home/fabian/ComfyUI/comfy/model_management.py.bak3')
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
f.write(code)
print("Written!")
else:
print("Already patched or structure changed")
# 4. RESTART
sh('rm -f /tmp/comfyui.log')
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(4)
print(f"PID: {sh('pgrep -f python3.*main.py')}")
# 5. WAIT FOR READY
for i in range(60):
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', t=5)
if '200' in r: print(f"Ready ({i*2}s)"); break
time.sleep(2)
# Quick check
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for l in log.split('\n'):
s = l.strip()
if any(x in s for x in ['vram state', 'SHARED', 'Device:']): print(f" {s}")
# 6. SUBMIT
wf = {"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(wf))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f"Submitted: {resp[:100]}")
# 7. MONITOR - compact, fast checks
print("\nWaiting for image...")
t0 = time.time()
last_shown = ''
for i in range(180):
el = int(time.time() - t0)
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
# Find latest status
status = ''
for l in log.split('\n'):
s = l.strip()
if any(x in s for x in ['/8', 'loaded', 'Requested', 'VAE', 'Prompt executed', 'Error']):
if 'FETCH' not in s: status = s
if status != last_shown:
gpu = sh('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null', t=5)
print(f" [{el:>3}s] GPU:{gpu}% | {status[-100:]}")
last_shown = status
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', t=5)
if imgs:
print(f"\n*** DONE in {el}s! ***")
for l in log.split('\n'):
s = l.strip()
if any(x in s for x in ['load device', 'offload device', 'loaded completely', 'Prompt executed']):
print(f" {s}")
break
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', t=5) == 'N':
print(f"\nCRASHED at {el}s!")
for l in log.split('\n')[-15:]:
if l.strip(): print(f" {l.strip()}")
break
time.sleep(3)
sftp.close()
c.close()
ComfyUI Scripts/bc250_fix_threads.py
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""Fix single-core bottleneck: set threading env vars and patch ComfyUI-GGUF for parallel dequant."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=120, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd[:300]}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 50:
print(f" ... ({len(lines)} lines, showing last 50)")
print('\n'.join(lines[-50:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-15:] if len(lines) > 15 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# 1. Kill stuck ComfyUI
run("bash -c 'kill -9 484588 2>/dev/null; pkill -9 -f \"python main.py\" 2>/dev/null; sleep 2; echo done'",
desc="Kill stuck ComfyUI process")
# 2. Check how many CPU cores
run("bash -c 'nproc'", desc="CPU core count")
# 3. Check the ComfyUI-GGUF dequant code to understand the bottleneck
run("bash -c 'grep -rn \"dequant\\|num_threads\\|torch.set_num_threads\\|ThreadPool\\|parallel\" ~/ComfyUI/custom_nodes/ComfyUI-GGUF/*.py 2>/dev/null | head -30'",
desc="Search for threading in ComfyUI-GGUF")
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py 2>/dev/null | head -80'",
desc="ComfyUI-GGUF ops.py (dequant logic)")
# 4. Check the dequant function
run("bash -c 'grep -n \"def dequantize\\|class GGMLTensor\\|def forward\" ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py 2>/dev/null'",
desc="Key functions in ops.py")
run("bash -c 'wc -l ~/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py ~/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py 2>/dev/null'",
desc="File sizes")
run("bash -c 'cat ~/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py 2>/dev/null | head -60'",
desc="dequant.py start")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_threads2.py
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Fix threading: set all 12 cores for PyTorch ops, patch ComfyUI startup, restart."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=120, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd[:300]}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 40:
print(f" ... ({len(lines)} lines, showing last 40)")
print('\n'.join(lines[-40:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-15:] if len(lines) > 15 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# 1. Kill any existing ComfyUI
run("bash -c 'pkill -9 -f \"python main.py\" 2>/dev/null; sleep 1; echo killed'",
desc="Kill existing ComfyUI")
# 2. Check the loader.py for the Dequantizing message source
run("bash -c 'grep -rn \"Dequantizing\" ~/ComfyUI/ --include=\"*.py\" 2>/dev/null | head -10'",
desc="Find 'Dequantizing' message source")
# 3. Check torch thread defaults without any env vars
run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \"import torch; print(torch.get_num_threads(), torch.get_num_interop_threads())\"'",
desc="Default torch thread count")
# 4. Verify it works with env vars
run("bash -c 'export OMP_NUM_THREADS=12; export MKL_NUM_THREADS=12; "
"source ~/comfyui-env/bin/activate && python3 -c \"import torch; "
"torch.set_num_threads(12); torch.set_num_interop_threads(4); "
"print(torch.get_num_threads(), torch.get_num_interop_threads())\"'",
desc="Torch threads with env vars set to 12")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_threads3.py
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Fix: maximize threading to 12 cores, add --lowvram for 7.6GB VRAM, restart ComfyUI."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=300, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd[:300]}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 40:
print(f" ... ({len(lines)} lines, showing last 40)")
print('\n'.join(lines[-40:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-15:] if len(lines) > 15 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# 1. Kill any existing ComfyUI
run("bash -c 'pkill -9 -f \"python main.py\" 2>/dev/null; sleep 2; echo ok'",
desc="Kill existing ComfyUI")
# 2. Write updated start_comfyui.sh with full threading + lowvram
startup_script = r'''#!/bin/bash
# ComfyUI Startup Script for AsRock BC-250 (AMD Cyan Skillfish / ROCm 7.2)
# Optimized for all 12 CPU cores + 7.6GB shared VRAM
set -euo pipefail
# ═══════════════ BC-250 GPU Environment ═══════════════
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# ═══════════════ THREADING — ALL 12 CORES ═══════════════
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
export VECLIB_MAXIMUM_THREADS=12
export NUMEXPR_NUM_THREADS=12
# PyTorch intra-op (tensor math) and inter-op (parallel node execution) threads
export TORCH_NUM_THREADS=12
# ═══════════════ Memory / ROCm tuning ═══════════════
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
# Disable HIP memory caching to avoid fragmentation on shared VRAM
export PYTORCH_NO_HIP_MEMORY_CACHING=0
# ═══════════════ Activate venv ═══════════════
source "$HOME/comfyui-env/bin/activate"
cd "$HOME/ComfyUI"
# Force PyTorch to use all 12 cores
python3 -c "import torch; torch.set_num_threads(12); torch.set_num_interop_threads(12); print(f'Threads: intra={torch.get_num_threads()} inter={torch.get_num_interop_threads()}')"
echo "=========================================="
echo " ComfyUI on BC-250 (ROCm 7.2)"
echo " GPU: AMD Cyan Skillfish (gfx1013→gfx1010)"
echo " PyTorch: $(python3 -c 'import torch; print(torch.__version__)')"
echo " HIP: $(python3 -c 'import torch; print(torch.version.hip)')"
echo " CUDA: $(python3 -c 'import torch; print(torch.cuda.is_available())')"
echo " CPU: $(nproc) cores (all used)"
echo " VRAM: 7.6GB shared — using --lowvram mode"
echo "=========================================="
# Default: listen on all interfaces, --lowvram for 7.6GB shared VRAM
LISTEN_ARGS="--listen 0.0.0.0 --port 8188 --lowvram"
if [ $# -gt 0 ]; then
LISTEN_ARGS="$@"
fi
echo "Starting: python main.py $LISTEN_ARGS"
echo "Access at: http://192.168.178.150:8188"
echo ""
# Set threads inside the actual process too
exec python3 -c "
import torch, sys, os
torch.set_num_threads(12)
torch.set_num_interop_threads(12)
# Now exec ComfyUI main
sys.argv = ['main.py'] + '$LISTEN_ARGS'.split()
exec(open('main.py').read())
"
'''
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
f.write(startup_script)
sftp.close()
run("chmod +x /home/fabian/start_comfyui.sh", desc="Make script executable")
# 3. Launch ComfyUI with new settings
run("bash -c 'rm -f /home/fabian/comfyui.log'", desc="Clean old log")
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
desc="Launch ComfyUI with 12-core threading + lowvram")
# 4. Wait for startup
time.sleep(15)
run("bash -c 'tail -30 /home/fabian/comfyui.log 2>/dev/null'",
desc="Startup log")
time.sleep(10)
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
desc="Check port 8188")
# Verify threads are set
run("bash -c 'tail -40 /home/fabian/comfyui.log 2>/dev/null'",
desc="Full startup log")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_threads4.py
+144
View File
@@ -0,0 +1,144 @@
#!/usr/bin/env python3
"""Fix startup: use proper subprocess instead of exec, set threads via sitecustomize."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=120, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd[:300]}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 40:
print(f" ... ({len(lines)} lines, showing last 40)")
print('\n'.join(lines[-40:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-15:] if len(lines) > 15 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# 1. Create a sitecustomize.py in the venv to set threads on import
sitecustomize = '''# Auto-set PyTorch threading to use all 12 CPU cores on BC-250
import os
os.environ.setdefault("OMP_NUM_THREADS", "12")
os.environ.setdefault("MKL_NUM_THREADS", "12")
os.environ.setdefault("OPENBLAS_NUM_THREADS", "12")
try:
import torch
torch.set_num_threads(12)
torch.set_num_interop_threads(12)
except Exception:
pass
'''
# Find the venv site-packages path
rc, out, _ = run("bash -c 'source ~/comfyui-env/bin/activate && python3 -c \"import site; print(site.getsitepackages()[0])\"'",
desc="Find venv site-packages")
site_packages = out.strip()
print(f" Site-packages: {site_packages}")
# Write sitecustomize.py
sftp = ssh.open_sftp()
sitecust_path = f"{site_packages}/sitecustomize.py"
# Check if it exists first
try:
sftp.stat(sitecust_path)
print(f" sitecustomize.py already exists, backing up")
sftp.rename(sitecust_path, f"{sitecust_path}.bak")
except FileNotFoundError:
pass
with sftp.open(sitecust_path, 'w') as f:
f.write(sitecustomize)
sftp.close()
print(f" Written: {sitecust_path}")
# 2. Update startup script — simple, using exec python main.py directly
startup_script = r'''#!/bin/bash
# ComfyUI Startup Script for AsRock BC-250 (AMD Cyan Skillfish / ROCm 7.2)
# All 12 CPU cores + lowvram for 7.6GB shared VRAM
set -euo pipefail
# ═══════════════ BC-250 GPU ═══════════════
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# ═══════════════ ALL 12 CORES ═══════════════
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
export VECLIB_MAXIMUM_THREADS=12
export NUMEXPR_NUM_THREADS=12
# ═══════════════ Memory tuning ═══════════════
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:False"
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
# ═══════════════ Activate venv ═══════════════
source "$HOME/comfyui-env/bin/activate"
cd "$HOME/ComfyUI"
echo "=========================================="
echo " ComfyUI on BC-250 (ROCm 7.2)"
echo " GPU: AMD Cyan Skillfish (gfx1010)"
echo " PyTorch: $(python3 -c 'import torch; print(torch.__version__)')"
echo " Threads: $(python3 -c 'import torch; print(f"intra={torch.get_num_threads()} inter={torch.get_num_interop_threads()}")')"
echo " CPU: $(nproc) cores"
echo " VRAM: 7.6GB shared — lowvram mode"
echo "=========================================="
# Default args: listen on all, lowvram for tight VRAM
ARGS="--listen 0.0.0.0 --port 8188 --lowvram"
if [ $# -gt 0 ]; then
ARGS="$@"
fi
echo "Starting: python main.py $ARGS"
echo "Access: http://192.168.178.150:8188"
echo ""
exec python3 main.py $ARGS
'''
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
f.write(startup_script)
sftp.close()
run("chmod +x /home/fabian/start_comfyui.sh", desc="Make executable")
# 3. Launch
run("bash -c 'rm -f /home/fabian/comfyui.log'", desc="Clean log")
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
desc="Launch ComfyUI (12 cores + lowvram)")
time.sleep(20)
run("bash -c 'tail -40 /home/fabian/comfyui.log 2>/dev/null'",
desc="Startup log")
time.sleep(10)
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
desc="Check port 8188")
run("bash -c 'tail -50 /home/fabian/comfyui.log 2>/dev/null'",
desc="Full log")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_torchvision.py
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""Fix torchvision compatibility on BC-250.
The pip-installed torchvision conflicts with the system python-pytorch-rocm.
Need to use system torchvision-rocm or fix the version.
"""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=300, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 40:
print(f" ... ({len(lines)} lines, showing last 40)")
print('\n'.join(lines[-40:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-20:] if len(lines) > 20 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# Check what torchvision packages exist in repos
run("bash -c 'pacman -Ss torchvision 2>/dev/null'",
desc="Search for torchvision packages in repos")
run("bash -c 'pacman -Ss torchaudio 2>/dev/null'",
desc="Search for torchaudio packages")
# Check what's currently installed
run("bash -c 'pacman -Qs torch 2>/dev/null'",
desc="Currently installed torch packages (system)")
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && pip list 2>/dev/null | grep -i torch'",
desc="torch packages in venv")
# Check the versions
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && python -c \""
"import torch; print(f\\\"torch: {torch.__version__} from {torch.__file__}\\\"); "
"\"'",
desc="Check torch location")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_torchvision2.py
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Fix torchvision — use system package instead of pip version."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=300, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 40:
print(f" ... ({len(lines)} lines, showing last 40)")
print('\n'.join(lines[-40:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-20:] if len(lines) > 20 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# 1. Uninstall pip torchvision and torchaudio from venv
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && pip uninstall -y torchvision torchaudio 2>&1'",
desc="Uninstall pip torchvision and torchaudio from venv")
# 2. Install system python-torchvision via pacman
run("sudo pacman -S --noconfirm python-torchvision",
desc="Install system python-torchvision (matches system pytorch)")
# 3. Verify torchvision now works
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && "
"export HSA_OVERRIDE_GFX_VERSION=10.1.0 && "
"export HIP_VISIBLE_DEVICES=0 && "
"export HSA_ENABLE_SDMA=0 && "
"python -c \""
"import torch; print(f\\\"torch {torch.__version__} from {torch.__file__}\\\"); "
"import torchvision; print(f\\\"torchvision {torchvision.__version__} from {torchvision.__file__}\\\"); "
"print(\\\"torchvision ops OK\\\"); "
"\"'",
desc="Verify torchvision import works")
# 4. Kill old ComfyUI and relaunch
run("bash -c 'pkill -f \"python main.py\" 2>/dev/null; sleep 2; echo done'",
desc="Kill old ComfyUI process")
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
desc="Relaunch ComfyUI")
time.sleep(20)
run("tail -40 /home/fabian/comfyui.log 2>/dev/null",
desc="ComfyUI startup log")
time.sleep(10)
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
desc="Check if port 8188 is listening")
run("tail -60 /home/fabian/comfyui.log 2>/dev/null",
desc="Full ComfyUI log")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_vae.py
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""Fix VAE decode hang: kill stuck, check available flags, restart with --cpu-vae."""
import paramiko, json, time, textwrap
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=60, desc=""):
if desc:
print(f"\n{'='*60}\n {desc}\n{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 50:
print(f" ... ({len(lines)} lines, showing last 50)")
print('\n'.join(lines[-50:]))
else:
print(out.strip())
if err.strip():
for l in err.strip().split('\n')[-5:]:
print(f" STDERR: {l}")
return rc, out, err
# Kill stuck
run("pkill -f 'python3 main.py' 2>/dev/null; sleep 2; pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1; echo killed",
desc="Kill stuck ComfyUI")
# Check available VAE flags
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && "
"python3 main.py --help 2>&1 | grep -i -E \"vae|fp16|fp32|force|cpu|novram|lowvram\"'",
desc="ComfyUI VAE/VRAM flags")
# Update startup script: add --cpu-vae to keep diffusion on GPU but VAE on CPU
startup_script = textwrap.dedent("""\
#!/bin/bash
# BC-250 ComfyUI Launcher — GPU inference with CPU VAE decode
# Diffusion sampling: GPU (~6s/step, 8 steps = 51s total)
# VAE decode: CPU (GPU hangs on float32 VAE ops on Cyan Skillfish)
# Text encoding: CPU (GGUF model, dequant on CPU)
# GPU identity
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# Use all 12 CPU cores
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
# Activate venv
source /home/fabian/comfyui-env/bin/activate
cd /home/fabian/ComfyUI
# --novram: send one layer at a time to GPU (needed for 7.6GB shared VRAM)
# --force-fp16: halve VRAM usage for diffusion model
# --cpu-vae: decode VAE on CPU (GPU hangs on VAE float32 conv2d ops)
# --disable-smart-memory: prevent memory heuristics from interfering
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--novram \\
--force-fp16 \\
--cpu-vae \\
--disable-smart-memory
""")
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
f.write(startup_script)
sftp.close()
run("chmod +x /home/fabian/start_comfyui.sh")
print("\n Updated: added --cpu-vae (GPU sampler + CPU VAE)")
# Launch
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'; sleep 1; echo launched",
desc="Launch ComfyUI")
print("\n Waiting for server...")
for i in range(40):
time.sleep(3)
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null || echo 0'")
if out.strip() == '200':
print(f" Server ready! ({(i+1)*3}s)")
break
if i % 5 == 4:
rc2, log, _ = run("tail -3 /home/fabian/comfyui.log 2>/dev/null")
print(f" [{(i+1)*3}s] waiting... {log.strip().split(chr(10))[-1][:80]}")
else:
print(" Timeout!")
run("tail -40 /home/fabian/comfyui.log", desc="Log")
ssh.close()
exit(1)
# Submit workflow
workflow = {
"prompt": {
"1": {
"class_type": "UnetLoaderGGUF",
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
},
"2": {
"class_type": "CLIPLoaderGGUF",
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
},
"3": {
"class_type": "VAELoader",
"inputs": {"vae_name": "ae.safetensors"}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "A red fox in a snowy forest, photorealistic",
"clip": ["2", 0]
}
},
"5": {
"class_type": "CLIPTextEncode",
"inputs": {"text": "", "clip": ["2", 0]}
},
"6": {
"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}
},
"7": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0],
"seed": 42,
"steps": 8,
"cfg": 1.0,
"sampler_name": "euler",
"scheduler": "simple",
"positive": ["4", 0],
"negative": ["5", 0],
"latent_image": ["6", 0],
"denoise": 1.0
}
},
"8": {
"class_type": "VAEDecode",
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
},
"9": {
"class_type": "SaveImage",
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}
}
}
}
sftp2 = ssh.open_sftp()
with sftp2.open('/tmp/zimage_workflow.json', 'w') as f:
f.write(json.dumps(workflow))
sftp2.close()
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
"-H \"Content-Type: application/json\" "
"-d @/tmp/zimage_workflow.json'",
desc="Submit workflow (GPU sampling + CPU VAE)")
try:
resp = json.loads(out.strip())
if 'error' in resp:
print(f" ERROR: {resp['error']}")
if 'node_errors' in resp:
for nid, e in resp['node_errors'].items():
print(f" Node {nid}: {e}")
ssh.close()
exit(1)
print(f" Prompt ID: {resp.get('prompt_id')}")
except:
print(f" Response: {out.strip()[:500]}")
# Monitor
print("\n Monitoring GPU generation + CPU VAE decode...")
last_log = ""
for i in range(120):
time.sleep(15)
stats = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" CPU=$(ps -p $PID -o %cpu --no-headers); "
" MEM=$(ps -p $PID -o rss --no-headers); "
" GPU_TEMP=$(cat /sys/class/drm/card1/device/hwmon/hwmon*/temp1_input 2>/dev/null || echo 0); "
" GPU_POWER=$(cat /sys/class/drm/card1/device/hwmon/hwmon*/power1_average 2>/dev/null || echo 0); "
" echo \"CPU:${CPU}% RSS:$((MEM/1024))MB GPU_T:$((GPU_TEMP/1000))C GPU_P:$((GPU_POWER/1000000))W\"; "
"else echo DEAD; fi'")[1].strip()
log = run("tail -10 /home/fabian/comfyui.log 2>/dev/null")[1].strip()
elapsed = (i+1)*15
m, s = divmod(elapsed, 60)
print(f" [{m}m{s:02d}s] {stats}")
# Show last meaningful log line if changed
if log != last_log:
for line in reversed(log.split('\n')):
l = line.strip()
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION'):
print(f" LOG: {l[:120]}")
break
last_log = log
if 'DEAD' in stats:
print("\n PROCESS DIED!")
run("tail -60 /home/fabian/comfyui.log", desc="Death log")
break
if 'Prompt executed in' in log:
print(f"\n IMAGE GENERATED!")
run("tail -30 /home/fabian/comfyui.log", desc="Success log")
break
if 'Traceback' in log or 'CUDA out of memory' in log:
print("\n ERROR!")
run("tail -60 /home/fabian/comfyui.log", desc="Error log")
break
# Output
run("ls -lah /home/fabian/ComfyUI/output/", desc="Output files")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_fix_workflow.py
+163
View File
@@ -0,0 +1,163 @@
"""Save a proper Z-Image-Turbo GGUF workflow as the default ComfyUI web UI workflow."""
import paramiko, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
# ComfyUI web UI workflow format (not API format)
workflow = {
"last_node_id": 8,
"last_link_id": 8,
"nodes": [
{
"id": 1,
"type": "UnetLoaderGGUF",
"pos": [100, 100],
"size": [300, 80],
"flags": {},
"order": 0,
"mode": 0,
"outputs": [{"name": "MODEL", "type": "MODEL", "links": [1], "slot_index": 0}],
"properties": {"Node name for S&R": "UnetLoaderGGUF"},
"widgets_values": ["z_image_turbo-Q5_K_S.gguf"]
},
{
"id": 2,
"type": "CLIPLoaderGGUF",
"pos": [100, 250],
"size": [300, 80],
"flags": {},
"order": 1,
"mode": 0,
"outputs": [{"name": "CLIP", "type": "CLIP", "links": [2], "slot_index": 0}],
"properties": {"Node name for S&R": "CLIPLoaderGGUF"},
"widgets_values": ["Qwen3-4B.i1-Q5_K_S.gguf", "qwen_image"]
},
{
"id": 3,
"type": "VAELoader",
"pos": [100, 400],
"size": [300, 60],
"flags": {},
"order": 2,
"mode": 0,
"outputs": [{"name": "VAE", "type": "VAE", "links": [3], "slot_index": 0}],
"properties": {"Node name for S&R": "VAELoader"},
"widgets_values": ["ae.safetensors"]
},
{
"id": 4,
"type": "CLIPTextEncode",
"pos": [500, 250],
"size": [400, 120],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [{"name": "clip", "type": "CLIP", "link": 2}],
"outputs": [{"name": "CONDITIONING", "type": "CONDITIONING", "links": [4], "slot_index": 0}],
"properties": {"Node name for S&R": "CLIPTextEncode"},
"widgets_values": ["A red fox in a snowy forest, photorealistic, highly detailed"]
},
{
"id": 5,
"type": "EmptyLatentImage",
"pos": [500, 450],
"size": [300, 110],
"flags": {},
"order": 4,
"mode": 0,
"outputs": [{"name": "LATENT", "type": "LATENT", "links": [5], "slot_index": 0}],
"properties": {"Node name for S&R": "EmptyLatentImage"},
"widgets_values": [512, 512, 1]
},
{
"id": 6,
"type": "KSampler",
"pos": [950, 100],
"size": [320, 474],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{"name": "model", "type": "MODEL", "link": 1},
{"name": "positive", "type": "CONDITIONING", "link": 4},
{"name": "negative", "type": "CONDITIONING", "link": None},
{"name": "latent_image", "type": "LATENT", "link": 5}
],
"outputs": [{"name": "LATENT", "type": "LATENT", "links": [6], "slot_index": 0}],
"properties": {"Node name for S&R": "KSampler"},
"widgets_values": [42, "fixed", 8, 1.0, "euler", "simple", 1.0]
},
{
"id": 7,
"type": "VAEDecode",
"pos": [1350, 100],
"size": [210, 50],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{"name": "samples", "type": "LATENT", "link": 6},
{"name": "vae", "type": "VAE", "link": 3}
],
"outputs": [{"name": "IMAGE", "type": "IMAGE", "links": [7], "slot_index": 0}],
"properties": {"Node name for S&R": "VAEDecode"}
},
{
"id": 8,
"type": "SaveImage",
"pos": [1350, 250],
"size": [320, 270],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [{"name": "images", "type": "IMAGE", "link": 7}],
"properties": {"Node name for S&R": "SaveImage"},
"widgets_values": ["ZImageTurbo"]
}
],
"links": [
[1, 1, 0, 6, 0, "MODEL"],
[2, 2, 0, 4, 0, "CLIP"],
[3, 3, 0, 7, 1, "VAE"],
[4, 4, 0, 6, 1, "CONDITIONING"],
[5, 5, 0, 6, 3, "LATENT"],
[6, 6, 0, 7, 0, "LATENT"],
[7, 7, 0, 8, 0, "IMAGE"]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}
sftp = c.open_sftp()
# Save as default workflow
def ensure_dir(sftp, path):
try:
sftp.stat(path)
except FileNotFoundError:
ensure_dir(sftp, '/'.join(path.split('/')[:-1]))
sftp.mkdir(path)
ensure_dir(sftp, '/home/fabian/ComfyUI/user/default/comfyui')
wf_json = json.dumps(workflow, indent=2)
# Save as default workflow
with sftp.open('/home/fabian/ComfyUI/user/default/comfyui/workflow.json', 'w') as f:
f.write(wf_json)
print("Saved default workflow: ~/ComfyUI/user/default/comfyui/workflow.json")
# Also save a loadable copy in the ComfyUI root
with sftp.open('/home/fabian/ComfyUI/z_image_turbo_workflow.json', 'w') as f:
f.write(wf_json)
print("Saved loadable copy: ~/ComfyUI/z_image_turbo_workflow.json")
sftp.close()
c.close()
print("\nDone. Refresh ComfyUI web UI — it will load the Z-Image-Turbo GGUF workflow by default.")
print("If it still shows the old workflow, click the menu and Load the z_image_turbo_workflow.json file.")
ComfyUI Scripts/bc250_go.py
+291
View File
@@ -0,0 +1,291 @@
"""BC-250: Start ComfyUI on GPU and generate an image. Single SSH connection. No shell escaping issues."""
import paramiko
import time
import json
import sys
import textwrap
# ==== CONFIG ====
SSH_HOST = '192.168.178.150'
SSH_USER = 'fabian'
SSH_KEY = r'C:\Users\fabia\.ssh\id_ed25519'
def connect():
k = paramiko.Ed25519Key.from_private_key_file(SSH_KEY)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(SSH_HOST, username=SSH_USER, pkey=k, timeout=15)
return c
def sh(c, cmd, timeout=60):
"""Run a bash command. All commands go through bash explicitly."""
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command(f'/bin/bash -l -c {_quote(cmd)}')
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk:
break
out += chunk
except Exception:
break
chan.close()
return out.decode(errors='replace').strip()
def _quote(s):
"""Shell-quote a string using single quotes."""
return "'" + s.replace("'", "'\\''") + "'"
def write_remote_file(c, path, content):
"""Write a file on the remote via SFTP. No shell escaping needed."""
sftp = c.open_sftp()
with sftp.open(path, 'w') as f:
f.write(content)
sftp.close()
# ================================================================
print("="*60)
print("STEP 1: Connect + kill old ComfyUI")
print("="*60)
c = connect()
sh(c, 'pkill -9 -f "python3.*main.py" 2>/dev/null || true')
time.sleep(2)
alive = sh(c, 'pgrep -af "python3.*main.py" 2>/dev/null || echo NONE')
print(f" Old processes: {alive}")
# ================================================================
print("\n" + "="*60)
print("STEP 2: Write launcher script on BC-250")
print("="*60)
# Write a bash launcher script directly via SFTP - avoids ALL shell escaping issues
launcher = textwrap.dedent("""\
#!/bin/bash
# GPU environment
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
# Threading
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
# MIOpen
export MIOPEN_FIND_MODE=1
cd ~/ComfyUI
source ~/comfyui-env/bin/activate
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--lowvram \\
--force-fp16 \\
--cpu-vae \\
--disable-smart-memory
""")
write_remote_file(c, '/tmp/run_comfyui.sh', launcher)
sh(c, 'chmod +x /tmp/run_comfyui.sh')
print(" Launcher script written to /tmp/run_comfyui.sh")
print(" Flags: --lowvram --force-fp16 --cpu-vae --disable-smart-memory")
# ================================================================
print("\n" + "="*60)
print("STEP 3: Verify GPU works with PyTorch")
print("="*60)
gpu_script = textwrap.dedent("""\
#!/bin/bash
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
source ~/comfyui-env/bin/activate
python3 -c "
import torch
print('PyTorch:', torch.__version__)
print('CUDA/ROCm available:', torch.cuda.is_available())
if torch.cuda.is_available():
print('Device:', torch.cuda.get_device_name(0))
f,t = torch.cuda.mem_get_info(0)
print(f'VRAM: {f//1048576}MB free / {t//1048576}MB total')
x = torch.randn(512,512,device='cuda',dtype=torch.float16)
y = x @ x
print('GPU compute test: PASS')
else:
print('FATAL: NO GPU')
exit(1)
"
""")
write_remote_file(c, '/tmp/gpu_test.sh', gpu_script)
sh(c, 'chmod +x /tmp/gpu_test.sh')
out = sh(c, '/tmp/gpu_test.sh', timeout=30)
print(f" {out}")
if 'FATAL' in out or 'False' in out:
print(" *** GPU not working! Aborting. ***")
c.close()
sys.exit(1)
print(" GPU OK!")
# ================================================================
print("\n" + "="*60)
print("STEP 4: Start ComfyUI")
print("="*60)
sh(c, 'rm -f /tmp/comfyui.log; touch /tmp/comfyui.log')
sh(c, 'nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(3)
pid = sh(c, 'pgrep -f "python3.*main.py" 2>/dev/null || echo DEAD')
if pid == 'DEAD':
print(" FAILED to start! Log:")
print(sh(c, 'cat /tmp/comfyui.log'))
c.close()
sys.exit(1)
print(f" PID: {pid}")
# Wait for HTTP 200
print(" Waiting for HTTP ready...", end='', flush=True)
for i in range(90):
code = sh(c, 'curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null || echo 000', timeout=5)
if '200' in code:
print(f" READY ({i*2}s)")
break
print('.', end='', flush=True)
time.sleep(2)
else:
print(f"\n TIMEOUT! Last log:")
print(sh(c, 'tail -20 /tmp/comfyui.log'))
c.close()
sys.exit(1)
# Show startup flags from log
log_head = sh(c, 'head -10 /tmp/comfyui.log')
print(f"\n Startup log:\n {log_head[:300]}")
# ================================================================
print("\n" + "="*60)
print("STEP 5: Submit workflow")
print("="*60)
workflow = {
"prompt": {
"1": {
"class_type": "UnetLoaderGGUF",
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
},
"2": {
"class_type": "CLIPLoaderGGUF",
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
},
"3": {
"class_type": "VAELoader",
"inputs": {"vae_name": "ae.safetensors"}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}
},
"5": {
"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}
},
"6": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 42, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
}
},
"7": {
"class_type": "VAEDecode",
"inputs": {"samples": ["6", 0], "vae": ["3", 0]}
},
"8": {
"class_type": "SaveImage",
"inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}
}
}
}
write_remote_file(c, '/tmp/wf.json', json.dumps(workflow))
# Verify it's valid JSON with correct nodes
verify = sh(c, 'python3 -c "import json; d=json.load(open(\'/tmp/wf.json\')); p=d[\'prompt\']; print(len(p), \'nodes:\', sorted(p.keys()))"')
print(f" Workflow: {verify}")
# Submit
resp = sh(c, 'curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json', timeout=10)
print(f" Response: {resp[:200]}")
if 'prompt_id' not in resp:
print(" *** SUBMIT FAILED! ***")
print(f" Full response: {resp}")
print(f" Log: {sh(c, 'tail -10 /tmp/comfyui.log')}")
c.close()
sys.exit(1)
prompt_id = json.loads(resp).get('prompt_id', '?')
print(f" Prompt ID: {prompt_id}")
# ================================================================
print("\n" + "="*60)
print("STEP 6: Monitor generation (checking GPU usage)")
print("="*60)
t0 = time.time()
for i in range(200): # up to ~50 min
elapsed = int(time.time() - t0)
gpu_pct = sh(c, 'cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null || echo ?', timeout=5)
gpu_temp = sh(c, 'cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null || echo 0', timeout=5)
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
log_tail = sh(c, 'tail -3 /tmp/comfyui.log 2>/dev/null', timeout=5)
last_line = log_tail.strip().split('\n')[-1] if log_tail else ''
# Check for output image
imgs = sh(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null || echo NONE', timeout=5)
print(f" [{elapsed:>4}s] GPU:{gpu_pct:>3}% {temp_c}C | {last_line[-90:]}")
if imgs != 'NONE':
print(f"\n >>> IMAGE GENERATED! <<<")
print(f" Files: {imgs}")
print(f" Time: {elapsed}s")
final = sh(c, 'tail -20 /tmp/comfyui.log')
print(f"\n Final log:\n{final}")
break
# Check queue empty (= done or error)
q = sh(c, 'curl -s http://127.0.0.1:8188/queue 2>/dev/null || echo {}', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 20:
time.sleep(3)
imgs2 = sh(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null || echo NONE', timeout=5)
if imgs2 != 'NONE':
print(f"\n >>> IMAGE GENERATED! <<<")
print(f" Files: {imgs2}")
print(f" Time: {elapsed}s")
else:
print(f"\n Queue empty, no image. Checking log for errors...")
print(sh(c, 'tail -30 /tmp/comfyui.log'))
break
except json.JSONDecodeError:
pass
# Check process still alive
alive = sh(c, 'pgrep -f "python3.*main.py" >/dev/null 2>&1 && echo YES || echo NO', timeout=5)
if alive == 'NO':
print(f"\n *** ComfyUI CRASHED! ***")
print(sh(c, 'tail -40 /tmp/comfyui.log'))
break
time.sleep(15)
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_go_now.py
+107
View File
@@ -0,0 +1,107 @@
"""Just start ComfyUI and submit workflow. All patches applied."""
import paramiko, time, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, t=30):
ch = c.get_transport().open_session()
ch.settimeout(t)
ch.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
o = b""
while True:
try:
d = ch.recv(65536)
if not d: break
o += d
except: break
ch.close()
return o.decode(errors='replace').strip()
# Kill any leftover, clean logs
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 1')
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
# Start
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(5)
pid = sh('pgrep -f "python3.*main.py"')
print(f"Started PID: {pid}")
# Wait for HTTP
for i in range(60):
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', t=5)
if '200' in r:
print(f"HTTP ready ({i*2}s)")
break
time.sleep(2)
# Confirm SHARED
log = ''
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: pass
for l in log.split('\n'):
s = l.strip()
if any(x in s for x in ['vram state', 'SHARED', 'Device:']): print(f" {s}")
# Submit
wf = {"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(wf))
print("Submitting...")
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f" {resp[:120]}")
# Monitor
t0 = time.time()
shown = set()
for i in range(150):
el = int(time.time() - t0)
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
for l in log.split('\n'):
s = l.strip()
if s and s not in shown and any(x in s for x in ['/8', 'loaded', 'load device', 'offload device',
'Requested', 'VAE', 'Prompt executed', 'Error', 'OOM', 'CUDA']):
if 'FETCH' not in s and 'audio_vae' not in s and 'split attention' not in s:
gpu = sh('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null', t=3)
print(f" [{el:>3}s] GPU:{gpu}% {s[-110:]}")
shown.add(s)
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', t=5)
if imgs:
print(f"\n*** DONE in {el}s! ***")
for l in log.split('\n'):
s = l.strip()
if 'Prompt executed' in s: print(f" {s}")
break
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', t=5) == 'N':
print(f"\nCRASHED at {el}s!")
for l in log.split('\n')[-15:]:
if l.strip(): print(f" {l.strip()}")
break
time.sleep(4)
sftp.close()
c.close()
ComfyUI Scripts/bc250_gpu_final.py
+358
View File
@@ -0,0 +1,358 @@
"""BC-250 Full GPU Fix: Verify ROCm, diagnose VRAM, start ComfyUI on GPU, generate image."""
import paramiko
import time
import json
import sys
SSH_HOST = '192.168.178.150'
SSH_USER = 'fabian'
SSH_KEY = r'C:\Users\fabia\.ssh\id_ed25519'
def ssh_connect():
k = paramiko.Ed25519Key.from_private_key_file(SSH_KEY)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(SSH_HOST, username=SSH_USER, pkey=k, timeout=15)
return c
def run(c, cmd, timeout=30):
"""Run command via fish shell, return stdout."""
wrapped = f'bash -c {repr(cmd)}'
_, o, e = c.exec_command(wrapped, timeout=timeout)
return o.read().decode(errors='replace').strip()
def run_full(c, cmd, timeout=30):
"""Run command, return (stdout, stderr)."""
wrapped = f'bash -c {repr(cmd)}'
_, o, e = c.exec_command(wrapped, timeout=timeout)
return o.read().decode(errors='replace').strip(), e.read().decode(errors='replace').strip()
# ============================================================
# PHASE 1: Kill any remnants
# ============================================================
print("="*60)
print("PHASE 1: Clean slate")
print("="*60)
c = ssh_connect()
run(c, 'pkill -f "python.*main.py" 2>/dev/null; pkill -f comfyui 2>/dev/null')
time.sleep(2)
leftover = run(c, 'pgrep -af "python.*main.py" 2>/dev/null')
if leftover:
print(f"WARNING: Still running: {leftover}")
run(c, 'pkill -9 -f "python.*main.py" 2>/dev/null')
time.sleep(1)
print("ComfyUI killed. Clean slate.")
# ============================================================
# PHASE 2: Verify ROCm + PyTorch GPU
# ============================================================
print("\n" + "="*60)
print("PHASE 2: Verify ROCm + PyTorch GPU access")
print("="*60)
# Set GPU env vars for ALL subsequent commands
GPU_ENV = (
'export HSA_OVERRIDE_GFX_VERSION=10.1.0; '
'export HIP_VISIBLE_DEVICES=0; '
'export HSA_ENABLE_SDMA=0; '
'export HSA_TOOLS_LIB=""; '
'export HSA_TOOLS_REPORT_LOAD_FAILURE=0; '
'export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False; '
)
# Check rocminfo
out = run(c, f'{GPU_ENV} rocminfo 2>&1 | grep -E "Name:|Marketing Name:|gfx" | head -10')
print(f"ROCm devices:\n{out}")
# Check PyTorch GPU
gpu_test = f'''{GPU_ENV} cd ~/ComfyUI && source ~/comfyui-env/bin/activate.fish 2>/dev/null; . ~/comfyui-env/bin/activate 2>/dev/null; python3 -c "
import torch
print(f'PyTorch: {{torch.__version__}}')
print(f'CUDA available: {{torch.cuda.is_available()}}')
print(f'Device count: {{torch.cuda.device_count()}}')
if torch.cuda.is_available():
print(f'Device name: {{torch.cuda.get_device_name(0)}}')
free, total = torch.cuda.mem_get_info(0)
print(f'VRAM: {{free//1024//1024}}MB free / {{total//1024//1024}}MB total')
# Quick GPU compute test
x = torch.randn(1024, 1024, device='cuda', dtype=torch.float16)
y = torch.mm(x, x)
print(f'GPU compute test: OK (result sum={{y.sum().item():.1f}})')
del x, y
torch.cuda.empty_cache()
else:
print('ERROR: GPU NOT AVAILABLE')
import sys; sys.exit(1)
"'''
out, err = run_full(c, gpu_test, timeout=60)
print(out)
if err:
print(f"STDERR: {err}")
if 'ERROR: GPU NOT AVAILABLE' in out or 'CUDA available: False' in out:
print("\n*** FATAL: PyTorch cannot see the GPU! ***")
c.close()
sys.exit(1)
print("\nGPU verified OK!")
# ============================================================
# PHASE 3: Start ComfyUI with correct GPU flags
# ============================================================
print("\n" + "="*60)
print("PHASE 3: Start ComfyUI with GPU")
print("="*60)
# The key insight: --novram was offloading EVERYTHING to CPU (0 MB on GPU)
# For this APU with shared memory, --lowvram is better:
# it keeps compute on GPU but swaps model layers in/out
# We also use --force-fp16 to reduce memory pressure
# --cpu-vae to avoid the known VAE decode hang on this GPU
COMFYUI_CMD = (
f'{GPU_ENV} '
'export OMP_NUM_THREADS=12; '
'export MKL_NUM_THREADS=12; '
'export OPENBLAS_NUM_THREADS=12; '
'export MIOPEN_FIND_MODE=1; ' # Fast MIOpen kernel search
'cd ~/ComfyUI && '
'source ~/comfyui-env/bin/activate 2>/dev/null; . ~/comfyui-env/bin/activate 2>/dev/null; '
'nohup python3 main.py '
'--listen 0.0.0.0 --port 8188 '
'--lowvram '
'--force-fp16 '
'--cpu-vae '
'--disable-smart-memory '
'> /tmp/comfyui.log 2>&1 &'
)
# Truncate old log first
run(c, 'truncate -s 0 /tmp/comfyui.log 2>/dev/null; touch /tmp/comfyui.log')
print("Starting ComfyUI with: --lowvram --force-fp16 --cpu-vae --disable-smart-memory")
print("(--lowvram keeps compute on GPU, swaps layers; --novram was wrong - it put everything on CPU)")
run(c, COMFYUI_CMD)
time.sleep(3)
# Verify it started
pid = run(c, 'pgrep -f "python.*main.py" 2>/dev/null')
if not pid:
print("ERROR: ComfyUI failed to start!")
log = run(c, 'cat /tmp/comfyui.log')
print(f"Log:\n{log}")
c.close()
sys.exit(1)
print(f"ComfyUI started, PID: {pid}")
# Wait for server ready
print("Waiting for server ready...")
for i in range(60):
try:
resp = run(c, 'curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
if resp == '200':
print(f"Server ready after {i*3}s!")
break
except:
pass
# Also check for crash
log_tail = run(c, 'tail -3 /tmp/comfyui.log 2>/dev/null')
if 'Traceback' in log_tail or 'Error' in log_tail:
print(f"Server log issue: {log_tail}")
if i % 5 == 0 and i > 0:
print(f" [{i*3}s] Still waiting... log: {log_tail[-80:]}")
time.sleep(3)
else:
print("TIMEOUT waiting for ComfyUI!")
log = run(c, 'tail -30 /tmp/comfyui.log')
print(f"Log:\n{log}")
c.close()
sys.exit(1)
# Print startup log to confirm flags
log = run(c, 'head -20 /tmp/comfyui.log')
print(f"\nStartup log:\n{log}")
# ============================================================
# PHASE 4: Submit workflow via SFTP
# ============================================================
print("\n" + "="*60)
print("PHASE 4: Submit workflow")
print("="*60)
workflow = {
"prompt": {
"1": {
"class_type": "UnetLoaderGGUF",
"inputs": {
"unet_name": "z_image_turbo-Q5_K_S.gguf"
}
},
"2": {
"class_type": "CLIPLoaderGGUF",
"inputs": {
"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf",
"type": "qwen_image"
}
},
"3": {
"class_type": "VAELoader",
"inputs": {
"vae_name": "ae.safetensors"
}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "A red fox in a snowy forest, photorealistic, highly detailed",
"clip": ["2", 0]
}
},
"5": {
"class_type": "EmptyLatentImage",
"inputs": {
"width": 512,
"height": 512,
"batch_size": 1
}
},
"6": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0],
"positive": ["4", 0],
"negative": ["4", 0],
"latent_image": ["5", 0],
"seed": 42,
"steps": 8,
"cfg": 1.0,
"sampler_name": "euler",
"scheduler": "simple",
"denoise": 1.0
}
},
"7": {
"class_type": "VAEDecode",
"inputs": {
"samples": ["6", 0],
"vae": ["3", 0]
}
},
"8": {
"class_type": "SaveImage",
"inputs": {
"images": ["7", 0],
"filename_prefix": "ZImageTurbo_GPU"
}
}
}
}
# Write via SFTP
sftp = c.open_sftp()
wf_json = json.dumps(workflow)
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(wf_json)
sftp.close()
print("Workflow written to /tmp/wf.json via SFTP")
# Verify JSON
verify = run(c, 'python3 -c "import json; d=json.load(open(\'/tmp/wf.json\')); print(f\'Nodes: {list(d[chr(34)+chr(34) if False else \"prompt\"].keys())}\')"')
print(f"Verify: {verify}")
# Submit
resp = run(c, 'curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json 2>/dev/null')
print(f"Submit response: {resp}")
if 'error' in resp.lower() and 'prompt_id' not in resp.lower():
print(f"\n*** SUBMISSION ERROR ***")
# Check what went wrong
log = run(c, 'tail -10 /tmp/comfyui.log')
print(f"Log: {log}")
c.close()
sys.exit(1)
try:
resp_data = json.loads(resp)
prompt_id = resp_data.get('prompt_id', 'unknown')
print(f"Prompt ID: {prompt_id}")
except:
print("Could not parse response, continuing anyway...")
# ============================================================
# PHASE 5: Monitor generation with GPU tracking
# ============================================================
print("\n" + "="*60)
print("PHASE 5: Monitor generation (GPU must be active!)")
print("="*60)
start_time = time.time()
last_log_len = 0
for i in range(120): # Up to 30 minutes
elapsed = int(time.time() - start_time)
# GPU metrics
gpu_pct = run(c, 'cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null')
gpu_temp = run(c, 'cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null')
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
# GPU power
gpu_power = run(c, f'{GPU_ENV} rocm-smi -P 2>&1 | grep "Graphics Package" | grep -oP "[\\d.]+" | head -1')
# Process info
proc = run(c, 'ps -p $(pgrep -f "python.*main.py" | head -1) -o %cpu,%mem,rss --no-headers 2>/dev/null')
# Log tail
log = run(c, 'tail -5 /tmp/comfyui.log 2>/dev/null')
last_line = log.split('\n')[-1] if log else ''
# Output files
files = run(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null')
# Queue
queue = run(c, 'curl -s http://127.0.0.1:8188/queue 2>/dev/null')
status = f"[{elapsed:>4}s] GPU:{gpu_pct:>3}% {temp_c}C {gpu_power}W | proc:{proc} | {last_line[-100:]}"
print(status)
# SUCCESS: Image generated!
if files:
print(f"\n{'='*60}")
print(f"*** SUCCESS! IMAGE GENERATED! ***")
print(f"Files: {files}")
print(f"Total time: {elapsed}s")
print(f"{'='*60}")
# Print final log
final_log = run(c, 'tail -20 /tmp/comfyui.log 2>/dev/null')
print(f"\nFinal log:\n{final_log}")
break
# Check if queue is empty (job done or failed)
try:
qdata = json.loads(queue)
running = len(qdata.get('queue_running', []))
pending = len(qdata.get('queue_pending', []))
if running == 0 and pending == 0 and elapsed > 30:
print(f"\nQueue empty after {elapsed}s. Checking if image was saved...")
time.sleep(2)
files = run(c, 'ls ~/ComfyUI/output/*.png 2>/dev/null')
if files:
print(f"*** SUCCESS! {files}")
else:
print("No image. Checking log for errors:")
err_log = run(c, 'tail -30 /tmp/comfyui.log 2>/dev/null')
print(err_log)
break
except:
pass
# Check for process death
alive = run(c, 'pgrep -f "python.*main.py" 2>/dev/null')
if not alive:
print("\n*** ComfyUI process died! ***")
crash_log = run(c, 'tail -40 /tmp/comfyui.log 2>/dev/null')
print(f"Crash log:\n{crash_log}")
break
time.sleep(15)
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_gpu_fix.py
+283
View File
@@ -0,0 +1,283 @@
#!/usr/bin/env python3
"""Fix GPU inference: kill stuck, diagnose, restart with --novram, test."""
import paramiko, json, time, textwrap
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=60, desc=""):
if desc:
print(f"\n{'='*60}\n {desc}\n{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
combined = out.strip()
if combined:
lines = combined.split('\n')
if len(lines) > 50:
print(f" ... ({len(lines)} lines, showing last 50)")
print('\n'.join(lines[-50:]))
else:
print(combined)
if err.strip():
for line in err.strip().split('\n')[-10:]:
print(f" STDERR: {line}")
return rc, out, err
# ============================================================
# STEP 1: Kill stuck ComfyUI
# ============================================================
run("pkill -f 'python3 main.py' 2>/dev/null; sleep 2; "
"pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1; "
"echo 'Killed.'", desc="Kill stuck ComfyUI")
# ============================================================
# STEP 2: Check dmesg for GPU errors
# ============================================================
run("dmesg | grep -i -E 'amdgpu|error|fault|gpu|kiq|gfx' | tail -30",
desc="Check dmesg for GPU errors")
# ============================================================
# STEP 3: Quick GPU sanity test
# ============================================================
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && "
"HSA_OVERRIDE_GFX_VERSION=10.1.0 HSA_ENABLE_SDMA=0 "
"python3 -c \""
"import torch; "
"print(f\\\"CUDA available: {torch.cuda.is_available()}\\\"); "
"print(f\\\"Device: {torch.cuda.get_device_name(0)}\\\"); "
"a = torch.randn(1024, 1024, device=\\\"cuda\\\"); "
"b = torch.randn(1024, 1024, device=\\\"cuda\\\"); "
"c = a @ b; "
"print(f\\\"Matmul result shape: {c.shape}, sum: {c.sum().item():.2f}\\\"); "
"# Test fp16 "
"a16 = a.half(); b16 = b.half(); c16 = a16 @ b16; "
"print(f\\\"FP16 matmul OK: {c16.shape}\\\"); "
"print(f\\\"Free VRAM: {torch.cuda.mem_get_info()[0]/1024**2:.0f} MB\\\"); "
"print(f\\\"Total VRAM: {torch.cuda.mem_get_info()[1]/1024**2:.0f} MB\\\"); "
"print(\\\"GPU SANITY: PASS\\\")\"'",
desc="Quick GPU sanity test")
# ============================================================
# STEP 4: Check what ComfyUI flags are available
# ============================================================
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && "
"python3 main.py --help 2>&1 | grep -E \"novram|lowvram|cpu|fp16|force|vram|disable-smart|channels\"'",
desc="ComfyUI VRAM-related flags")
# ============================================================
# STEP 5: Write new startup script with --novram
# ============================================================
startup_script = textwrap.dedent("""\
#!/bin/bash
# BC-250 ComfyUI Launcher - GPU mode with aggressive offloading
# GPU identity
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
# Disable SDMA (known issue on Cyan Skillfish)
export HSA_ENABLE_SDMA=0
# Suppress HSA tool warnings
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# Threading: use all 12 cores for CPU-side work
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
# HIP memory: allow expandable segments to reduce fragmentation
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:True
# Activate venv
source /home/fabian/comfyui-env/bin/activate
cd /home/fabian/ComfyUI
# --novram: most aggressive offloading - keeps almost nothing on GPU,
# sends individual layers to GPU one at a time during forward pass.
# This is needed because BC-250 has only ~7.6GB shared VRAM.
# --disable-smart-memory: prevents ComfyUI from trying to be clever about memory
# --force-fp16: force fp16 to halve VRAM usage
exec python3 main.py --listen 0.0.0.0 --port 8188 --novram --force-fp16 --disable-smart-memory
""")
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
f.write(startup_script)
sftp.close()
run("chmod +x /home/fabian/start_comfyui.sh", desc="Make script executable")
print("\n Startup script updated with --novram --force-fp16 --disable-smart-memory")
# ============================================================
# STEP 6: Launch ComfyUI with new settings
# ============================================================
run("bash -c 'nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &'; sleep 1; echo 'Launched'",
desc="Launch ComfyUI with --novram")
# Wait for server to be ready
print("\n Waiting for server to start...")
for i in range(30):
time.sleep(3)
rc, out, _ = run("bash -c 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8188/ 2>/dev/null || echo 0'")
code = out.strip()
if code == '200':
print(f" Server ready after {(i+1)*3}s!")
break
# Check log for errors
rc2, log, _ = run("tail -3 /home/fabian/comfyui.log 2>/dev/null")
if 'Error' in log or 'error' in log.lower():
print(f" Log: {log.strip()}")
print(f" [{(i+1)*3}s] HTTP {code}...")
else:
print(" Server didn't start in 90s!")
run("tail -40 /home/fabian/comfyui.log", desc="Startup log")
ssh.close()
exit(1)
# Confirm server info
run("tail -30 /home/fabian/comfyui.log", desc="Startup log")
# ============================================================
# STEP 7: Submit test workflow (smaller 512x512 image first)
# ============================================================
workflow = {
"prompt": {
"1": {
"class_type": "UnetLoaderGGUF",
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}
},
"2": {
"class_type": "CLIPLoaderGGUF",
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}
},
"3": {
"class_type": "VAELoader",
"inputs": {"vae_name": "ae.safetensors"}
},
"4": {
"class_type": "CLIPTextEncode",
"inputs": {
"text": "A red fox in a snowy forest, photorealistic",
"clip": ["2", 0]
}
},
"5": {
"class_type": "CLIPTextEncode",
"inputs": {"text": "", "clip": ["2", 0]}
},
"6": {
"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}
},
"7": {
"class_type": "KSampler",
"inputs": {
"model": ["1", 0],
"seed": 12345,
"steps": 8,
"cfg": 1.0,
"sampler_name": "euler",
"scheduler": "simple",
"positive": ["4", 0],
"negative": ["5", 0],
"latent_image": ["6", 0],
"denoise": 1.0
}
},
"8": {
"class_type": "VAEDecode",
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}
},
"9": {
"class_type": "SaveImage",
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}
}
}
}
sftp2 = ssh.open_sftp()
with sftp2.open('/tmp/zimage_workflow.json', 'w') as f:
f.write(json.dumps(workflow))
sftp2.close()
rc, out, _ = run("bash -c 'curl -s -X POST http://localhost:8188/prompt "
"-H \"Content-Type: application/json\" "
"-d @/tmp/zimage_workflow.json'",
desc="Submit 512x512 test workflow")
prompt_id = None
try:
resp = json.loads(out.strip())
if 'error' in resp:
print(f"\n API ERROR: {resp['error']}")
if 'node_errors' in resp:
for nid, e in resp['node_errors'].items():
print(f" Node {nid}: {e}")
ssh.close()
exit(1)
prompt_id = resp.get('prompt_id', 'unknown')
print(f"\n Prompt ID: {prompt_id}")
except:
print(f" Raw response: {out.strip()[:500]}")
# ============================================================
# STEP 8: Monitor generation
# ============================================================
print("\n Monitoring GPU generation...")
last_log = ""
for i in range(120): # up to 30 minutes
time.sleep(15)
# CPU + GPU status
rc, status, _ = run("bash -c '"
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" CPU=$(ps -p $PID -o %cpu --no-headers); "
" MEM=$(ps -p $PID -o %mem --no-headers); "
" THREADS=$(ps -p $PID -o nlwp --no-headers); "
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
" GPU_USE=$(cat /sys/class/drm/card1/device/gpu_busy_percent 2>/dev/null || echo N/A); "
" VRAM_USED=$(cat /sys/class/drm/card1/device/mem_info_vram_used 2>/dev/null || echo 0); "
" VRAM_TOTAL=$(cat /sys/class/drm/card1/device/mem_info_vram_total 2>/dev/null || echo 1); "
" echo \"CPU:${CPU}% MEM:${MEM}% THR:${THREADS} LOAD:${LOAD} GPU:${GPU_USE}% VRAM:$((VRAM_USED/1048576))/$((VRAM_TOTAL/1048576))MB\"; "
"else echo DEAD; fi'")
rc, log, _ = run("bash -c 'tail -8 /home/fabian/comfyui.log 2>/dev/null'")
log_lines = log.strip()
# Show status
status_line = status.strip()
print(f" [{i+1}] {(i+1)*15}s | {status_line}")
# Show new log lines
if log_lines != last_log:
new_part = log_lines
for line in new_part.split('\n')[-5:]:
if line.strip():
print(f" LOG: {line.strip()}")
last_log = log_lines
if 'DEAD' in status_line:
print("\n ComfyUI DIED!")
run("tail -60 /home/fabian/comfyui.log", desc="Death log")
break
if 'Prompt executed in' in log_lines:
print(f"\n SUCCESS! Image generated at check {i+1} (~{(i+1)*15}s)")
break
if 'Error' in log_lines or 'Traceback' in log_lines:
print("\n ERROR detected!")
run("tail -60 /home/fabian/comfyui.log", desc="Error log")
break
# Final check
run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null", desc="Output files")
run("tail -25 /home/fabian/comfyui.log", desc="Final log")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_gpu_fix2.py
+226
View File
@@ -0,0 +1,226 @@
#!/usr/bin/env python3
"""
Single SSH connection: kill stuck, check flags, update startup, launch, submit, monitor.
Properly closes connection when done.
"""
import paramiko
import json
import time
import sys
KEY = r'C:\Users\fabia\.ssh\id_ed25519'
HOST = '192.168.178.150'
USER = 'fabian'
def connect():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for attempt in range(5):
try:
ssh.connect(HOST, username=USER, key_filename=KEY, timeout=10)
return ssh
except Exception as e:
print(f" SSH attempt {attempt+1}/5 failed: {e}")
time.sleep(10)
print("FATAL: Cannot connect to BC-250")
sys.exit(1)
def run(ssh, cmd, timeout=120):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
return out, err
def main():
ssh = connect()
print("Connected to BC-250.\n")
try:
# ── 1. Kill stuck ComfyUI ──
print("=== STEP 1: Kill stuck ComfyUI ===")
out, _ = run(ssh, "pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 2; "
"pgrep -f 'python3 main.py' || echo 'all_dead'")
print(f" {out.strip()}")
# ── 2. Check available flags ──
print("\n=== STEP 2: ComfyUI flags ===")
out, err = run(ssh, "bash -c 'source /home/fabian/comfyui-env/bin/activate && "
"cd /home/fabian/ComfyUI && python3 main.py --help 2>&1'")
combined = out + err
for line in combined.split('\n'):
low = line.lower()
if any(w in low for w in ['vae', 'fp16', 'fp32', 'force', 'cpu', 'vram',
'memory', 'offload', 'precision', 'novram', 'lowvram']):
print(f" {line.strip()}")
# ── 3. Write startup script ──
print("\n=== STEP 3: Update startup script ===")
script = r"""#!/bin/bash
# BC-250 ComfyUI Launcher - GPU inference with CPU VAE decode
# GPU identity
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# Use all 12 CPU cores
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
# Activate venv
source /home/fabian/comfyui-env/bin/activate
cd /home/fabian/ComfyUI
# --novram: aggressive offload, one layer at a time to GPU
# --force-fp16: halve VRAM for diffusion
# --cpu-vae: VAE decode on CPU (GPU hangs on float32 VAE conv2d)
# --disable-smart-memory: no memory heuristics
exec python3 main.py \
--listen 0.0.0.0 --port 8188 \
--novram \
--force-fp16 \
--cpu-vae \
--disable-smart-memory
"""
# Write via heredoc to avoid SFTP
escaped = script.replace("'", "'\\''")
out, _ = run(ssh, f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n"
f"chmod +x /home/fabian/start_comfyui.sh && echo 'written'")
print(f" {out.strip()}")
# ── 4. Launch ComfyUI ──
print("\n=== STEP 4: Launch ComfyUI ===")
run(ssh, "nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &")
time.sleep(2)
print(" Waiting for server...")
for i in range(40):
time.sleep(3)
out, _ = run(ssh, "curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0")
code = out.strip()
if code == '200':
print(f" Server ready ({(i+1)*3}s)")
break
if i % 5 == 4:
log, _ = run(ssh, "tail -2 /home/fabian/comfyui.log 2>/dev/null")
last = [l.strip() for l in log.strip().split('\n') if l.strip()][-1:]
print(f" [{(i+1)*3}s] HTTP {code} ... {last[0][:80] if last else ''}")
else:
print(" Server didn't start in 120s!")
out, _ = run(ssh, "tail -40 /home/fabian/comfyui.log 2>/dev/null")
print(out)
return
# Show startup log
out, _ = run(ssh, "tail -20 /home/fabian/comfyui.log 2>/dev/null")
for line in out.strip().split('\n'):
l = line.strip()
if l and not l.startswith('FETCH'):
print(f" LOG: {l[:120]}")
# ── 5. Submit workflow ──
print("\n=== STEP 5: Submit workflow (512x512, 8 steps) ===")
workflow = {
"prompt": {
"1": {"class_type": "UnetLoaderGGUF",
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF",
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader",
"inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode",
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "CLIPTextEncode",
"inputs": {"text": "", "clip": ["2", 0]}},
"6": {"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
"7": {"class_type": "KSampler",
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple",
"positive": ["4", 0], "negative": ["5", 0],
"latent_image": ["6", 0], "denoise": 1.0}},
"8": {"class_type": "VAEDecode",
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
"9": {"class_type": "SaveImage",
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
}
}
wf_json = json.dumps(workflow).replace("'", "'\\''")
out, _ = run(ssh, f"curl -s -X POST http://localhost:8188/prompt "
f"-H 'Content-Type: application/json' "
f"-d '{wf_json}'")
try:
resp = json.loads(out.strip())
if 'error' in resp:
print(f" API ERROR: {resp['error']}")
if 'node_errors' in resp:
for nid, e in resp['node_errors'].items():
print(f" Node {nid}: {e}")
return
print(f" Prompt ID: {resp.get('prompt_id')}")
except:
print(f" Response: {out.strip()[:500]}")
# ── 6. Monitor generation ──
print("\n=== STEP 6: Monitoring generation ===")
last_log = ""
for i in range(120): # up to 30 minutes
time.sleep(15)
stats, _ = run(ssh, "bash -c '"
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" CPU=$(ps -p $PID -o %cpu --no-headers); "
" MEM=$(ps -p $PID -o rss --no-headers); "
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1-3); "
" echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:${LOAD}\"; "
"else echo DEAD; fi'")
log, _ = run(ssh, "tail -10 /home/fabian/comfyui.log 2>/dev/null")
log_s = log.strip()
elapsed = (i+1) * 15
m, s = divmod(elapsed, 60)
print(f" [{m}m{s:02d}s] {stats.strip()}")
if log_s != last_log:
for line in reversed(log_s.split('\n')):
l = line.strip()
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
print(f" LOG: {l[:120]}")
break
last_log = log_s
if 'DEAD' in stats:
print("\n PROCESS DIED!")
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
print(out)
break
if 'Prompt executed in' in log_s:
print(f"\n SUCCESS! Image generated!")
out, _ = run(ssh, "tail -30 /home/fabian/comfyui.log 2>/dev/null")
print(out)
break
if 'Traceback' in log_s or 'RuntimeError' in log_s:
print("\n ERROR detected!")
out, _ = run(ssh, "tail -60 /home/fabian/comfyui.log 2>/dev/null")
print(out)
break
# ── 7. Check output ──
print("\n=== Output files ===")
out, _ = run(ssh, "ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null")
print(out.strip())
finally:
ssh.close()
print("\nSSH connection closed.")
if __name__ == '__main__':
main()
ComfyUI Scripts/bc250_gpu_fix3.py
+316
View File
@@ -0,0 +1,316 @@
#!/usr/bin/env python3
"""
Fix: Add MIOpen fast-find, pre-warm GPU, restart ComfyUI, generate.
All GPU ops confirmed working. The hang is likely cold MIOpen kernel cache.
Single SSH connection.
"""
import paramiko, json, time, sys
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for attempt in range(5):
try:
ssh.connect('192.168.178.150', username='fabian',
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
break
except Exception as e:
print(f" SSH attempt {attempt+1}/5: {e}")
time.sleep(10)
else:
print("FATAL: Cannot connect"); sys.exit(1)
def run(cmd, timeout=300):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
return out, err
try:
# 1. Kill any lingering ComfyUI
print("=== Kill any ComfyUI ===")
run("pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1")
print(" Done")
# 2. Pre-warm MIOpen kernel cache with DiT-like operations
print("\n=== Pre-warming MIOpen kernel cache ===")
print(" This compiles HIP kernels that Z-Image-Turbo will need.")
print(" First run after reboot is slow (kernel compilation)...")
warmup_code = r"""
import torch, torch.nn as nn, time
# Simulate Z-Image-Turbo DiT operations
device = 'cuda'
dtype = torch.float16
print("Warming up HIP kernels for DiT inference...")
t0 = time.time()
# 1. Linear layers (DiT blocks)
print(" Linear layers...", end=" ", flush=True)
for size in [(1024,1024), (4096,1024), (1024,4096)]:
l = nn.Linear(*size).to(device, dtype)
x = torch.randn(1, 64, size[0], device=device, dtype=dtype)
y = l(x)
del l, x, y
torch.cuda.synchronize()
print(f"{time.time()-t0:.1f}s")
# 2. Attention (SDPA - the core of DiT)
print(" Scaled dot-product attention...", end=" ", flush=True)
t1 = time.time()
for heads in [8, 16, 24]:
q = torch.randn(1, heads, 64, 64, device=device, dtype=dtype)
k = torch.randn(1, heads, 64, 64, device=device, dtype=dtype)
v = torch.randn(1, heads, 64, 64, device=device, dtype=dtype)
y = torch.nn.functional.scaled_dot_product_attention(q, k, v)
del q, k, v, y
torch.cuda.synchronize()
print(f"{time.time()-t1:.1f}s")
# 3. LayerNorm / RMSNorm
print(" Normalization layers...", end=" ", flush=True)
t1 = time.time()
for dim in [1024, 2048, 4096]:
ln = nn.LayerNorm(dim).to(device, dtype)
x = torch.randn(1, 64, dim, device=device, dtype=dtype)
y = ln(x)
del ln, x, y
torch.cuda.synchronize()
print(f"{time.time()-t1:.1f}s")
# 4. Conv2d (VAE-like, but we'll run VAE on CPU)
print(" Conv2d layers...", end=" ", flush=True)
t1 = time.time()
for ch in [64, 128, 256]:
c = nn.Conv2d(ch, ch, 3, padding=1).to(device, dtype)
x = torch.randn(1, ch, 32, 32, device=device, dtype=dtype)
y = c(x)
del c, x, y
torch.cuda.synchronize()
print(f"{time.time()-t1:.1f}s")
# 5. Full mini-DiT forward pass simulation
print(" Mini-DiT forward pass simulation...", end=" ", flush=True)
t1 = time.time()
hidden = 1024
seq_len = 256
heads = 16
head_dim = hidden // heads
# Simulate a DiT block
x = torch.randn(1, seq_len, hidden, device=device, dtype=dtype)
norm = nn.LayerNorm(hidden).to(device, dtype)
qkv = nn.Linear(hidden, hidden*3).to(device, dtype)
proj = nn.Linear(hidden, hidden).to(device, dtype)
ff1 = nn.Linear(hidden, hidden*4).to(device, dtype)
ff2 = nn.Linear(hidden*4, hidden).to(device, dtype)
for step in range(3):
h = norm(x)
q, k, v = qkv(h).chunk(3, dim=-1)
q = q.view(1, seq_len, heads, head_dim).transpose(1,2)
k = k.view(1, seq_len, heads, head_dim).transpose(1,2)
v = v.view(1, seq_len, heads, head_dim).transpose(1,2)
attn = torch.nn.functional.scaled_dot_product_attention(q, k, v)
attn = attn.transpose(1,2).contiguous().view(1, seq_len, hidden)
x = x + proj(attn)
x = x + ff2(torch.nn.functional.gelu(ff1(norm(x))))
torch.cuda.synchronize()
print(f"{time.time()-t1:.1f}s")
total = time.time() - t0
print(f"\nGPU kernel warmup complete in {total:.1f}s")
print(f"VRAM used: {torch.cuda.memory_allocated()//1048576} MB")
torch.cuda.empty_cache()
print(f"VRAM after cleanup: {torch.cuda.memory_allocated()//1048576} MB")
print("WARMUP_DONE")
"""
# Write warmup script
run(f"cat > /tmp/gpu_warmup.py << 'PYEOF'\n{warmup_code}\nPYEOF")
out, err = run("bash -c 'source ~/comfyui-env/bin/activate && "
"HSA_OVERRIDE_GFX_VERSION=10.1.0 HSA_ENABLE_SDMA=0 "
"MIOPEN_FIND_MODE=3 MIOPEN_FIND_ENFORCE=3 "
"python3 /tmp/gpu_warmup.py' 2>&1", timeout=300)
print(out.strip())
if 'WARMUP_DONE' not in out:
print(f" WARNING: Warmup may have failed")
print(f" STDERR: {err.strip()[:500]}")
# 3. Update startup script with MIOpen settings
print("\n=== Update startup script ===")
script = """#!/bin/bash
# BC-250 ComfyUI Launcher — GPU (ROCm) + CPU VAE
# GPU identity (Cyan Skillfish gfx1013 -> gfx1010)
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# MIOpen: fast kernel selection (avoid long auto-tune on first run)
export MIOPEN_FIND_MODE=3
export MIOPEN_FIND_ENFORCE=3
# Use all 12 CPU cores for CPU-side work (dequant, text encoding)
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
# Activate venv
source /home/fabian/comfyui-env/bin/activate
cd /home/fabian/ComfyUI
# --novram: offload models to RAM, send layers to GPU one at a time
# --force-fp16: fp16 diffusion to halve VRAM usage
# --cpu-vae: VAE decode on CPU (GPU hangs on full VAE forward pass)
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--novram \\
--force-fp16 \\
--cpu-vae
"""
run(f"cat > /home/fabian/start_comfyui.sh << 'HEREDOC_END'\n{script}HEREDOC_END\n"
f"chmod +x /home/fabian/start_comfyui.sh")
print(" Written with MIOpen fast-find + --novram --force-fp16 --cpu-vae")
# 4. Launch ComfyUI
print("\n=== Launch ComfyUI ===")
run("nohup /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 &")
time.sleep(2)
print(" Waiting for server...")
for i in range(50):
time.sleep(3)
out, _ = run("curl -s -o /dev/null -w '%{http_code}' http://localhost:8188/ 2>/dev/null || echo 0")
if out.strip() == '200':
print(f" Server ready ({(i+1)*3}s)")
break
if i % 5 == 4:
log, _ = run("tail -2 /home/fabian/comfyui.log 2>/dev/null")
last = [l.strip() for l in log.strip().split('\n') if l.strip()]
print(f" [{(i+1)*3}s] ... {last[-1][:80] if last else ''}")
else:
print(" Timeout!")
out, _ = run("tail -40 /home/fabian/comfyui.log")
print(out)
sys.exit(1)
# 5. Submit workflow
print("\n=== Submit workflow (512x512, 8 steps) ===")
workflow = {
"prompt": {
"1": {"class_type": "UnetLoaderGGUF",
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF",
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader",
"inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode",
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "CLIPTextEncode",
"inputs": {"text": "", "clip": ["2", 0]}},
"6": {"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
"7": {"class_type": "KSampler",
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple",
"positive": ["4", 0], "negative": ["5", 0],
"latent_image": ["6", 0], "denoise": 1.0}},
"8": {"class_type": "VAEDecode",
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
"9": {"class_type": "SaveImage",
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
}
}
wf_json = json.dumps(workflow)
# Write workflow to file to avoid shell escaping issues
run(f"cat > /tmp/zimage_wf.json << 'JSONEOF'\n{wf_json}\nJSONEOF")
out, _ = run("curl -s -X POST http://localhost:8188/prompt "
"-H 'Content-Type: application/json' "
"-d @/tmp/zimage_wf.json")
try:
resp = json.loads(out.strip())
if 'error' in resp:
print(f" API ERROR: {resp['error']}")
sys.exit(1)
print(f" Prompt ID: {resp.get('prompt_id')}")
except:
print(f" Response: {out.strip()[:500]}")
# 6. Monitor — wait up to 15 minutes (first run can be slow due to kernel cache)
print("\n=== Monitoring generation (GPU kernels may compile on first step) ===")
last_log = ""
for i in range(60): # up to 15 minutes
time.sleep(15)
stats, _ = run("bash -c '"
"PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" CPU=$(ps -p $PID -o %cpu --no-headers); "
" RSS=$(ps -p $PID -o rss --no-headers); "
" LOAD=$(cat /proc/loadavg | cut -d\" \" -f1); "
" GPU_T=$(cat /sys/class/drm/card1/device/hwmon/hwmon*/temp1_input 2>/dev/null || echo 0); "
" echo \"CPU:${CPU}% RSS:$((RSS/1024))M LOAD:${LOAD} GPU:$((GPU_T/1000))C\"; "
"else echo DEAD; fi'")
log, _ = run("tail -12 /home/fabian/comfyui.log 2>/dev/null")
log_s = log.strip()
elapsed = (i+1) * 15
m, s = divmod(elapsed, 60)
stats_s = stats.strip()
print(f" [{m}m{s:02d}s] {stats_s}")
# Show new log content
if log_s != last_log:
for line in reversed(log_s.split('\n')):
l = line.strip()
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
print(f" LOG: {l[:120]}")
break
last_log = log_s
if 'DEAD' in stats_s:
print("\n PROCESS DIED!")
out, _ = run("tail -60 /home/fabian/comfyui.log")
print(out)
break
if 'Prompt executed in' in log_s:
print(f"\n SUCCESS! Image generated!")
out, _ = run("tail -25 /home/fabian/comfyui.log")
print(out)
break
if 'Traceback' in log_s or 'RuntimeError' in log_s:
print("\n ERROR detected!")
out, _ = run("tail -60 /home/fabian/comfyui.log")
print(out)
break
# 7. Check output
print("\n=== Output files ===")
out, _ = run("ls -lah ~/ComfyUI/output/ 2>/dev/null")
print(out.strip())
# 8. Check history
out, _ = run("curl -s http://localhost:8188/history 2>/dev/null")
try:
h = json.loads(out)
for pid, info in h.items():
status = info.get('status', {})
outputs = info.get('outputs', {})
print(f"\n Prompt {pid[:12]}...: status={status}")
if outputs:
for nid, nout in outputs.items():
if isinstance(nout, dict) and 'images' in nout:
for img in nout['images']:
print(f" Image: {img.get('filename', 'unknown')}")
except:
pass
finally:
ssh.close()
print("\nSSH connection closed.")
ComfyUI Scripts/bc250_monitor.py
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Monitor ComfyUI generation progress - poll every 20s."""
import paramiko, json, time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=30):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
return stdout.read().decode()
last_log_hash = ""
for i in range(90): # up to 30 minutes
time.sleep(20)
# Process stats
stats = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" CPU=$(ps -p $PID -o %cpu --no-headers); "
" MEM=$(ps -p $PID -o rss --no-headers); "
" echo \"CPU:${CPU}% RSS:$((MEM/1024))MB LOAD:$(cat /proc/loadavg | cut -d\" \" -f1-3)\"; "
"else echo DEAD; fi'").strip()
# GPU
gpu = run("bash -c 'rocm-smi --showuse --showmemuse 2>/dev/null | grep -E \"GPU|%\" | head -5 || echo no-gpu'").strip()
# Log tail
log = run("tail -10 /home/fabian/comfyui.log 2>/dev/null").strip()
log_hash = hash(log)
elapsed = (i+1) * 20
mins = elapsed // 60
secs = elapsed % 60
print(f"[{mins}m{secs:02d}s] {stats}")
# Show GPU line
for line in gpu.split('\n'):
if '%' in line or 'GPU' in line:
print(f" GPU: {line.strip()}")
break
# Show last meaningful log line
if log_hash != last_log_hash:
for line in reversed(log.split('\n')):
l = line.strip()
if l and not l.startswith('FETCH'):
print(f" LOG: {l}")
break
last_log_hash = log_hash
if 'DEAD' in stats:
print("\nPROCESS DIED!")
print(run("tail -60 /home/fabian/comfyui.log 2>/dev/null"))
break
if 'Prompt executed in' in log:
print(f"\nSUCCESS! Image generated!")
print(run("tail -30 /home/fabian/comfyui.log 2>/dev/null"))
print("\n=== OUTPUT FILES ===")
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null"))
break
if 'Traceback' in log or 'CUDA out of memory' in log or 'RuntimeError' in log:
print(f"\nERROR!")
print(run("tail -60 /home/fabian/comfyui.log 2>/dev/null"))
break
else:
print("\nTimed out after 30 minutes")
print(run("tail -40 /home/fabian/comfyui.log 2>/dev/null"))
ssh.close()
ComfyUI Scripts/bc250_monitor2.py
+58
View File
@@ -0,0 +1,58 @@
import paramiko, time, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
print("Connected. Monitoring sampling progress...")
for i in range(40): # up to 10 minutes
# Check log for sampling progress
_, o, _ = c.exec_command('tail -5 /tmp/comfyui.log 2>/dev/null')
log = o.read().decode(errors='replace').strip()
# Check GPU temp
_, o, _ = c.exec_command('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null')
temp = o.read().decode().strip()
temp_c = int(temp) // 1000 if temp.isdigit() else '?'
# Check GPU usage
_, o, _ = c.exec_command('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null')
gpu_pct = o.read().decode().strip()
# Check output dir for generated images
_, o, _ = c.exec_command('ls ~/ComfyUI/output/*.png 2>/dev/null')
files = o.read().decode().strip()
# Check queue
_, o, _ = c.exec_command('curl -s http://127.0.0.1:8188/queue 2>/dev/null')
queue_raw = o.read().decode().strip()
# Parse last log line for progress
last_line = log.split('\n')[-1] if log else ''
print(f"[{i*15:>3}s] GPU:{gpu_pct}% temp:{temp_c}C | {last_line[-120:]}")
if files:
print(f"\n*** IMAGE GENERATED! ***")
print(f"Files: {files}")
# Get last 10 lines of log for timing info
_, o, _ = c.exec_command('tail -10 /tmp/comfyui.log 2>/dev/null')
print(o.read().decode(errors='replace'))
break
try:
qdata = json.loads(queue_raw)
running = len(qdata.get('queue_running', []))
pending = len(qdata.get('queue_pending', []))
if running == 0 and pending == 0 and i > 3:
print("\nQueue empty - job finished or failed. Last 30 log lines:")
_, o, _ = c.exec_command('tail -30 /tmp/comfyui.log 2>/dev/null')
print(o.read().decode(errors='replace'))
break
except:
pass
time.sleep(15)
c.close()
print("Monitor done.")
ComfyUI Scripts/bc250_novram.py
+227
View File
@@ -0,0 +1,227 @@
"""Fix: Back to --novram (proven working for GPU sampling) + --cpu-vae."""
import paramiko, time, json, textwrap
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
def sh(cmd, timeout=60):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
# Use bash array to avoid quoting issues
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
def sftp_write(path, content):
sftp = c.open_sftp()
with sftp.open(path, 'w') as f:
f.write(content)
sftp.close()
def sftp_read(path):
sftp = c.open_sftp()
with sftp.open(path, 'r') as f:
data = f.read().decode(errors='replace')
sftp.close()
return data
# ---- STEP 1: Kill ----
print("STEP 1: Kill ComfyUI")
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
print(" Killed.")
# ---- STEP 2: Write launcher ----
print("\nSTEP 2: Write launcher with --novram (PROVEN to work on this APU)")
launcher = textwrap.dedent("""\
#!/bin/bash
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
export MIOPEN_FIND_MODE=1
cd ~/ComfyUI
source ~/comfyui-env/bin/activate
# --novram = model weights on CPU, GPU only for compute (correct for shared-memory APU)
# --cpu-vae = VAE decode on CPU (fixes known hang on this GPU)
# --force-fp16 = half precision to save memory
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--novram \\
--force-fp16 \\
--cpu-vae \\
--disable-smart-memory
""")
sftp_write('/tmp/run_comfyui.sh', launcher)
sh('chmod +x /tmp/run_comfyui.sh')
print(" Written: --novram --force-fp16 --cpu-vae --disable-smart-memory")
# ---- STEP 3: Start ----
print("\nSTEP 3: Start ComfyUI")
sh('rm -f /tmp/comfyui.log; touch /tmp/comfyui.log')
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(3)
pid = sh('pgrep -f "python3.*main.py"')
if not pid:
print(" FAILED!")
print(sftp_read('/tmp/comfyui.log'))
c.close()
exit(1)
print(f" PID: {pid}")
# ---- STEP 4: Wait for HTTP 200 ----
print("\nSTEP 4: Wait for server ready", end='', flush=True)
for i in range(120):
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
if '200' in code:
print(f" READY ({i*2}s)")
break
# Check if process died
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
if alive == 'N':
print("\n Process died!")
print(sftp_read('/tmp/comfyui.log'))
c.close()
exit(1)
if i % 10 == 0 and i > 0:
log = sftp_read('/tmp/comfyui.log')
lines = [l for l in log.split('\n') if l.strip()]
print(f"\n [{i*2}s] {lines[-1][:80] if lines else '...'}", end='', flush=True)
else:
print('.', end='', flush=True)
time.sleep(2)
else:
print("\n TIMEOUT!")
print(sftp_read('/tmp/comfyui.log')[-1000:])
c.close()
exit(1)
# Verify startup flags
log = sftp_read('/tmp/comfyui.log')
if 'NO_VRAM' in log or 'NOVRAM' in log.upper():
print(" Confirmed: NOVRAM mode (GPU compute only, model on CPU)")
for line in log.split('\n'):
if 'vram state' in line.lower():
print(f" {line.strip()}")
if 'Device:' in line:
print(f" {line.strip()}")
# ---- STEP 5: Submit workflow ----
print("\nSTEP 5: Submit workflow")
workflow = {
"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 42, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}
}
sftp_write('/tmp/wf.json', json.dumps(workflow))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json', timeout=10)
print(f" Response: {resp[:200]}")
if 'prompt_id' not in resp:
print(" FAILED!")
c.close()
exit(1)
prompt_id = json.loads(resp).get('prompt_id', '?')
print(f" Prompt ID: {prompt_id}")
# ---- STEP 6: Monitor ----
print("\nSTEP 6: Monitor (expect GPU power >100W during sampling)")
t0 = time.time()
sampling_seen = False
for i in range(200):
elapsed = int(time.time() - t0)
gpu_temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
# Read log via SFTP to avoid shell issues
try:
log = sftp_read('/tmp/comfyui.log')
except:
log = ''
lines = log.strip().split('\n')
# Find last meaningful line (skip manager spam)
last = ''
for line in reversed(lines):
if 'FETCH ComfyRegistry' not in line and 'All startup tasks' not in line and 'FETCH DATA' not in line and line.strip():
last = line.strip()
break
# Detect sampling progress
for line in lines:
if '/8' in line and 'it/s' in line:
sampling_seen = True
print(f" [{elapsed:>4}s] {temp_c}C | {last[-100:]}")
# Check for output image
imgs = sh('ls ~/ComfyUI/output/*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** IMAGE GENERATED! ***")
print(f" File: {imgs}")
print(f" Total: {elapsed}s")
# Show last 15 lines
for line in lines[-15:]:
if line.strip():
print(f" {line.strip()}")
break
# Check queue
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 30:
time.sleep(3)
imgs = sh('ls ~/ComfyUI/output/*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** IMAGE GENERATED! ***")
print(f" File: {imgs}")
else:
print(f"\n Queue empty, no image. Error in log:")
for line in lines[-20:]:
if line.strip():
print(f" {line}")
break
except:
pass
# Check process alive
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
if alive == 'N':
print(f"\n *** CRASHED ***")
for line in lines[-30:]:
if line.strip():
print(f" {line}")
break
time.sleep(15)
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_patch_read.py
+76
View File
@@ -0,0 +1,76 @@
"""Patch ComfyUI: Force VAE to GPU even in --novram mode. Restart and test."""
import paramiko, time, json, textwrap
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
sftp = c.open_sftp()
def sh(cmd, timeout=60):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# =============================================
# STEP 1: Kill ComfyUI
# =============================================
print("1) Kill ComfyUI")
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
# =============================================
# STEP 2: Read and patch model_management.py
# =============================================
print("2) Patch model_management.py — force VAE to GPU")
# First, read the file to understand the structure
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
mgmt = f.read().decode()
print(f" File size: {len(mgmt)} bytes")
# Find the vae_offload_device function
# In ComfyUI, with NO_VRAM, vae_offload_device() returns CPU
# We need to make it return GPU instead
# Also find vae_dtype — it's set to float32 by default, we want float16
# Let's search for relevant functions
for i, line in enumerate(mgmt.split('\n')):
if 'def vae_offload_device' in line or 'def vae_dtype' in line or 'def vae_device' in line:
print(f" Line {i+1}: {line.strip()}")
# Also check what functions exist
found = []
for i, line in enumerate(mgmt.split('\n')):
if line.startswith('def ') or (line.startswith(' ') and 'def ' in line[:12]):
if 'vae' in line.lower():
found.append((i+1, line.strip()))
for ln, l in found:
print(f" L{ln}: {l}")
# Let's read the specific area around these functions
lines = mgmt.split('\n')
# Find and show context around vae functions
for keyword in ['vae_offload_device', 'vae_dtype', 'vae_device']:
for i, line in enumerate(lines):
if f'def {keyword}' in line:
start = max(0, i-2)
end = min(len(lines), i+15)
print(f"\n --- {keyword} (L{i+1}) ---")
for j in range(start, end):
print(f" {j+1:>5}: {lines[j]}")
sftp.close()
c.close()
print("\n Reading complete. Will patch next.")
ComfyUI Scripts/bc250_preload.py
+242
View File
@@ -0,0 +1,242 @@
"""Fix: keep UNet+VAE both on GPU (shared memory). No offloading."""
import paramiko, time, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, timeout=30):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# Kill first
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
print("Killed ComfyUI")
# Read model_management.py
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
code = f.read().decode()
lines = code.split('\n')
# Show current offload functions to understand exact code
print("\n=== Finding offload functions ===")
for i, line in enumerate(lines):
if 'def unet_offload_device' in line or 'def vae_offload_device' in line:
print(f"\n--- {line.strip()} at line {i+1} ---")
for j in range(i, min(i+10, len(lines))):
print(f" {j+1}: {lines[j]}")
# ============ PATCH unet_offload_device ============
# Current: returns CPU unless HIGH_VRAM
# Fix: also return GPU for SHARED (APU shared memory = no point offloading)
old_unet = None
new_unet = None
for i, line in enumerate(lines):
if 'def unet_offload_device' in line:
# Grab the function body (next ~6 lines)
chunk = '\n'.join(lines[i:i+8])
print(f"\n=== unet_offload_device chunk ===\n{chunk}")
# The function checks HIGH_VRAM only. Add SHARED.
if 'HIGH_VRAM' in chunk and 'SHARED' not in chunk:
old_unet = chunk
new_unet = chunk.replace(
'vram_state == VRAMState.HIGH_VRAM',
'vram_state == VRAMState.HIGH_VRAM or vram_state == VRAMState.SHARED'
)
print(f"\n -> Will patch to include SHARED")
elif 'SHARED' in chunk:
print(f"\n -> Already patched for SHARED")
break
# ============ PATCH vae_offload_device ============
# Current: returns CPU unless --gpu-only
# Fix: also return GPU for SHARED
old_vae = None
new_vae = None
for i, line in enumerate(lines):
if 'def vae_offload_device' in line:
chunk = '\n'.join(lines[i:i+8])
print(f"\n=== vae_offload_device chunk ===\n{chunk}")
if 'args.gpu_only' in chunk and 'SHARED' not in chunk:
old_vae = chunk
new_vae = chunk.replace(
'args.gpu_only',
'args.gpu_only or vram_state == VRAMState.SHARED'
)
print(f"\n -> Will patch to include SHARED")
elif 'SHARED' in chunk:
print(f"\n -> Already patched for SHARED")
break
# Also check text_encoder_offload_device
for i, line in enumerate(lines):
if 'def text_encoder_offload_device' in line:
chunk = '\n'.join(lines[i:i+8])
print(f"\n=== text_encoder_offload_device chunk ===\n{chunk}")
break
# Apply patches
patched = False
if old_unet and new_unet:
code = code.replace(old_unet, new_unet)
patched = True
print("\n[OK] Patched unet_offload_device")
if old_vae and new_vae:
code = code.replace(old_vae, new_vae)
patched = True
print("[OK] Patched vae_offload_device")
if patched:
# Backup and write
sh('cp /home/fabian/ComfyUI/comfy/model_management.py /home/fabian/ComfyUI/comfy/model_management.py.bak2')
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
f.write(code)
print("[OK] Written to disk")
else:
print("[INFO] No patches needed (already applied or code changed)")
# Verify
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
verify = f.read().decode()
for i, line in enumerate(verify.split('\n')):
if 'def unet_offload_device' in line or 'def vae_offload_device' in line:
print(f"\n--- VERIFY {line.strip()} ---")
for j in range(i, min(i+8, len(verify.split(chr(10))))):
print(f" {j+1}: {verify.split(chr(10))[j]}")
# ============ RESTART ============
print("\n=== RESTARTING ===")
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(3)
pid = sh('pgrep -f "python3.*main.py"')
print(f"PID: {pid}")
# Wait for ready
print("Waiting for HTTP", end='', flush=True)
for i in range(90):
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
if '200' in r:
print(f" OK ({i*2}s)")
break
print('.', end='', flush=True)
time.sleep(2)
else:
print(" TIMEOUT")
# Check SHARED mode active
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n'):
s = line.strip()
if any(x in s.lower() for x in ['vram state', 'shared', 'device:', 'total vram']):
print(f" {s}")
# Submit workflow
print("\nSubmitting workflow...")
wf = {"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(wf))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f" {resp[:150]}")
# Monitor - watch for UNet+VAE both on GPU, fast VAE
print("\nMonitoring...")
t0 = time.time()
for i in range(120):
el = int(time.time() - t0)
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
# GPU usage
gpu = sh('cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null || echo ?', timeout=5)
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
tc = int(temp)//1000 if temp.isdigit() else '?'
# Get latest progress line
last_progress = ''
last_line = ''
for line in log.split('\n'):
s = line.strip()
if '/8' in s and ('it/s' in s or 's/it' in s): last_progress = s
if 'loaded' in s.lower() or 'VAE' in s or 'Requested' in s or 'Prompt executed' in s:
last_line = s
if s and 'FETCH' not in s: last_line = s
status = last_progress or last_line
print(f" [{el:>3}s] GPU:{gpu}% {tc}C | {status[-100:]}")
# Check for output
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** IMAGE DONE! *** {imgs}")
print(f" Wall time: {el}s")
# Print key log lines
for line in log.split('\n'):
s = line.strip()
if any(x in s for x in ['loaded', 'load device', 'offload device', 'Prompt executed', '/8', 'Requested', 'VAE']):
if 'FETCH' not in s:
print(f" {s}")
break
# Check queue empty
if el > 30:
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending'):
time.sleep(3)
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** DONE: {imgs} ***")
else:
print(f"\n Queue empty, no image. Error?")
for line in log.split('\n')[-20:]:
if line.strip() and 'FETCH' not in line: print(f" {line.strip()}")
break
except: pass
# Check alive
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
print("\n CRASHED!")
for line in log.split('\n')[-20:]:
if line.strip(): print(f" {line.strip()}")
break
time.sleep(5)
sftp.close()
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_q.py
+31
View File
@@ -0,0 +1,31 @@
import paramiko
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
# Full log
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n'):
s = line.strip()
if not s or 'FETCH' in s or 'startup tasks' in s or 'DEPRECATION' in s: continue
print(s)
# Process + GPU
chan = c.get_transport().open_session()
chan.settimeout(10)
chan.exec_command('/bin/bash -c "echo === PROC ===; ps aux | grep main.py | grep -v grep; echo === GPU ===; rocm-smi 2>/dev/null | head -12; echo === VRAM ===; rocm-smi --showmeminfo vram 2>/dev/null"')
o = b""
while True:
try:
ch = chan.recv(65536)
if not ch: break
o += ch
except: break
chan.close()
print(o.decode(errors='replace'))
sftp.close()
c.close()
ComfyUI Scripts/bc250_quick.py
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env python3
"""Quick status check."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd):
_, so, se = ssh.exec_command(cmd, timeout=15)
return so.read().decode()
print("=== PROCESS ===")
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); ps -p $PID -o pid,%cpu,%mem,nlwp --no-headers 2>/dev/null; echo LOAD: $(cat /proc/loadavg)'"))
print("=== GPU ===")
print(run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null | tail -8"))
print("=== LOG (last 15) ===")
print(run("tail -15 /home/fabian/comfyui.log 2>/dev/null"))
print("=== MEMORY ===")
print(run("free -h"))
ssh.close()
ComfyUI Scripts/bc250_rawlog.py
+47
View File
@@ -0,0 +1,47 @@
"""Read raw ComfyUI log - no filtering, no quoting issues."""
import paramiko
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
# Read the ENTIRE log via SFTP - no shell, no grep, no quoting
sftp = c.open_sftp()
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
lines = log.split('\n')
print(f"Total log lines: {len(lines)}")
print()
# Print everything that's NOT ComfyUI-Manager registry spam
for line in lines:
if 'FETCH ComfyRegistry' in line:
continue
if 'All startup tasks' in line:
continue
if line.strip():
print(line)
except Exception as e:
print(f"Error reading log: {e}")
finally:
sftp.close()
# Also check: is the process actually using GPU memory?
chan = c.get_transport().open_session()
chan.settimeout(10)
chan.exec_command('/bin/bash -c "rocm-smi --showmeminfo vram 2>/dev/null"')
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
print("\n=== VRAM Info ===")
print(out.decode(errors='replace'))
c.close()
ComfyUI Scripts/bc250_read2.py
+64
View File
@@ -0,0 +1,64 @@
"""Read more of model_management.py — find how --novram affects GPU compute."""
import paramiko
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
mgmt = f.read().decode()
lines = mgmt.split('\n')
# Find all functions related to device/offload
print("=== KEY FUNCTIONS ===")
for keyword in ['unet_offload_device', 'unet_device', 'NO_VRAM', 'should_use', 'get_torch_device',
'def text_encoder_device', 'def text_encoder_offload', 'VRAMState']:
for i, line in enumerate(lines):
if keyword in line and ('def ' in line or 'class ' in line or '=' in line[:50]):
print(f" L{i+1}: {line.strip()[:100]}")
# Show VRAMState enum
print("\n=== VRAMState ===")
for i, line in enumerate(lines):
if 'class VRAMState' in line or (i > 0 and 'VRAMState' in lines[i-1] and 'class' in lines[i-1]):
for j in range(i, min(i+15, len(lines))):
print(f" {j+1}: {lines[j]}")
break
# Show unet_offload_device
print("\n=== unet_offload_device ===")
for i, line in enumerate(lines):
if 'def unet_offload_device' in line:
for j in range(max(0,i-2), min(i+15, len(lines))):
print(f" {j+1}: {lines[j]}")
# Show text_encoder functions
print("\n=== text_encoder_device ===")
for i, line in enumerate(lines):
if 'def text_encoder_device' in line:
for j in range(max(0,i-2), min(i+12, len(lines))):
print(f" {j+1}: {lines[j]}")
print("\n=== text_encoder_offload_device ===")
for i, line in enumerate(lines):
if 'def text_encoder_offload_device' in line:
for j in range(max(0,i-2), min(i+12, len(lines))):
print(f" {j+1}: {lines[j]}")
# Show how NO_VRAM is used in loading logic
print("\n=== NO_VRAM usage in model loading ===")
for i, line in enumerate(lines):
if 'NO_VRAM' in line:
print(f" L{i+1}: {line.strip()[:120]}")
# Show the VRAM state setting logic
print("\n=== vram_state assignment ===")
for i, line in enumerate(lines):
if 'vram_state' in line and ('=' in line) and 'VRAMState' in line:
print(f" L{i+1}: {line.strip()[:120]}")
sftp.close()
c.close()
ComfyUI Scripts/bc250_read3.py
+43
View File
@@ -0,0 +1,43 @@
"""Read the SHARED vram state logic and CLI args."""
import paramiko
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
# Read model_management.py around line 440-470 (where SHARED is set)
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
mgmt = f.read().decode()
lines = mgmt.split('\n')
print("=== L430-475: VRAM state setting ===")
for i in range(429, min(475, len(lines))):
print(f" {i+1}: {lines[i]}")
print("\n=== L750-790: NO_VRAM model loading ===")
for i in range(749, min(790, len(lines))):
print(f" {i+1}: {lines[i]}")
print("\n=== L850-870: Smart memory / offload ===")
for i in range(849, min(870, len(lines))):
print(f" {i+1}: {lines[i]}")
# Check CLI args for shared memory
with sftp.open('/home/fabian/ComfyUI/comfy/cli_args.py', 'r') as f:
cli = f.read().decode()
print("\n=== CLI args with 'shared' or 'SHARED' ===")
for i, line in enumerate(cli.split('\n')):
if 'shared' in line.lower():
print(f" L{i+1}: {line.strip()}")
# Check what --gpu-only does
print("\n=== CLI args with 'gpu_only' ===")
for i, line in enumerate(cli.split('\n')):
if 'gpu_only' in line.lower() or 'gpu-only' in line.lower():
print(f" L{i+1}: {line.strip()}")
sftp.close()
c.close()
ComfyUI Scripts/bc250_read_offload.py
+55
View File
@@ -0,0 +1,55 @@
"""Reconnect, check patches, fix missing ones, restart."""
import paramiko, time, json
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, t=30):
ch = c.get_transport().open_session()
ch.settimeout(t)
ch.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
o = b""
while True:
try:
d = ch.recv(65536)
if not d: break
o += d
except: break
ch.close()
return o.decode(errors='replace').strip()
# Read current state
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
code = f.read().decode()
lines = code.split('\n')
# Show ALL offload functions
for fname in ['unet_offload_device', 'vae_offload_device', 'text_encoder_offload_device']:
for i, line in enumerate(lines):
if f'def {fname}' in line:
print(f"\n=== {fname} (L{i+1}) ===")
for j in range(i, min(i+10, len(lines))):
print(f" {j+1}: {lines[j]}")
break
# Also show unet_inital_load_device
for i, line in enumerate(lines):
if 'def unet_inital_load_device' in line:
print(f"\n=== unet_inital_load_device (L{i+1}) ===")
for j in range(i, min(i+10, len(lines))):
print(f" {j+1}: {lines[j]}")
break
# Show SHARED patch
for i, line in enumerate(lines):
if 'COMFYUI_SHARED_MEMORY' in line:
print(f"\n=== SHARED patch (L{i+1}) ===")
for j in range(max(0,i-2), min(i+5, len(lines))):
print(f" {j+1}: {lines[j]}")
sftp.close()
c.close()
print("\nDone reading.")
ComfyUI Scripts/bc250_real_gpu.py
+203
View File
@@ -0,0 +1,203 @@
"""Fix: Remove --novram so ComfyUI actually uses the GPU for compute."""
import paramiko, time, json, textwrap
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
sftp = c.open_sftp()
def sh(cmd, timeout=60):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# STEP 1: Kill
print("STEP 1: Kill")
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
print(" Done")
# STEP 2: Write new launcher - NO --novram, NO --lowvram
# ComfyUI sees 7602MB VRAM → will use NORMAL_VRAM mode → GPU compute
print("\nSTEP 2: New launcher (NO memory flags = auto GPU)")
launcher = textwrap.dedent("""\
#!/bin/bash
# GPU
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
# Threading
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
# MIOpen - use fast mode, persistent cache
export MIOPEN_FIND_MODE=3
export MIOPEN_LOG_LEVEL=3
cd ~/ComfyUI
source ~/comfyui-env/bin/activate
# NO --novram, NO --lowvram = ComfyUI auto-detects 7602MB VRAM = GPU compute
# --force-fp16 = half precision saves memory
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--force-fp16
""")
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
f.write(launcher)
sh('chmod +x /tmp/run_comfyui.sh')
print(" Flags: --force-fp16 ONLY (auto VRAM mode)")
# STEP 3: Start
print("\nSTEP 3: Start ComfyUI")
sh('rm -f /tmp/comfyui.log; touch /tmp/comfyui.log')
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(3)
pid = sh('pgrep -f "python3.*main.py"')
print(f" PID: {pid}")
# STEP 4: Wait for ready
print("\nSTEP 4: Wait for HTTP ready", end='', flush=True)
for i in range(120):
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
if '200' in code:
print(f" READY ({i*2}s)")
break
if i % 10 == 0 and i > 0:
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
lines = [l.strip() for l in log.split('\n') if l.strip() and 'FETCH' not in l and 'DEPRECATION' not in l]
print(f"\n [{i*2}s] {lines[-1][:80] if lines else '...'}", end='', flush=True)
except: pass
else:
print('.', end='', flush=True)
time.sleep(2)
else:
print("\n TIMEOUT!")
exit(1)
# Show VRAM state
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['vram state', 'Device:', 'Total VRAM', 'VRAM', 'pytorch version']):
print(f" {s}")
# STEP 5: Submit workflow
print("\nSTEP 5: Submit workflow")
workflow = {
"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {
"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 42, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0
}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}
}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(workflow))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json', timeout=10)
print(f" {resp[:150]}")
if 'prompt_id' not in resp:
print(" FAILED!")
exit(1)
# STEP 6: Monitor - focus on GPU usage and sampling speed
print("\nSTEP 6: Monitor")
print(" First step may be slow (MIOpen kernel compilation). Be patient.")
t0 = time.time()
for i in range(200):
elapsed = int(time.time() - t0)
gpu_temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
temp_c = int(gpu_temp) // 1000 if gpu_temp.isdigit() else '?'
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
# Find sampling progress and last meaningful line
sampling = ''
last = ''
for line in log.split('\n'):
s = line.strip()
if '/8' in s and ('it/s' in s or 's/it' in s):
sampling = s
if s and 'FETCH' not in s and 'startup tasks' not in s and 'DEPRECATION' not in s:
last = s
display = sampling if sampling else last[-100:]
print(f" [{elapsed:>4}s] {temp_c}C | {display}")
# Check output
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
# Get timing from log
exec_time = ''
for line in log.split('\n'):
if 'Prompt executed in' in line:
exec_time = line.strip()
print(f"\n *** IMAGE GENERATED! ***")
print(f" File: {imgs}")
print(f" {exec_time}")
print(f" Wall time: {elapsed}s")
# Show vram state and model loading details
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['loaded completely', 'loaded partially', 'vram state', '/8']):
print(f" {s}")
break
# Check queue empty
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending') and elapsed > 30:
time.sleep(3)
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** IMAGE GENERATED: {imgs} ***")
else:
print(f"\n Queue empty, no image. Log errors:")
for line in log.split('\n')[-25:]:
if line.strip(): print(f" {line.strip()}")
break
except: pass
# Process alive?
alive = sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5)
if alive == 'N':
print(f"\n *** CRASHED ***")
for line in log.split('\n')[-30:]:
if line.strip(): print(f" {line.strip()}")
break
time.sleep(15)
sftp.close()
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_reboot_go.py
+133
View File
@@ -0,0 +1,133 @@
"""Recreate launcher (lost on reboot) and start ComfyUI."""
import paramiko, time, json, sys
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, t=30):
stdin, stdout, stderr = c.exec_command(f"bash -lc '{cmd}'", timeout=t)
return stdout.read().decode(errors='replace').strip()
# Recreate launcher (wiped by reboot since /tmp)
launcher = """#!/bin/bash
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
export MIOPEN_FIND_MODE=3
export COMFYUI_SHARED_MEMORY=1
cd ~/ComfyUI
source ~/comfyui-env/bin/activate
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--force-fp16 \\
--fp16-vae
"""
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
f.write(launcher)
sh("chmod +x /tmp/run_comfyui.sh")
print("Launcher recreated")
# Verify patches
p = sh("grep -c COMFYUI_SHARED_MEMORY ~/ComfyUI/comfy/model_management.py")
print(f"SHARED patch refs: {p}")
p2 = sh("grep 'def unet_offload_device' -A2 ~/ComfyUI/comfy/model_management.py | head -3")
print(f"unet_offload: {p2}")
p3 = sh("grep 'def vae_offload_device' -A2 ~/ComfyUI/comfy/model_management.py | head -3")
print(f"vae_offload: {p3}")
# Start
sh("rm -f /tmp/comfyui.log")
sh("nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &")
time.sleep(4)
pid = sh("pgrep -f 'python3.*main.py'")
print(f"PID: {pid}")
if not pid:
print("FAILED! Log:")
print(sh("cat /tmp/comfyui.log 2>/dev/null"))
c.close()
sys.exit(1)
# Wait for HTTP
print("Waiting for HTTP...", end='', flush=True)
for i in range(90):
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null')
if '200' in code:
print(f" ready ({i*2}s)")
break
print('.', end='', flush=True)
time.sleep(2)
else:
print(" TIMEOUT")
print(sh("tail -30 /tmp/comfyui.log"))
c.close()
sys.exit(1)
# Log state
log = sh("cat /tmp/comfyui.log")
for l in log.split('\n'):
s = l.strip()
if any(x in s for x in ['vram state', 'SHARED', 'Device:', 'Total VRAM']):
print(f" {s}")
# Submit
wf = {"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(wf))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f"Submitted: {resp[:100]}")
# Monitor
t0 = time.time()
shown = set()
for _ in range(200):
el = int(time.time() - t0)
log = sh("cat /tmp/comfyui.log 2>/dev/null")
for l in log.split('\n'):
s = l.strip()
if s not in shown and any(x in s for x in ['/8', 'loaded completely', 'load device', 'offload device', 'Requested to load', 'Prompt executed', 'Error', 'OOM']):
if 'FETCH' not in s and 'audio' not in s and 'split attention' not in s:
gpu = sh("cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null")
print(f" [{el:>3}s] GPU:{gpu}% | {s[-110:]}")
shown.add(s)
if 'Prompt executed' in log:
print(f"\n*** DONE in {el}s! ***")
imgs = sh("ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null")
print(f" Images: {imgs}")
break
alive = sh("pgrep -c -f 'python3.*main.py' 2>/dev/null")
if alive == '0':
print(f"\nCRASHED at {el}s!")
for l in log.split('\n')[-20:]:
if l.strip(): print(f" {l.strip()}")
break
time.sleep(5)
sftp.close()
c.close()
ComfyUI Scripts/bc250_recon.py
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""Recon the BC-250 for PyTorch/ComfyUI installation."""
import paramiko, sys
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
cmds = [
("Python version", "python3 --version 2>&1"),
("pip version", "pip --version 2>&1 || pip3 --version 2>&1"),
("Disk space", "df -h / /home 2>&1"),
("RAM", "free -h 2>&1"),
("CPU cores", "nproc 2>&1"),
("ROCm version", "cat /opt/rocm/.info/version 2>/dev/null || echo 'no version file'; ls /opt/rocm/lib/libamdhip64.so* 2>&1"),
("hipcc", "which hipcc 2>&1 && hipcc --version 2>&1 | head -5"),
("rocminfo GPU", "HSA_OVERRIDE_GFX_VERSION=10.1.0 rocminfo 2>&1 | grep -E 'Marketing|gfx|Name:' | head -10"),
("Existing PyTorch", "python3 -c 'import torch; print(torch.__version__); print(torch.cuda.is_available()); print(torch.version.hip)' 2>&1"),
("Existing venvs", "ls -la ~/venv* ~/env* ~/.local/lib/python*/site-packages/torch* 2>&1 | head -20"),
("git version", "git --version 2>&1"),
("cmake version", "cmake --version 2>&1 | head -1"),
("ninja version", "ninja --version 2>&1"),
("Available Python packages", "python3 -m venv --help >/dev/null 2>&1 && echo 'venv OK' || echo 'venv missing'"),
("Swap", "swapon --show 2>&1"),
("GPU device check", "ls -la /dev/kfd /dev/dri/render* 2>&1"),
("Existing ComfyUI", "ls -la ~/ComfyUI 2>&1 || echo 'not found'"),
("pacman cmake/ninja", "pacman -Q cmake ninja 2>&1"),
]
for label, cmd in cmds:
print(f"\n{'='*60}")
print(f" {label}")
print(f"{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=30)
out = stdout.read().decode()
err = stderr.read().decode()
if out.strip():
print(out.strip())
if err.strip():
print(f"STDERR: {err.strip()}")
ssh.close()
print("\n\nDone.")
ComfyUI Scripts/bc250_recon2.py
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
"""Check Python versions and z-image-turbo info on BC-250."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
cmds = [
("Python 3.12 available?", "pacman -Ss python | grep -E 'python3\\.1[0-3]|python 3\\.' 2>&1 | head -20"),
("All python packages", "pacman -Q | grep python 2>&1 | head -30"),
("pip via python", "python3 -m pip --version 2>&1"),
("pip package", "pacman -Q python-pip 2>&1"),
("check pyenv", "which pyenv 2>&1; pacman -Q pyenv 2>&1"),
("check python3.12", "which python3.12 2>&1; pacman -Q python312 2>&1; ls /usr/bin/python3.1* 2>&1"),
("ninja available", "pacman -Ss '^ninja$' 2>&1 | head -5"),
("check ccache", "which ccache 2>&1; pacman -Q ccache 2>&1"),
("check z-image-turbo", "pacman -Ss z-image 2>&1; pip3 search z-image-turbo 2>&1 || true"),
("check huggingface tools", "pacman -Q | grep -i hugging 2>&1; python3 -c 'import huggingface_hub' 2>&1 || true"),
]
for label, cmd in cmds:
print(f"\n{'='*60}")
print(f" {label}")
print(f"{'='*60}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=30)
out = stdout.read().decode()
err = stderr.read().decode()
if out.strip():
print(out.strip())
if err.strip():
print(f"STDERR: {err.strip()}")
ssh.close()
ComfyUI Scripts/bc250_run.py
+93
View File
@@ -0,0 +1,93 @@
"""Post-reboot: start patched ComfyUI, submit, monitor. Single connection."""
import paramiko, time, json, sys
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
def sh(cmd, t=30):
stdin, stdout, stderr = c.exec_command(f"bash -lc '{cmd}'", timeout=t)
return stdout.read().decode(errors='replace').strip()
# Verify patches survived reboot
p = sh("grep -c SHARED ~/ComfyUI/comfy/model_management.py")
print(f"SHARED refs in code: {p}")
# Start ComfyUI
sh("rm -f /tmp/comfyui.log")
sh("nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &")
time.sleep(4)
pid = sh("pgrep -f main.py")
print(f"PID: {pid}")
# Wait for HTTP ready
print("Waiting for HTTP...", end='', flush=True)
for i in range(90):
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null')
if '200' in code:
print(f" ready ({i*2}s)")
break
print('.', end='', flush=True)
time.sleep(2)
else:
print(" TIMEOUT")
print(sh("tail -30 /tmp/comfyui.log"))
sys.exit(1)
# Show startup state
log = sh("cat /tmp/comfyui.log")
for l in log.split('\n'):
s = l.strip()
if any(x in s for x in ['vram state', 'SHARED', 'Device:', 'Total VRAM']):
print(f" {s}")
# Submit workflow
wf = json.dumps({"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 99999, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}})
# Write workflow and submit
sh(f"echo '{wf}' > /tmp/wf.json")
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f"Submitted: {resp[:100]}")
# Monitor
t0 = time.time()
shown = set()
for _ in range(200):
el = int(time.time() - t0)
log = sh("cat /tmp/comfyui.log 2>/dev/null")
for l in log.split('\n'):
s = l.strip()
if s not in shown and any(x in s for x in ['/8', 'loaded completely', 'load device', 'Requested to load', 'Prompt executed', 'Error', 'OOM']):
if 'FETCH' not in s and 'audio' not in s:
gpu = sh("cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null")
print(f" [{el:>3}s] GPU:{gpu}% | {s[-110:]}")
shown.add(s)
if 'Prompt executed' in log:
print(f"\n*** DONE in {el}s! ***")
imgs = sh("ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null")
print(f" Images: {imgs}")
break
alive = sh("pgrep -c -f main.py 2>/dev/null")
if alive == '0':
print(f"\nCRASHED at {el}s!")
for l in log.split('\n')[-15:]:
if l.strip(): print(f" {l.strip()}")
break
time.sleep(5)
c.close()
ComfyUI Scripts/bc250_shared.py
+279
View File
@@ -0,0 +1,279 @@
"""Patch ComfyUI for BC-250 APU: Use SHARED VRAM mode + force fp16 VAE.
This is the correct mode for an APU where CPU and GPU share the same physical memory."""
import paramiko, time, json, textwrap
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=15)
sftp = c.open_sftp()
def sh(cmd, timeout=60):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# ======================================
# 1) Kill
# ======================================
print("1) Kill ComfyUI")
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
# ======================================
# 2) Backup + Patch model_management.py
# ======================================
print("2) Patch model_management.py: SHARED mode for APU")
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
code = f.read().decode()
# Backup
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py.bak', 'w') as f:
f.write(code)
print(" Backup saved")
# PATCH 1: After MPS sets SHARED, also set SHARED for this AMD APU
# Current code (L458-462):
# if cpu_state != CPUState.GPU:
# vram_state = VRAMState.DISABLED
# if cpu_state == CPUState.MPS:
# vram_state = VRAMState.SHARED
#
# We add: if the GPU has shared memory (small dedicated VRAM), set SHARED
old_block = '''if cpu_state == CPUState.MPS:
vram_state = VRAMState.SHARED
logging.info(f"Set vram state to: {vram_state.name}")'''
new_block = '''if cpu_state == CPUState.MPS:
vram_state = VRAMState.SHARED
# BC-250 APU: shared memory between CPU and GPU. Dedicated VRAM is tiny (512MB)
# but the full system RAM is accessible to both. SHARED mode loads models
# directly on GPU (zero-copy for shared memory APUs).
if cpu_state == CPUState.GPU and vram_state not in (VRAMState.DISABLED, VRAMState.SHARED):
try:
import os
if os.environ.get("COMFYUI_SHARED_MEMORY") == "1":
vram_state = VRAMState.SHARED
logging.info("Forcing SHARED vram state (COMFYUI_SHARED_MEMORY=1)")
except:
pass
logging.info(f"Set vram state to: {vram_state.name}")'''
if old_block in code:
code = code.replace(old_block, new_block)
print(" PATCH 1 applied: COMFYUI_SHARED_MEMORY env var support")
else:
print(" PATCH 1: Could not find exact block, trying alternate...")
# Try line by line
lines = code.split('\n')
for i, line in enumerate(lines):
if 'cpu_state == CPUState.MPS' in line and 'SHARED' in lines[i+1] if i+1 < len(lines) else '':
# Insert after the MPS block
insert_idx = i + 2 # After "vram_state = VRAMState.SHARED"
patch_lines = [
'',
'# BC-250 APU shared memory support',
'if cpu_state == CPUState.GPU and vram_state not in (VRAMState.DISABLED, VRAMState.SHARED):',
' try:',
' import os',
' if os.environ.get("COMFYUI_SHARED_MEMORY") == "1":',
' vram_state = VRAMState.SHARED',
' logging.info("Forcing SHARED vram state (COMFYUI_SHARED_MEMORY=1)")',
' except:',
' pass',
]
for j, pl in enumerate(patch_lines):
lines.insert(insert_idx + j, pl)
code = '\n'.join(lines)
print(f" PATCH 1 applied (alternate) at line {insert_idx}")
break
# Write patched file
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
f.write(code)
print(" File written")
# Verify patch
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
verify = f.read().decode()
if 'COMFYUI_SHARED_MEMORY' in verify:
print(" Patch verified!")
else:
print(" ERROR: Patch not found in file!")
# ======================================
# 3) Write launcher with SHARED mode
# ======================================
print("3) Write launcher with COMFYUI_SHARED_MEMORY=1")
launcher = textwrap.dedent("""\
#!/bin/bash
# GPU
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
# Threading
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
# MIOpen
export MIOPEN_FIND_MODE=3
# Shared memory APU mode: CPU and GPU share the same physical RAM
export COMFYUI_SHARED_MEMORY=1
cd ~/ComfyUI
source ~/comfyui-env/bin/activate
# --force-fp16: half precision (saves memory)
# --fp16-vae: VAE in fp16 (320MB instead of 640MB, fits in GPU memory)
# SHARED mode: models load directly on GPU, no offloading overhead
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--force-fp16 \\
--fp16-vae
""")
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
f.write(launcher)
sh('chmod +x /tmp/run_comfyui.sh')
print(" Flags: --force-fp16 --fp16-vae + COMFYUI_SHARED_MEMORY=1")
# ======================================
# 4) Start
# ======================================
print("4) Start ComfyUI")
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(3)
pid = sh('pgrep -f "python3.*main.py"')
print(f" PID: {pid}")
# ======================================
# 5) Wait ready
# ======================================
print("5) Wait HTTP", end='', flush=True)
for i in range(90):
code = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
if '200' in code:
print(f" OK ({i*2}s)")
break
if i % 10 == 0 and i > 0:
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
ls = [l.strip() for l in log.split('\n') if l.strip() and 'FETCH' not in l and 'DEPRECATION' not in l]
print(f"\n [{i*2}s] {ls[-1][:80] if ls else ''}", end='', flush=True)
except: pass
else:
print('.', end='', flush=True)
time.sleep(2)
# Verify SHARED mode
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['vram state', 'SHARED', 'Device:', 'Total VRAM', 'pytorch version']):
print(f" {s}")
# ======================================
# 6) Submit
# ======================================
print("6) Submit")
wf = {"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 999, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(wf))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f" {resp[:150]}")
# ======================================
# 7) Monitor
# ======================================
print("7) Monitor (SHARED mode = everything on GPU)")
t0 = time.time()
for i in range(200):
el = int(time.time() - t0)
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
tc = int(temp)//1000 if temp.isdigit() else '?'
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
samp = ''
last = ''
for line in log.split('\n'):
s = line.strip()
if '/8' in s and ('it/s' in s or 's/it' in s): samp = s
if s and 'FETCH' not in s and 'startup' not in s and 'DEPRECATION' not in s: last = s
print(f" [{el:>4}s] {tc}C | {(samp or last)[-90:]}")
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
et = ''
for line in log.split('\n'):
if 'Prompt executed' in line: et = line.strip()
print(f"\n *** DONE! ***")
print(f" File: {imgs}")
print(f" {et}")
print(f" Wall: {el}s")
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['loaded', '/8', 'Prompt executed', 'VAE load', 'Requested']):
if 'FETCH' not in s:
print(f" {s}")
break
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending') and el > 30:
time.sleep(2)
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** DONE: {imgs} ***")
else:
print(f"\n Queue empty, no image. Log:")
for line in log.split('\n')[-20:]:
if line.strip() and 'FETCH' not in line: print(f" {line.strip()}")
break
except: pass
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
print("\n CRASHED!")
for line in log.split('\n')[-30:]:
if line.strip(): print(f" {line.strip()}")
break
time.sleep(10)
sftp.close()
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_shared_fix.py
+216
View File
@@ -0,0 +1,216 @@
"""Quick: check current state, patch for SHARED mode, restart."""
import paramiko, time, json, textwrap
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
def sh(cmd, timeout=60):
chan = c.get_transport().open_session()
chan.settimeout(timeout)
chan.exec_command('/bin/bash -l -c ' + "'" + cmd.replace("'", "'\\''") + "'")
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
return out.decode(errors='replace').strip()
# Check current cmdline
print("=== Current process ===")
print(sh('ps aux | grep main.py | grep -v grep'))
# Check if patch exists
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'r') as f:
code = f.read().decode()
print(f"\n=== Patch status: {'APPLIED' if 'COMFYUI_SHARED_MEMORY' in code else 'NOT applied'} ===")
# Check current vram state in log
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n'):
s = line.strip()
if 'vram state' in s or 'VAE load' in s or 'Device:' in s:
print(f" {s}")
except: pass
# =============================
# APPLY PATCH if not done
# =============================
if 'COMFYUI_SHARED_MEMORY' not in code:
print("\nApplying SHARED memory patch...")
# Backup
sh('cp /home/fabian/ComfyUI/comfy/model_management.py /home/fabian/ComfyUI/comfy/model_management.py.bak')
old = 'if cpu_state == CPUState.MPS:\n vram_state = VRAMState.SHARED'
new = '''if cpu_state == CPUState.MPS:
vram_state = VRAMState.SHARED
# Shared memory APU: CPU+GPU share physical RAM (e.g. AMD BC-250)
import os as _os
if _os.environ.get("COMFYUI_SHARED_MEMORY") == "1" and cpu_state == CPUState.GPU:
vram_state = VRAMState.SHARED
logging.info("SHARED vram: APU shared memory mode enabled")'''
if old in code:
code = code.replace(old, new)
with sftp.open('/home/fabian/ComfyUI/comfy/model_management.py', 'w') as f:
f.write(code)
print(" Patch applied!")
else:
print(" ERROR: Could not find patch target. Dumping area:")
for i, line in enumerate(code.split('\n')):
if 'MPS' in line and 'SHARED' in code.split('\n')[i+1] if i+1 < len(code.split('\n')) else '':
for j in range(max(0,i-3), min(i+5, len(code.split('\n')))):
print(f" {j+1}: {code.split(chr(10))[j]}")
else:
print(" Patch already applied, good.")
# =============================
# KILL + RESTART with SHARED
# =============================
print("\nKilling ComfyUI...")
sh('pkill -9 -f "python3.*main.py" 2>/dev/null; sleep 2')
launcher = textwrap.dedent("""\
#!/bin/bash
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HIP_VISIBLE_DEVICES=0
export HSA_ENABLE_SDMA=0
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
export PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
export OMP_NUM_THREADS=12
export MKL_NUM_THREADS=12
export OPENBLAS_NUM_THREADS=12
export MIOPEN_FIND_MODE=3
# APU shared memory mode: everything on GPU
export COMFYUI_SHARED_MEMORY=1
cd ~/ComfyUI
source ~/comfyui-env/bin/activate
# SHARED mode: models load on GPU directly (shared memory = zero copy)
# --force-fp16: half precision for models
# --fp16-vae: VAE in fp16 (160MB, fast on GPU)
exec python3 main.py \\
--listen 0.0.0.0 --port 8188 \\
--force-fp16 \\
--fp16-vae
""")
with sftp.open('/tmp/run_comfyui.sh', 'w') as f:
f.write(launcher)
sh('chmod +x /tmp/run_comfyui.sh')
sh('rm -f /tmp/comfyui.log; rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png')
sh('nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &')
time.sleep(3)
pid = sh('pgrep -f "python3.*main.py"')
print(f"Started PID: {pid}")
# Wait for ready
print("Waiting for HTTP", end='', flush=True)
for i in range(90):
r = sh('curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null', timeout=5)
if '200' in r:
print(f" OK ({i*2}s)")
break
print('.', end='', flush=True)
time.sleep(2)
# Verify SHARED mode
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['vram state', 'SHARED', 'Device:', 'Total VRAM']):
print(f" {s}")
# Submit test
print("\nSubmitting test workflow...")
wf = {"prompt": {
"1": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode", "inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["4", 0], "negative": ["4", 0],
"latent_image": ["5", 0], "seed": 12345, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["3", 0]}},
"8": {"class_type": "SaveImage", "inputs": {"images": ["7", 0], "filename_prefix": "ZImageTurbo_GPU"}}
}}
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(wf))
resp = sh('curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json')
print(f" {resp[:120]}")
# Monitor
print("\nMonitoring (SHARED = VAE on GPU, everything on GPU)...")
t0 = time.time()
for i in range(200):
el = int(time.time() - t0)
temp = sh('cat /sys/class/drm/card0/device/hwmon/hwmon*/temp1_input 2>/dev/null', timeout=5)
tc = int(temp)//1000 if temp.isdigit() else '?'
try:
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
except: log = ''
samp = ''
last = ''
for line in log.split('\n'):
s = line.strip()
if '/8' in s and ('it/s' in s or 's/it' in s): samp = s
if s and 'FETCH' not in s and 'startup' not in s and 'DEPRECATION' not in s: last = s
print(f" [{el:>4}s] {tc}C | {(samp or last)[-90:]}")
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
et = ''
for line in log.split('\n'):
if 'Prompt executed' in line: et = line.strip()
print(f"\n *** DONE! *** {imgs}")
print(f" {et}")
print(f" Wall: {el}s")
for line in log.split('\n'):
s = line.strip()
if any(k in s for k in ['loaded', '/8', 'Prompt executed', 'VAE load', 'Requested']):
if 'FETCH' not in s: print(f" {s}")
break
q = sh('curl -s http://127.0.0.1:8188/queue 2>/dev/null', timeout=5)
try:
qd = json.loads(q)
if not qd.get('queue_running') and not qd.get('queue_pending') and el > 30:
time.sleep(2)
imgs = sh('ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null', timeout=5)
if imgs:
print(f"\n *** DONE: {imgs} ***")
else:
print(f"\n Queue empty, no image:")
for line in log.split('\n')[-20:]:
s = line.strip()
if s and 'FETCH' not in s: print(f" {s}")
break
except: pass
if sh('pgrep -f "python3.*main.py" >/dev/null && echo Y || echo N', timeout=5) == 'N':
print("\n CRASHED!")
for line in log.split('\n')[-30:]:
if line.strip(): print(f" {line.strip()}")
break
time.sleep(10)
sftp.close()
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_single.py
+118
View File
@@ -0,0 +1,118 @@
"""Upload a self-contained bash script and run it in ONE SSH session."""
import paramiko, time
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
# Write the entire fix+restart+test as ONE bash script
script = r'''#!/bin/bash
set -e
# Kill any running ComfyUI
pkill -9 -f "python3.*main.py" 2>/dev/null || true
sleep 2
# Clean
rm -f /tmp/comfyui.log
rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png
# Start ComfyUI
echo "Starting ComfyUI..."
nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &
sleep 3
PID=$(pgrep -f "python3.*main.py" | head -1)
echo "PID: $PID"
if [ -z "$PID" ]; then
echo "FAILED TO START!"
cat /tmp/comfyui.log 2>/dev/null | tail -20
exit 1
fi
# Wait for HTTP
echo "Waiting for HTTP..."
for i in $(seq 1 90); do
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null || echo 000)
if [ "$CODE" = "200" ]; then
echo "Ready after ${i}s"
break
fi
sleep 2
done
# Show key startup info
grep -E "vram state|SHARED|Device:|Total VRAM|load device|offload" /tmp/comfyui.log 2>/dev/null || true
# Submit workflow
echo ""
echo "Submitting workflow..."
cat > /tmp/wf.json << 'WFEOF'
{"prompt":{"1":{"class_type":"UnetLoaderGGUF","inputs":{"unet_name":"z_image_turbo-Q5_K_S.gguf"}},"2":{"class_type":"CLIPLoaderGGUF","inputs":{"clip_name":"Qwen3-4B.i1-Q5_K_S.gguf","type":"qwen_image"}},"3":{"class_type":"VAELoader","inputs":{"vae_name":"ae.safetensors"}},"4":{"class_type":"CLIPTextEncode","inputs":{"text":"A red fox in a snowy forest, photorealistic","clip":["2",0]}},"5":{"class_type":"EmptyLatentImage","inputs":{"width":512,"height":512,"batch_size":1}},"6":{"class_type":"KSampler","inputs":{"model":["1",0],"positive":["4",0],"negative":["4",0],"latent_image":["5",0],"seed":99999,"steps":8,"cfg":1.0,"sampler_name":"euler","scheduler":"simple","denoise":1.0}},"7":{"class_type":"VAEDecode","inputs":{"samples":["6",0],"vae":["3",0]}},"8":{"class_type":"SaveImage","inputs":{"images":["7",0],"filename_prefix":"ZImageTurbo_GPU"}}}}
WFEOF
RESP=$(curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json)
echo "Response: ${RESP:0:120}"
# Monitor
echo ""
echo "Monitoring..."
START=$(date +%s)
LAST=""
while true; do
NOW=$(date +%s)
ELAPSED=$((NOW - START))
if [ $ELAPSED -gt 600 ]; then
echo "TIMEOUT after 600s"
tail -20 /tmp/comfyui.log
break
fi
# Check for output image
if ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null; then
echo ""
echo "*** IMAGE DONE in ${ELAPSED}s! ***"
grep -E "loaded|load device|offload|Prompt executed|/8" /tmp/comfyui.log 2>/dev/null | grep -v FETCH || true
break
fi
# Show progress
LINE=$(grep -E "/8|loaded|Requested|VAE|Prompt executed|Error|OOM" /tmp/comfyui.log 2>/dev/null | grep -v FETCH | grep -v audio_vae | grep -v "split attention" | tail -1)
GPU=$(cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null || echo "?")
if [ "$LINE" != "$LAST" ] && [ -n "$LINE" ]; then
echo "[${ELAPSED}s] GPU:${GPU}% ${LINE:0:110}"
LAST="$LINE"
fi
# Check alive
if ! pgrep -f "python3.*main.py" > /dev/null 2>&1; then
echo "CRASHED at ${ELAPSED}s!"
tail -20 /tmp/comfyui.log
break
fi
sleep 3
done
'''
with sftp.open('/tmp/fix_and_run.sh', 'w') as f:
f.write(script)
sftp.close()
# Execute in ONE session
print("Running fix+restart+monitor on BC-250...")
stdin, stdout, stderr = c.exec_command('bash /tmp/fix_and_run.sh', timeout=660)
# Stream output
for line in iter(stdout.readline, ''):
print(line.rstrip())
err = stderr.read().decode(errors='replace').strip()
if err:
for l in err.split('\n')[-10:]:
if l.strip(): print(f"STDERR: {l.strip()}")
c.close()
print("\nDone.")
ComfyUI Scripts/bc250_st.py
+44
View File
@@ -0,0 +1,44 @@
"""Quick status: is ComfyUI still running and what's the log say?"""
import paramiko
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
sftp = c.open_sftp()
# Read full log
with sftp.open('/tmp/comfyui.log', 'r') as f:
log = f.read().decode(errors='replace')
sftp.close()
lines = log.split('\n')
print(f"Total lines: {len(lines)}")
print()
# Show only meaningful lines
for line in lines:
s = line.strip()
if not s:
continue
if 'FETCH ComfyRegistry' in s or 'All startup tasks' in s or 'FETCH DATA' in s:
continue
print(s)
# Check process + GPU
chan = c.get_transport().open_session()
chan.settimeout(10)
chan.exec_command('/bin/bash -c "echo; echo === PROCESS ===; ps aux | grep python3 | grep -v grep; echo; echo === GPU ===; rocm-smi 2>/dev/null | head -12; echo; echo === OUTPUT ===; ls -la ~/ComfyUI/output/ 2>/dev/null; echo; echo === QUEUE ===; curl -s http://127.0.0.1:8188/queue 2>/dev/null"')
out = b""
while True:
try:
chunk = chan.recv(65536)
if not chunk: break
out += chunk
except: break
chan.close()
print(out.decode(errors='replace'))
c.close()
ComfyUI Scripts/bc250_start_build.py
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env python3
"""Start PyTorch build on BC-250 properly using SFTP for the script."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=120, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 60:
print(f" ... ({len(lines)} lines, showing last 60)")
print('\n'.join(lines[-60:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-20:] if len(lines) > 20 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# Upload the build script via SFTP
build_script = '''#!/bin/bash
set -euo pipefail
LOG="/home/fabian/pytorch_build.log"
exec > >(tee -a "$LOG") 2>&1
echo "=========================================="
echo " PyTorch Build for ROCm gfx1010 (BC-250)"
echo " Started: $(date)"
echo "=========================================="
# Activate venv
source /home/fabian/comfyui-env/bin/activate
cd /home/fabian/pytorch
# ROCm build configuration
export USE_ROCM=1
export USE_CUDA=0
export PYTORCH_ROCM_ARCH="gfx1010"
export HIP_VISIBLE_DEVICES=0
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export ROCM_PATH=/opt/rocm
export HIP_PATH=/opt/rocm
export CMAKE_PREFIX_PATH="/opt/rocm;$(python3 -c 'import sys; print(sys.prefix)')"
export PATH=/opt/rocm/bin:$PATH
# Build settings
export USE_NINJA=1
export CMAKE_GENERATOR=Ninja
export MAX_JOBS=6
export USE_CCACHE=1
export CCACHE_DIR=/home/fabian/.ccache
# Disable unnecessary components for faster build
export USE_FBGEMM=0
export USE_KINETO=0
export USE_CUPTI_SO=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_TENSORPIPE=0
export USE_GLOO=0
export USE_MPI=0
export USE_OPENMP=1
export USE_MKLDNN=1
export BUILD_TEST=0
export USE_CUDNN=0
echo ""
echo "Build config:"
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo " USE_ROCM=$USE_ROCM"
echo " MAX_JOBS=$MAX_JOBS"
echo " Python: $(python3 --version)"
echo " hipcc: $(hipcc --version 2>&1 | head -1)"
echo " ROCm: $(cat /opt/rocm/.info/version)"
echo ""
# Install requirements
echo "Installing PyTorch requirements..."
pip install -r requirements.txt 2>&1 | tail -10
echo ""
# Clean any partial build
echo "Cleaning previous build artifacts..."
python3 setup.py clean 2>&1 || true
echo ""
# Build the wheel
echo "Starting PyTorch build..."
echo "=========================================="
python3 setup.py bdist_wheel 2>&1
BUILD_RC=$?
echo ""
echo "=========================================="
echo " Build exit code: $BUILD_RC"
echo " Finished: $(date)"
echo "=========================================="
if [ $BUILD_RC -eq 0 ]; then
echo ""
echo "Wheel files:"
ls -lh dist/*.whl 2>/dev/null
echo ""
echo "Installing wheel..."
pip install dist/*.whl 2>&1
echo ""
echo "=== VERIFICATION ==="
python3 -c "
import torch
print(f'PyTorch version: {torch.__version__}')
print(f'HIP version: {torch.version.hip}')
print(f'CUDA available (HIP): {torch.cuda.is_available()}')
if torch.cuda.is_available():
print(f'Device name: {torch.cuda.get_device_name(0)}')
print(f'Device count: {torch.cuda.device_count()}')
t = torch.randn(4, 4, device=\"cuda\")
print(f'Tensor on GPU: {t.device}')
print(f'Tensor sum: {t.sum().item():.4f}')
print('GPU COMPUTE: WORKING')
else:
print('WARNING: CUDA/HIP not available')
" 2>&1
fi
echo ""
echo "BUILD_COMPLETE_RC=$BUILD_RC"
'''
print("Uploading build script via SFTP...")
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/build_pytorch.sh', 'w') as f:
f.write(build_script)
sftp.close()
run("chmod +x /home/fabian/build_pytorch.sh", desc="Make executable")
# Remove old log if it exists
run("rm -f /home/fabian/pytorch_build.log", desc="Clean old log")
# Start the build using nohup inside bash (not fish)
# Using bash explicitly to avoid fish issues with nohup
run("bash -c 'nohup bash /home/fabian/build_pytorch.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
desc="Start build in background")
# Wait for it to actually start
time.sleep(10)
# Verify it's running
run("pgrep -fa 'build_pytorch\\|setup.py' | head -10",
desc="Verify build is running")
# Check initial log
time.sleep(5)
run("cat /home/fabian/pytorch_build.log 2>/dev/null | head -30 || echo 'Log not yet available'",
desc="Initial build log")
# Monitor for first compile steps
time.sleep(30)
run("tail -30 /home/fabian/pytorch_build.log 2>/dev/null || echo 'Waiting for log...'",
desc="Build progress after 30 seconds")
ssh.close()
print("\n" + "="*60)
print(" PyTorch build running on BC-250!")
print(" Monitor: tail -f ~/pytorch_build.log")
print("="*60)
ComfyUI Scripts/bc250_status.py
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Quick single-connection status check."""
import paramiko, json
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian',
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
def run(cmd):
_, so, se = ssh.exec_command(cmd, timeout=15)
return so.read().decode()
try:
# Is ComfyUI running?
print("=== PROCESS ===")
print(run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" ps -p $PID -o pid,%cpu,%mem,nlwp,etime --no-headers; "
"else echo NOT_RUNNING; fi'").strip())
# Log
print("\n=== LOG (last 20) ===")
print(run("tail -20 /home/fabian/comfyui.log 2>/dev/null").strip())
# GPU
print("\n=== GPU ===")
gpu = run("HSA_OVERRIDE_GFX_VERSION=10.1.0 rocm-smi 2>/dev/null | grep -E '0x|GPU%'")
print(gpu.strip() if gpu.strip() else "no output")
# Output files
print("\n=== OUTPUT ===")
print(run("ls -lah /home/fabian/ComfyUI/output/ 2>/dev/null").strip())
# Queue
print("\n=== QUEUE ===")
q = run("curl -s http://localhost:8188/queue 2>/dev/null")
if q.strip():
qj = json.loads(q)
print(f"Running: {len(qj.get('queue_running',[]))}, Pending: {len(qj.get('queue_pending',[]))}")
else:
print("Server not responding")
finally:
ssh.close()
print("\nSSH closed.")
ComfyUI Scripts/bc250_step1_deps.py
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
"""Step 1: Install build dependencies on BC-250 for PyTorch build."""
import paramiko
import sys
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=300, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
print(out.strip()[-2000:]) # Last 2000 chars
if err.strip():
# Filter out common noise
lines = [l for l in err.strip().split('\n') if not l.startswith('warning:')]
if lines:
print(f"STDERR: {chr(10).join(lines[-20:])}")
print(f" Exit code: {rc}")
return rc, out, err
# Install build tools
run("sudo pacman -S --needed --noconfirm python-pip ninja ccache",
desc="Install pip, ninja, ccache")
# Install PyTorch build dependencies
run("sudo pacman -S --needed --noconfirm cmake blas lapack openblas "
"python-numpy python-pyyaml python-typing_extensions "
"intel-oneapi-mkl 2>/dev/null; echo done",
desc="Install build dependencies (cmake, blas, numpy, etc.)")
# Install additional deps that PyTorch needs
run("sudo pacman -S --needed --noconfirm python-cffi python-setuptools "
"python-wheel python-filelock python-sympy python-networkx",
desc="Install Python dependencies")
# Verify installs
run("pip --version && ninja --version && ccache --version | head -1 && cmake --version | head -1",
desc="Verify installations")
# Check pip can install packages
run("pip install --user --upgrade pip setuptools wheel 2>&1 | tail -5",
desc="Upgrade pip/setuptools")
ssh.close()
print("\n\nDone — build dependencies installed.")
ComfyUI Scripts/bc250_step1_kill.py
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
"""Step 1: Kill stuck ComfyUI and check flags."""
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=60):
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
print(out.strip() if out.strip() else "")
if err.strip():
for l in err.strip().split('\n')[-5:]:
print(f"STDERR: {l}")
print("=== Kill stuck ===")
run("pkill -f 'python3 main.py' 2>/dev/null; sleep 2; pkill -9 -f 'python3 main.py' 2>/dev/null; sleep 1; echo killed")
print("\n=== Check flags ===")
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && cd /home/fabian/ComfyUI && python3 main.py --help 2>&1 | grep -i -E \"vae|fp16|fp32|force|cpu|novram|lowvram\"'")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_step2_clone.py
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Step 2: Create venv and clone PyTorch on BC-250."""
import paramiko
import sys
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=600, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
# Print last portion for long outputs
lines = out.strip().split('\n')
if len(lines) > 50:
print(f" ... ({len(lines)} lines, showing last 50)")
print('\n'.join(lines[-50:]))
else:
print(out.strip())
if err.strip():
lines = [l for l in err.strip().split('\n') if 'warning:' not in l.lower()]
if lines:
if len(lines) > 30:
print(f"STDERR ({len(lines)} lines, last 30):")
print('\n'.join(lines[-30:]))
else:
print(f"STDERR: {chr(10).join(lines)}")
print(f" Exit code: {rc}")
return rc, out, err
# Create the venv
run("python3 -m venv ~/comfyui-env --system-site-packages",
desc="Create venv with system site-packages (for numpy, etc.)")
# Activate and install basic build deps in venv
run("bash -c 'source ~/comfyui-env/bin/activate && pip install --upgrade pip setuptools wheel'",
desc="Upgrade pip in venv")
run("bash -c 'source ~/comfyui-env/bin/activate && pip install cmake ninja pyyaml typing-extensions cffi future six requests dataclasses filelock sympy networkx jinja2 numpy'",
desc="Install PyTorch build deps in venv", timeout=120)
# Check if PyTorch source already exists
rc, out, _ = run("test -d ~/pytorch && echo EXISTS || echo MISSING",
desc="Check for existing PyTorch source")
if "EXISTS" in out:
print("\n PyTorch source directory exists. Checking if it's a valid repo...")
run("cd ~/pytorch && git log --oneline -1 2>&1", desc="Check PyTorch repo")
else:
# Clone PyTorch — this is the big download
print("\n Cloning PyTorch (this will take a while)...")
run("git clone --depth 1 --recursive --shallow-submodules https://github.com/pytorch/pytorch.git ~/pytorch 2>&1 | tail -20",
desc="Clone PyTorch (shallow, with submodules)",
timeout=1200) # 20 minutes timeout
# Verify clone
run("ls -la ~/pytorch/setup.py ~/pytorch/torch/ 2>&1 | head -5",
desc="Verify PyTorch source")
run("cd ~/pytorch && git log --oneline -1",
desc="PyTorch version")
run("du -sh ~/pytorch",
desc="PyTorch source size")
ssh.close()
print("\n\nDone — venv created and PyTorch cloned.")
ComfyUI Scripts/bc250_step3_build.py
+180
View File
@@ -0,0 +1,180 @@
#!/usr/bin/env python3
"""Step 3: Build PyTorch from source for ROCm gfx1010 on BC-250.
This build will take a long time (1-3 hours on 12 cores).
We run it non-interactively via nohup so it survives SSH disconnects.
"""
import paramiko
import sys
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=600, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 50:
print(f" ... ({len(lines)} lines, showing last 50)")
print('\n'.join(lines[-50:]))
else:
print(out.strip())
if err.strip():
lines = [l for l in err.strip().split('\n')]
if lines:
show = lines[-30:] if len(lines) > 30 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# First, create the build script on the BC-250
build_script = r'''#!/bin/bash
set -euo pipefail
LOG="/home/fabian/pytorch_build.log"
exec > >(tee -a "$LOG") 2>&1
echo "=========================================="
echo " PyTorch Build for ROCm gfx1010 (BC-250)"
echo " Started: $(date)"
echo "=========================================="
# Activate venv
source /home/fabian/comfyui-env/bin/activate
# Go to PyTorch source
cd /home/fabian/pytorch
# Set environment for ROCm build
export USE_ROCM=1
export USE_CUDA=0
export PYTORCH_ROCM_ARCH="gfx1010"
export HIP_VISIBLE_DEVICES=0
export HSA_OVERRIDE_GFX_VERSION=10.1.0
export HSA_ENABLE_SDMA=0
export ROCM_PATH=/opt/rocm
export HIP_PATH=/opt/rocm
export CMAKE_PREFIX_PATH=/opt/rocm
export PATH=/opt/rocm/bin:$PATH
# Use ninja for faster builds
export USE_NINJA=1
export CMAKE_GENERATOR=Ninja
# Limit parallel jobs to avoid OOM (14GB RAM + 14GB swap)
# Each compilation unit can use ~1-2GB during link, so limit to 6 jobs
export MAX_JOBS=6
# Use ccache to speed up rebuilds
export USE_CCACHE=1
export CCACHE_DIR=/home/fabian/.ccache
# Disable unnecessary components to speed up build
export USE_FBGEMM=0
export USE_KINETO=0
export USE_CUPTI_SO=0
export USE_NCCL=0
export USE_DISTRIBUTED=0
export USE_TENSORPIPE=0
export USE_GLOO=0
export USE_MPI=0
export USE_OPENMP=1
export USE_MKLDNN=1
export BUILD_TEST=0
# Disable CUDA-specific stuff
export USE_CUDNN=0
echo ""
echo "Build config:"
echo " PYTORCH_ROCM_ARCH=$PYTORCH_ROCM_ARCH"
echo " USE_ROCM=$USE_ROCM"
echo " MAX_JOBS=$MAX_JOBS"
echo " USE_CCACHE=$USE_CCACHE"
echo " Python: $(python3 --version)"
echo " hipcc: $(hipcc --version 2>&1 | head -1)"
echo ""
# Install requirements
echo "Installing PyTorch requirements..."
pip install -r requirements.txt 2>&1 | tail -5
# Run the build
echo ""
echo "Starting PyTorch build... (this will take 1-3 hours)"
echo "=========================================="
python3 setup.py bdist_wheel 2>&1
BUILD_RC=$?
echo ""
echo "=========================================="
echo " Build finished with exit code: $BUILD_RC"
echo " Time: $(date)"
echo "=========================================="
if [ $BUILD_RC -eq 0 ]; then
echo "Wheel file:"
ls -lh dist/*.whl 2>/dev/null || echo "No wheel found, trying develop install..."
# Install the wheel
echo "Installing PyTorch wheel..."
pip install dist/*.whl 2>&1 | tail -5
# Verify
echo ""
echo "Verification:"
python3 -c "import torch; print(f'PyTorch {torch.__version__}'); print(f'ROCm: {torch.version.hip}'); print(f'CUDA available: {torch.cuda.is_available()}'); print(f'Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"N/A\"}')"
fi
echo "BUILD_COMPLETE_RC=$BUILD_RC" >> "$LOG"
'''
# Write the build script to BC-250
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/build_pytorch.sh', 'w') as f:
f.write(build_script)
sftp.close()
run("chmod +x /home/fabian/build_pytorch.sh", desc="Make build script executable")
# Check if a build is already running
rc, out, _ = run("pgrep -f 'setup.py bdist_wheel' || echo 'NOT RUNNING'",
desc="Check if build is already running")
if 'NOT RUNNING' not in out:
print("\n BUILD IS ALREADY RUNNING — not starting a new one.")
print(" Monitor with: tail -f ~/pytorch_build.log")
else:
# Start the build in background using nohup
# This way it survives SSH disconnects
run("nohup bash /home/fabian/build_pytorch.sh > /dev/null 2>&1 &",
desc="Starting PyTorch build in background (nohup)")
# Give it a moment to start
time.sleep(5)
# Verify it started
run("pgrep -fa 'build_pytorch.sh' || pgrep -fa 'setup.py' || echo 'WARNING: Build may have failed to start'",
desc="Verify build process started")
# Check initial log output
time.sleep(10)
run("tail -30 /home/fabian/pytorch_build.log 2>/dev/null || echo 'Log not yet created'",
desc="Initial build log output")
ssh.close()
print("\n" + "="*60)
print(" PyTorch build started in background on BC-250!")
print(" Monitor: ssh fabian@BC-250 'tail -f ~/pytorch_build.log'")
print(" Check status: ssh fabian@BC-250 'pgrep -fa setup.py'")
print("="*60)
ComfyUI Scripts/bc250_step3b_fix.py
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Install missing ROCm math libraries for PyTorch build on BC-250."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=300, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 60:
print(f" ... ({len(lines)} lines, showing last 60)")
print('\n'.join(lines[-60:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-20:] if len(lines) > 20 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# Find all available ROCm packages
run("pacman -Ss rocm | grep -E '^cachyos|^extra|^core' | head -40",
desc="Available ROCm packages")
# Install all ROCm math/compute libraries needed by PyTorch
run("sudo pacman -S --needed --noconfirm "
"hiprand rocrand "
"hipblas rocblas "
"hipfft rocfft "
"hipsparse rocsparse "
"hipsolver rocsolver "
"miopen-hip "
"rocprim hipcub "
"rocthrust "
"rccl "
"hipblaslt "
"roctracer "
"2>&1 | tail -40",
desc="Install ROCm math libraries",
timeout=600)
# Verify hiprand is now available
run("find /opt/rocm -name 'hiprandConfig.cmake' -o -name 'hiprand-config.cmake' 2>/dev/null | head -5",
desc="Verify hiprand cmake config")
# Check all libraries
run("ls /opt/rocm/lib/libhiprand.so /opt/rocm/lib/librocblas.so /opt/rocm/lib/libhipblas.so /opt/rocm/lib/librocfft.so /opt/rocm/lib/libMIOpen.so 2>&1",
desc="Verify key libraries exist")
# Clean the failed build and restart
run("bash -c 'source ~/comfyui-env/bin/activate && cd ~/pytorch && python3 setup.py clean 2>&1 | tail -5'",
desc="Clean failed build")
# Restart build
run("rm -f /home/fabian/pytorch_build.log", desc="Clean old log")
run("bash -c 'nohup bash /home/fabian/build_pytorch.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
desc="Restart PyTorch build")
time.sleep(15)
run("pgrep -fa 'setup.py\\|cmake\\|ninja' | head -10",
desc="Verify build restarted")
time.sleep(45)
run("tail -40 /home/fabian/pytorch_build.log 2>/dev/null",
desc="Build progress")
ssh.close()
print("\nDone — ROCm libs installed and build restarted.")
ComfyUI Scripts/bc250_step4_comfyui.py
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Install ComfyUI and dependencies on BC-250."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=300, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 60:
print(f" ... ({len(lines)} lines, showing last 60)")
print('\n'.join(lines[-60:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-20:] if len(lines) > 20 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# Clone ComfyUI
run("git clone --depth 1 https://github.com/comfyanonymous/ComfyUI.git ~/ComfyUI 2>&1 | tail -10",
desc="Clone ComfyUI")
# Install ComfyUI requirements in venv
run("bash -c 'source ~/comfyui-env/bin/activate && cd ~/ComfyUI && pip install -r requirements.txt 2>&1 | tail -20'",
desc="Install ComfyUI requirements",
timeout=300)
# Install diffusers from source (needed for ZImagePipeline)
run("bash -c 'source ~/comfyui-env/bin/activate && pip install git+https://github.com/huggingface/diffusers 2>&1 | tail -10'",
desc="Install diffusers from source (for ZImagePipeline)",
timeout=300)
# Install additional deps that ComfyUI/Z-Image might need
run("bash -c 'source ~/comfyui-env/bin/activate && pip install transformers accelerate safetensors sentencepiece huggingface_hub aiohttp einops torchvision 2>&1 | tail -15'",
desc="Install transformers, accelerate, etc.",
timeout=300)
# Install ComfyUI-Manager (custom node manager)
run("git clone --depth 1 https://github.com/Comfy-Org/ComfyUI-Manager.git ~/ComfyUI/custom_nodes/ComfyUI-Manager 2>&1 | tail -5",
desc="Install ComfyUI-Manager")
# Install ComfyUI-GGUF (needed for GGUF checkpoint format)
run("git clone --depth 1 https://github.com/city96/ComfyUI-GGUF.git ~/ComfyUI/custom_nodes/ComfyUI-GGUF 2>&1 | tail -5",
desc="Install ComfyUI-GGUF nodes")
# Install GGUF dependencies
run("bash -c 'source ~/comfyui-env/bin/activate && pip install gguf 2>&1 | tail -5'",
desc="Install gguf Python package")
# Install Z-Image Power Nodes
run("git clone --depth 1 https://github.com/martin-rizzo/ComfyUI-ZImagePowerNodes.git ~/ComfyUI/custom_nodes/ComfyUI-ZImagePowerNodes 2>&1 | tail -5",
desc="Install Z-Image Power Nodes")
# Verify ComfyUI structure
run("ls -la ~/ComfyUI/main.py ~/ComfyUI/custom_nodes/ 2>&1",
desc="Verify ComfyUI structure")
run("ls ~/ComfyUI/custom_nodes/",
desc="Custom nodes installed")
# Create model directories
run("mkdir -p ~/ComfyUI/models/diffusion_models ~/ComfyUI/models/text_encoders ~/ComfyUI/models/vae ~/ComfyUI/models/checkpoints",
desc="Create model directories")
# Quick test: can ComfyUI import?
run("""bash -c 'source ~/comfyui-env/bin/activate && \
HSA_OVERRIDE_GFX_VERSION=10.1.0 \
HIP_VISIBLE_DEVICES=0 \
HSA_ENABLE_SDMA=0 \
cd ~/ComfyUI && python3 -c "
import torch
print(f\\"torch {torch.__version__} hip={torch.version.hip} cuda={torch.cuda.is_available()}\\")
import comfy
print(\\"ComfyUI import OK\\")
" 2>&1'""",
desc="Test ComfyUI import",
timeout=60)
ssh.close()
print("\nDone — ComfyUI installed.")
ComfyUI Scripts/bc250_step5_models.py
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Download Z-Image-Turbo GGUF model checkpoints on BC-250.
Files needed (GGUF format — memory-efficient for 14GB RAM):
1. z_image_turbo-Q5_K_S.gguf (5.19 GB) → diffusion_models/
2. Qwen3-4B.i1-Q5_K_S.gguf (2.82 GB) → text_encoders/
3. ae.safetensors (335 MB) → vae/
Total: ~8.35 GB
"""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=3600, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 30:
print(f" ... ({len(lines)} lines, showing last 30)")
print('\n'.join(lines[-30:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-15:] if len(lines) > 15 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# Create download script for background execution
dl_script = '''#!/bin/bash
set -euo pipefail
LOG="/home/fabian/model_download.log"
exec > >(tee -a "$LOG") 2>&1
source /home/fabian/comfyui-env/bin/activate
COMFY="$HOME/ComfyUI"
echo "=========================================="
echo " Downloading Z-Image-Turbo GGUF Models"
echo " Started: $(date)"
echo "=========================================="
# 1. Diffusion model (5.19 GB)
echo ""
echo "[1/3] Downloading z_image_turbo-Q5_K_S.gguf (5.19 GB)..."
if [ -f "$COMFY/models/diffusion_models/z_image_turbo-Q5_K_S.gguf" ]; then
echo " Already exists, skipping."
else
HF_XET_HIGH_PERFORMANCE=1 huggingface-cli download \
jayn7/Z-Image-Turbo-GGUF \
z_image_turbo-Q5_K_S.gguf \
--local-dir "$COMFY/models/diffusion_models/" \
--local-dir-use-symlinks False
echo " Done."
fi
# 2. Text encoder Qwen3-4B (2.82 GB)
echo ""
echo "[2/3] Downloading Qwen3-4B.i1-Q5_K_S.gguf (2.82 GB)..."
if [ -f "$COMFY/models/text_encoders/Qwen3-4B.i1-Q5_K_S.gguf" ]; then
echo " Already exists, skipping."
else
HF_XET_HIGH_PERFORMANCE=1 huggingface-cli download \
mradermacher/Qwen3-4B-i1-GGUF \
Qwen3-4B.i1-Q5_K_S.gguf \
--local-dir "$COMFY/models/text_encoders/" \
--local-dir-use-symlinks False
echo " Done."
fi
# 3. VAE (335 MB)
echo ""
echo "[3/3] Downloading ae.safetensors (VAE, 335 MB)..."
if [ -f "$COMFY/models/vae/ae.safetensors" ]; then
echo " Already exists, skipping."
else
HF_XET_HIGH_PERFORMANCE=1 huggingface-cli download \
Comfy-Org/z_image_turbo \
split_files/vae/ae.safetensors \
--local-dir "$COMFY/models/vae/" \
--local-dir-use-symlinks False
# Move from subdirectory if needed
if [ -f "$COMFY/models/vae/split_files/vae/ae.safetensors" ]; then
mv "$COMFY/models/vae/split_files/vae/ae.safetensors" "$COMFY/models/vae/ae.safetensors"
rm -rf "$COMFY/models/vae/split_files"
fi
echo " Done."
fi
echo ""
echo "=========================================="
echo " Model Download Summary"
echo "=========================================="
echo "Diffusion model:"
ls -lh "$COMFY/models/diffusion_models/"*.gguf 2>/dev/null || echo " NOT FOUND"
echo "Text encoder:"
ls -lh "$COMFY/models/text_encoders/"*.gguf 2>/dev/null || echo " NOT FOUND"
echo "VAE:"
ls -lh "$COMFY/models/vae/"*.safetensors 2>/dev/null || echo " NOT FOUND"
echo ""
echo "Total model size:"
du -sh "$COMFY/models/"
echo ""
echo "DOWNLOAD_COMPLETE"
echo "Finished: $(date)"
'''
# Upload download script
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/download_models.sh', 'w') as f:
f.write(dl_script)
sftp.close()
run("chmod +x /home/fabian/download_models.sh", desc="Make download script executable")
run("rm -f /home/fabian/model_download.log", desc="Clean old log")
# Start download in background
run("bash -c 'nohup bash /home/fabian/download_models.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
desc="Start model download in background")
# Wait and check progress
time.sleep(10)
run("tail -20 /home/fabian/model_download.log 2>/dev/null || echo 'Waiting for log...'",
desc="Initial download progress")
# Keep checking
for i in range(6):
time.sleep(30)
rc, out, _ = run(f"tail -10 /home/fabian/model_download.log 2>/dev/null",
desc=f"Download progress check {i+1}")
if 'DOWNLOAD_COMPLETE' in out:
print("\n ALL DOWNLOADS COMPLETE!")
break
# Final check
run("tail -20 /home/fabian/model_download.log 2>/dev/null",
desc="Final download status")
run("du -sh ~/ComfyUI/models/diffusion_models/ ~/ComfyUI/models/text_encoders/ ~/ComfyUI/models/vae/ 2>/dev/null",
desc="Model directory sizes")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_step5b_models.py
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Download Z-Image-Turbo GGUF models on BC-250 using wget."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=7200, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 30:
print(f" ... ({len(lines)} lines, showing last 30)")
print('\n'.join(lines[-30:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-15:] if len(lines) > 15 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
dl_script = r'''#!/bin/bash
set -euo pipefail
LOG="/home/fabian/model_download.log"
exec > >(tee -a "$LOG") 2>&1
COMFY="$HOME/ComfyUI"
echo "=========================================="
echo " Downloading Z-Image-Turbo GGUF Models"
echo " Started: $(date)"
echo "=========================================="
# 1. Diffusion model (5.19 GB)
echo ""
echo "[1/3] Downloading z_image_turbo-Q5_K_S.gguf (5.19 GB)..."
DEST1="$COMFY/models/diffusion_models/z_image_turbo-Q5_K_S.gguf"
if [ -f "$DEST1" ]; then
echo " Already exists ($(du -h "$DEST1" | cut -f1)), skipping."
else
wget -c -q --show-progress \
"https://huggingface.co/jayn7/Z-Image-Turbo-GGUF/resolve/main/z_image_turbo-Q5_K_S.gguf" \
-O "$DEST1.tmp"
mv "$DEST1.tmp" "$DEST1"
echo " Done: $(du -h "$DEST1" | cut -f1)"
fi
# 2. Text encoder Qwen3-4B (2.82 GB)
echo ""
echo "[2/3] Downloading Qwen3-4B.i1-Q5_K_S.gguf (2.82 GB)..."
DEST2="$COMFY/models/text_encoders/Qwen3-4B.i1-Q5_K_S.gguf"
if [ -f "$DEST2" ]; then
echo " Already exists ($(du -h "$DEST2" | cut -f1)), skipping."
else
wget -c -q --show-progress \
"https://huggingface.co/mradermacher/Qwen3-4B-i1-GGUF/resolve/main/Qwen3-4B.i1-Q5_K_S.gguf" \
-O "$DEST2.tmp"
mv "$DEST2.tmp" "$DEST2"
echo " Done: $(du -h "$DEST2" | cut -f1)"
fi
# 3. VAE (335 MB)
echo ""
echo "[3/3] Downloading ae.safetensors (VAE, 335 MB)..."
DEST3="$COMFY/models/vae/ae.safetensors"
if [ -f "$DEST3" ]; then
echo " Already exists ($(du -h "$DEST3" | cut -f1)), skipping."
else
wget -c -q --show-progress \
"https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/vae/ae.safetensors" \
-O "$DEST3.tmp"
mv "$DEST3.tmp" "$DEST3"
echo " Done: $(du -h "$DEST3" | cut -f1)"
fi
echo ""
echo "=========================================="
echo " Model Download Summary"
echo "=========================================="
echo "Diffusion model:"
ls -lh "$COMFY/models/diffusion_models/"*.gguf 2>/dev/null || echo " NOT FOUND"
echo "Text encoder:"
ls -lh "$COMFY/models/text_encoders/"*.gguf 2>/dev/null || echo " NOT FOUND"
echo "VAE:"
ls -lh "$COMFY/models/vae/"*.safetensors 2>/dev/null || echo " NOT FOUND"
echo ""
echo "Total model size:"
du -sh "$COMFY/models/"
echo ""
echo "DOWNLOAD_COMPLETE"
echo "Finished: $(date)"
'''
# Upload download script
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/download_models.sh', 'w') as f:
f.write(dl_script)
sftp.close()
run("chmod +x /home/fabian/download_models.sh", desc="Make script executable")
run("rm -f /home/fabian/model_download.log", desc="Clean old log")
run("bash -c 'which wget'", desc="Verify wget exists")
# Start download in background via nohup
run("bash -c 'nohup bash /home/fabian/download_models.sh </dev/null >/dev/null 2>&1 & echo PID=$!'",
desc="Start model download in background")
time.sleep(15)
run("tail -20 /home/fabian/model_download.log 2>/dev/null || echo 'Waiting for log...'",
desc="Initial download progress")
# Monitor download progress - check every 60s for up to 30 minutes
for i in range(30):
time.sleep(60)
rc, out, _ = run(f"tail -5 /home/fabian/model_download.log 2>/dev/null; echo '---'; "
f"ls -lh ~/ComfyUI/models/diffusion_models/ ~/ComfyUI/models/text_encoders/ ~/ComfyUI/models/vae/ 2>/dev/null",
desc=f"Progress check {i+1}/30 ({(i+1)}min)")
if 'DOWNLOAD_COMPLETE' in out:
print("\n ALL DOWNLOADS COMPLETE!")
break
# Check if background process still running
_, pout, _ = run("bash -c 'pgrep -f download_models.sh || echo NOPROCESS'")
if 'NOPROCESS' in pout and 'DOWNLOAD_COMPLETE' not in out:
print("\n WARNING: Download process exited without completion!")
run("cat /home/fabian/model_download.log", desc="Full download log")
break
# Final verification
run("tail -25 /home/fabian/model_download.log 2>/dev/null", desc="Final download status")
run("du -sh ~/ComfyUI/models/diffusion_models/ ~/ComfyUI/models/text_encoders/ ~/ComfyUI/models/vae/",
desc="Model directory sizes")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_step6_launch.py
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""Create ComfyUI startup script and launch it on BC-250."""
import paramiko
import time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian', key_filename=r'C:\Users\fabia\.ssh\id_ed25519')
def run(cmd, timeout=120, desc=""):
if desc:
print(f"\n{'='*60}")
print(f" {desc}")
print(f"{'='*60}")
print(f"$ {cmd}")
_, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode()
err = stderr.read().decode()
rc = stdout.channel.recv_exit_status()
if out.strip():
lines = out.strip().split('\n')
if len(lines) > 40:
print(f" ... ({len(lines)} lines, showing last 40)")
print('\n'.join(lines[-40:]))
else:
print(out.strip())
if err.strip():
lines = err.strip().split('\n')
show = lines[-20:] if len(lines) > 20 else lines
print(f"STDERR: {chr(10).join(show)}")
print(f" Exit code: {rc}")
return rc, out, err
# ──────────────────────────────────────────────────────────
# 1. Create the startup script
# ──────────────────────────────────────────────────────────
startup_script = r'''#!/bin/bash
# ComfyUI Startup Script for AsRock BC-250 (AMD Cyan Skillfish / ROCm 7.2)
# Usage: ~/start_comfyui.sh [--listen] [--port PORT]
set -euo pipefail
# ═══════════════════════════════════════════════════════════
# BC-250 AMD GPU Environment Variables
# ═══════════════════════════════════════════════════════════
# Override gfx1013 → gfx1010 (RDNA 1.5 → RDNA 1 compat)
export HSA_OVERRIDE_GFX_VERSION=10.1.0
# Use device 0
export HIP_VISIBLE_DEVICES=0
# Disable SDMA (avoids queue errors on Cyan Skillfish)
export HSA_ENABLE_SDMA=0
# Suppress tool library warnings
export HSA_TOOLS_LIB=""
export HSA_TOOLS_REPORT_LOAD_FAILURE=0
# PyTorch / ROCm tuning
export PYTORCH_HIP_ALLOC_CONF="expandable_segments:True"
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
# Avoid OOM on 14GB shared VRAM — force float16 where possible
export COMFY_PRECISION=16
# ═══════════════════════════════════════════════════════════
# Activate Virtual Environment
# ═══════════════════════════════════════════════════════════
source "$HOME/comfyui-env/bin/activate"
# ═══════════════════════════════════════════════════════════
# Launch ComfyUI
# ═══════════════════════════════════════════════════════════
cd "$HOME/ComfyUI"
echo "=========================================="
echo " ComfyUI on BC-250 (ROCm 7.2)"
echo "=========================================="
echo " GPU: AMD Cyan Skillfish (gfx1013→gfx1010)"
echo " PyTorch: $(python -c 'import torch; print(torch.__version__)')"
echo " HIP: $(python -c 'import torch; print(torch.version.hip)')"
echo " CUDA: $(python -c 'import torch; print(torch.cuda.is_available())')"
echo " Device: $(python -c 'import torch; print(torch.cuda.get_device_name(0) if torch.cuda.is_available() else "N/A")')"
echo "=========================================="
# Default: listen on all interfaces for remote access
LISTEN_ARGS="--listen 0.0.0.0 --port 8188"
# Parse arguments (override defaults if provided)
if [ $# -gt 0 ]; then
LISTEN_ARGS="$@"
fi
echo ""
echo "Starting ComfyUI with: $LISTEN_ARGS"
echo "Access at: http://$(hostname -I | awk '{print $1}'):8188"
echo ""
exec python main.py $LISTEN_ARGS
'''
# ──────────────────────────────────────────────────────────
# 2. Create fish shell wrapper too
# ──────────────────────────────────────────────────────────
fish_script = r'''#!/usr/bin/env fish
# ComfyUI launcher for fish shell on BC-250
# BC-250 GPU env vars
set -gx HSA_OVERRIDE_GFX_VERSION 10.1.0
set -gx HIP_VISIBLE_DEVICES 0
set -gx HSA_ENABLE_SDMA 0
set -gx HSA_TOOLS_LIB ""
set -gx HSA_TOOLS_REPORT_LOAD_FAILURE 0
set -gx PYTORCH_HIP_ALLOC_CONF "expandable_segments:True"
set -gx TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL 1
# Activate venv
source $HOME/comfyui-env/bin/activate.fish
# Launch
cd $HOME/ComfyUI
echo "Starting ComfyUI on BC-250..."
python main.py --listen 0.0.0.0 --port 8188 $argv
'''
# Upload scripts
sftp = ssh.open_sftp()
with sftp.open('/home/fabian/start_comfyui.sh', 'w') as f:
f.write(startup_script)
with sftp.open('/home/fabian/start_comfyui.fish', 'w') as f:
f.write(fish_script)
sftp.close()
run("chmod +x /home/fabian/start_comfyui.sh /home/fabian/start_comfyui.fish",
desc="Make startup scripts executable")
# ──────────────────────────────────────────────────────────
# 3. Quick pre-flight check
# ──────────────────────────────────────────────────────────
run("bash -c 'source /home/fabian/comfyui-env/bin/activate && "
"export HSA_OVERRIDE_GFX_VERSION=10.1.0 && "
"export HIP_VISIBLE_DEVICES=0 && "
"export HSA_ENABLE_SDMA=0 && "
"cd /home/fabian/ComfyUI && "
"python -c \""
"import torch; "
"print(f\\\"PyTorch {torch.__version__}, HIP {torch.version.hip}, CUDA {torch.cuda.is_available()}\\\"); "
"print(f\\\"Device: {torch.cuda.get_device_name(0)}\\\"); "
"import comfy.model_management; "
"print(f\\\"ComfyUI model_management imported OK\\\"); "
"\"'",
desc="Pre-flight: PyTorch + ComfyUI import check")
# ──────────────────────────────────────────────────────────
# 4. Launch ComfyUI in background
# ──────────────────────────────────────────────────────────
run("bash -c 'pkill -f \"python main.py\" 2>/dev/null; echo killed || echo no_existing'",
desc="Kill any existing ComfyUI process")
time.sleep(2)
run("bash -c 'nohup bash /home/fabian/start_comfyui.sh > /home/fabian/comfyui.log 2>&1 & echo PID=$!'",
desc="Launch ComfyUI in background")
# Wait for startup
time.sleep(15)
run("tail -30 /home/fabian/comfyui.log 2>/dev/null", desc="ComfyUI startup log")
# Check if port is listening
time.sleep(10)
run("bash -c 'ss -tlnp | grep 8188 || echo PORT_NOT_LISTENING'",
desc="Check if port 8188 is listening")
run("tail -50 /home/fabian/comfyui.log 2>/dev/null", desc="Full startup log")
ssh.close()
print("\nDone.")
ComfyUI Scripts/bc250_submit.py
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""Fix workflow submission: write JSON to file, curl from file, monitor."""
import paramiko, json, time
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('192.168.178.150', username='fabian',
key_filename=r'C:\Users\fabia\.ssh\id_ed25519', timeout=10)
def run(cmd, timeout=60):
_, so, se = ssh.exec_command(cmd, timeout=timeout)
return so.read().decode(), se.read().decode()
try:
# Step 1: Write workflow JSON via SFTP (reliable, no shell escaping)
workflow = {
"prompt": {
"1": {"class_type": "UnetLoaderGGUF",
"inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
"2": {"class_type": "CLIPLoaderGGUF",
"inputs": {"clip_name": "Qwen3-4B.i1-Q5_K_S.gguf", "type": "qwen_image"}},
"3": {"class_type": "VAELoader",
"inputs": {"vae_name": "ae.safetensors"}},
"4": {"class_type": "CLIPTextEncode",
"inputs": {"text": "A red fox in a snowy forest, photorealistic", "clip": ["2", 0]}},
"5": {"class_type": "CLIPTextEncode",
"inputs": {"text": "", "clip": ["2", 0]}},
"6": {"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
"7": {"class_type": "KSampler",
"inputs": {"model": ["1", 0], "seed": 42, "steps": 8, "cfg": 1.0,
"sampler_name": "euler", "scheduler": "simple",
"positive": ["4", 0], "negative": ["5", 0],
"latent_image": ["6", 0], "denoise": 1.0}},
"8": {"class_type": "VAEDecode",
"inputs": {"samples": ["7", 0], "vae": ["3", 0]}},
"9": {"class_type": "SaveImage",
"inputs": {"images": ["8", 0], "filename_prefix": "ZImageTurbo_test"}}
}
}
sftp = ssh.open_sftp()
with sftp.open('/tmp/wf.json', 'w') as f:
f.write(json.dumps(workflow))
sftp.close()
print("Workflow JSON written via SFTP.")
# Verify JSON is valid
out, _ = run("python3 -c \"import json; d=json.load(open('/tmp/wf.json')); print('nodes:', sorted(d['prompt'].keys()))\"")
print(f"Verify: {out.strip()}")
# Step 2: Check startup script has --cpu-vae
out, _ = run("cat /home/fabian/start_comfyui.sh")
has_cpu_vae = '--cpu-vae' in out
print(f"Startup has --cpu-vae: {has_cpu_vae}")
print(f"Startup has --novram: {'--novram' in out}")
print(f"Startup has --force-fp16: {'--force-fp16' in out}")
# Step 3: Submit
out, err = run("curl -s -X POST http://localhost:8188/prompt -H 'Content-Type: application/json' -d @/tmp/wf.json")
print(f"\nSubmit response: {out.strip()[:500]}")
try:
resp = json.loads(out.strip())
except:
print(f"Failed to parse response!")
raise SystemExit(1)
if 'error' in resp:
print(f"\nAPI ERROR: {resp['error']}")
print(f"Details: {resp.get('details','')}")
print(f"Node errors: {resp.get('node_errors',{})}")
raise SystemExit(1)
prompt_id = resp.get('prompt_id', 'unknown')
print(f"Prompt ID: {prompt_id}")
# Step 4: Monitor (15s intervals, up to 30 min)
print("\nMonitoring generation (GPU sampling + CPU VAE)...")
last_log = ""
for i in range(120):
time.sleep(15)
stats, _ = run("bash -c 'PID=$(pgrep -f \"python3 main.py\" | head -1); "
"if [ -n \"$PID\" ]; then "
" CPU=$(ps -p $PID -o %cpu --no-headers); "
" MEM=$(ps -p $PID -o rss --no-headers); "
" echo \"CPU:${CPU}% RSS:$((MEM/1024))M LOAD:$(cut -d\" \" -f1-3 /proc/loadavg)\"; "
"else echo DEAD; fi'")
log, _ = run("tail -8 /home/fabian/comfyui.log 2>/dev/null")
m, s = divmod((i+1)*15, 60)
print(f" [{m}m{s:02d}s] {stats.strip()}")
if log.strip() != last_log:
for line in reversed(log.strip().split('\n')):
l = line.strip()
if l and not l.startswith('FETCH') and not l.startswith('[DEPRECATION') and not l.startswith('[ComfyUI-Manager]'):
print(f" LOG: {l[:120]}")
break
last_log = log.strip()
if 'DEAD' in stats:
print("\nPROCESS DIED!")
out, _ = run("tail -50 /home/fabian/comfyui.log")
print(out)
break
if 'Prompt executed in' in log:
print("\nSUCCESS! Image generated!")
out, _ = run("tail -20 /home/fabian/comfyui.log")
print(out)
break
if 'Traceback' in log or 'RuntimeError' in log:
print("\nERROR detected!")
out, _ = run("tail -50 /home/fabian/comfyui.log")
print(out)
break
# Output files
print("\n=== Output files ===")
out, _ = run("ls -lah /home/fabian/ComfyUI/output/")
print(out.strip())
finally:
ssh.close()
print("\nSSH closed.")

Some files were not shown because too many files have changed in this diff Show More