Fabian
119 lines
4.0 KiB
Python
119 lines
4.0 KiB
Python
"""Upload a self-contained bash script and run it in ONE SSH session."""
|
|
import paramiko, time
|
|
|
|
k = paramiko.Ed25519Key.from_private_key_file(r'C:\Users\fabia\.ssh\id_ed25519')
|
|
c = paramiko.SSHClient()
|
|
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
c.connect('192.168.178.150', username='fabian', pkey=k, timeout=10)
|
|
sftp = c.open_sftp()
|
|
|
|
# Write the entire fix+restart+test as ONE bash script
|
|
script = r'''#!/bin/bash
|
|
set -e
|
|
|
|
# Kill any running ComfyUI
|
|
pkill -9 -f "python3.*main.py" 2>/dev/null || true
|
|
sleep 2
|
|
|
|
# Clean
|
|
rm -f /tmp/comfyui.log
|
|
rm -f ~/ComfyUI/output/ZImageTurbo_GPU*.png
|
|
|
|
# Start ComfyUI
|
|
echo "Starting ComfyUI..."
|
|
nohup /tmp/run_comfyui.sh > /tmp/comfyui.log 2>&1 &
|
|
sleep 3
|
|
PID=$(pgrep -f "python3.*main.py" | head -1)
|
|
echo "PID: $PID"
|
|
|
|
if [ -z "$PID" ]; then
|
|
echo "FAILED TO START!"
|
|
cat /tmp/comfyui.log 2>/dev/null | tail -20
|
|
exit 1
|
|
fi
|
|
|
|
# Wait for HTTP
|
|
echo "Waiting for HTTP..."
|
|
for i in $(seq 1 90); do
|
|
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8188/ 2>/dev/null || echo 000)
|
|
if [ "$CODE" = "200" ]; then
|
|
echo "Ready after ${i}s"
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Show key startup info
|
|
grep -E "vram state|SHARED|Device:|Total VRAM|load device|offload" /tmp/comfyui.log 2>/dev/null || true
|
|
|
|
# Submit workflow
|
|
echo ""
|
|
echo "Submitting workflow..."
|
|
cat > /tmp/wf.json << 'WFEOF'
|
|
{"prompt":{"1":{"class_type":"UnetLoaderGGUF","inputs":{"unet_name":"z_image_turbo-Q5_K_S.gguf"}},"2":{"class_type":"CLIPLoaderGGUF","inputs":{"clip_name":"Qwen3-4B.i1-Q5_K_S.gguf","type":"qwen_image"}},"3":{"class_type":"VAELoader","inputs":{"vae_name":"ae.safetensors"}},"4":{"class_type":"CLIPTextEncode","inputs":{"text":"A red fox in a snowy forest, photorealistic","clip":["2",0]}},"5":{"class_type":"EmptyLatentImage","inputs":{"width":512,"height":512,"batch_size":1}},"6":{"class_type":"KSampler","inputs":{"model":["1",0],"positive":["4",0],"negative":["4",0],"latent_image":["5",0],"seed":99999,"steps":8,"cfg":1.0,"sampler_name":"euler","scheduler":"simple","denoise":1.0}},"7":{"class_type":"VAEDecode","inputs":{"samples":["6",0],"vae":["3",0]}},"8":{"class_type":"SaveImage","inputs":{"images":["7",0],"filename_prefix":"ZImageTurbo_GPU"}}}}
|
|
WFEOF
|
|
|
|
RESP=$(curl -s -X POST http://127.0.0.1:8188/prompt -H "Content-Type: application/json" -d @/tmp/wf.json)
|
|
echo "Response: ${RESP:0:120}"
|
|
|
|
# Monitor
|
|
echo ""
|
|
echo "Monitoring..."
|
|
START=$(date +%s)
|
|
LAST=""
|
|
while true; do
|
|
NOW=$(date +%s)
|
|
ELAPSED=$((NOW - START))
|
|
|
|
if [ $ELAPSED -gt 600 ]; then
|
|
echo "TIMEOUT after 600s"
|
|
tail -20 /tmp/comfyui.log
|
|
break
|
|
fi
|
|
|
|
# Check for output image
|
|
if ls ~/ComfyUI/output/ZImageTurbo_GPU*.png 2>/dev/null; then
|
|
echo ""
|
|
echo "*** IMAGE DONE in ${ELAPSED}s! ***"
|
|
grep -E "loaded|load device|offload|Prompt executed|/8" /tmp/comfyui.log 2>/dev/null | grep -v FETCH || true
|
|
break
|
|
fi
|
|
|
|
# Show progress
|
|
LINE=$(grep -E "/8|loaded|Requested|VAE|Prompt executed|Error|OOM" /tmp/comfyui.log 2>/dev/null | grep -v FETCH | grep -v audio_vae | grep -v "split attention" | tail -1)
|
|
GPU=$(cat /sys/class/drm/card0/device/gpu_busy_percent 2>/dev/null || echo "?")
|
|
|
|
if [ "$LINE" != "$LAST" ] && [ -n "$LINE" ]; then
|
|
echo "[${ELAPSED}s] GPU:${GPU}% ${LINE:0:110}"
|
|
LAST="$LINE"
|
|
fi
|
|
|
|
# Check alive
|
|
if ! pgrep -f "python3.*main.py" > /dev/null 2>&1; then
|
|
echo "CRASHED at ${ELAPSED}s!"
|
|
tail -20 /tmp/comfyui.log
|
|
break
|
|
fi
|
|
|
|
sleep 3
|
|
done
|
|
'''
|
|
|
|
with sftp.open('/tmp/fix_and_run.sh', 'w') as f:
|
|
f.write(script)
|
|
sftp.close()
|
|
|
|
# Execute in ONE session
|
|
print("Running fix+restart+monitor on BC-250...")
|
|
stdin, stdout, stderr = c.exec_command('bash /tmp/fix_and_run.sh', timeout=660)
|
|
# Stream output
|
|
for line in iter(stdout.readline, ''):
|
|
print(line.rstrip())
|
|
err = stderr.read().decode(errors='replace').strip()
|
|
if err:
|
|
for l in err.split('\n')[-10:]:
|
|
if l.strip(): print(f"STDERR: {l.strip()}")
|
|
|
|
c.close()
|
|
print("\nDone.")
|