Uploaded sanitized BC250/ROCm Repository.
This commit is contained in:
Fabian
@@ -0,0 +1,378 @@
|
||||
{
|
||||
"id": "bc250-z-image-turbo-default",
|
||||
"revision": 0,
|
||||
"last_node_id": 9,
|
||||
"last_link_id": 9,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "CLIPLoaderGGUF",
|
||||
"pos": [100, 200],
|
||||
"size": [300, 82],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "clip_name",
|
||||
"type": "COMBO",
|
||||
"widget": {"name": "clip_name"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "type",
|
||||
"type": "COMBO",
|
||||
"widget": {"name": "type"},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CLIP",
|
||||
"type": "CLIP",
|
||||
"slot_index": 0,
|
||||
"links": [1, 2]
|
||||
}
|
||||
],
|
||||
"properties": {"Node name for S&R": "CLIPLoaderGGUF"},
|
||||
"widgets_values": ["Qwen_3_4b-Q8_0.gguf", "lumina2"]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPTextEncode",
|
||||
"pos": [500, 150],
|
||||
"size": [400, 120],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "STRING",
|
||||
"widget": {"name": "text"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CONDITIONING",
|
||||
"type": "CONDITIONING",
|
||||
"slot_index": 0,
|
||||
"links": [4]
|
||||
}
|
||||
],
|
||||
"title": "Positive Prompt",
|
||||
"properties": {"Node name for S&R": "CLIPTextEncode"},
|
||||
"widgets_values": ["a highly detailed photograph of a beautiful landscape, mountains, lake, sunset, golden hour, dramatic clouds, sharp focus, 8k, cinematic lighting"]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "CLIPTextEncode",
|
||||
"pos": [500, 350],
|
||||
"size": [400, 120],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "STRING",
|
||||
"widget": {"name": "text"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": 2
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CONDITIONING",
|
||||
"type": "CONDITIONING",
|
||||
"slot_index": 0,
|
||||
"links": [5]
|
||||
}
|
||||
],
|
||||
"title": "Negative Prompt",
|
||||
"properties": {"Node name for S&R": "CLIPTextEncode"},
|
||||
"widgets_values": [""]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "UnetLoaderGGUF",
|
||||
"pos": [100, 450],
|
||||
"size": [300, 58],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "unet_name",
|
||||
"type": "COMBO",
|
||||
"widget": {"name": "unet_name"},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "MODEL",
|
||||
"type": "MODEL",
|
||||
"slot_index": 0,
|
||||
"links": [3]
|
||||
}
|
||||
],
|
||||
"properties": {"Node name for S&R": "UnetLoaderGGUF"},
|
||||
"widgets_values": ["z_image_turbo-Q5_K_S.gguf"]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "EmptyLatentImage",
|
||||
"pos": [500, 550],
|
||||
"size": [300, 106],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "width",
|
||||
"type": "INT",
|
||||
"widget": {"name": "width"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": "INT",
|
||||
"widget": {"name": "height"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "batch_size",
|
||||
"type": "INT",
|
||||
"widget": {"name": "batch_size"},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "LATENT",
|
||||
"type": "LATENT",
|
||||
"slot_index": 0,
|
||||
"links": [6]
|
||||
}
|
||||
],
|
||||
"properties": {"Node name for S&R": "EmptyLatentImage"},
|
||||
"widgets_values": [512, 512, 1]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "KSampler",
|
||||
"pos": [1000, 200],
|
||||
"size": [300, 262],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MODEL",
|
||||
"link": 3
|
||||
},
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"link": 4
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"link": 5
|
||||
},
|
||||
{
|
||||
"name": "latent_image",
|
||||
"type": "LATENT",
|
||||
"link": 6
|
||||
},
|
||||
{
|
||||
"name": "seed",
|
||||
"type": "INT",
|
||||
"widget": {"name": "seed"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "steps",
|
||||
"type": "INT",
|
||||
"widget": {"name": "steps"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "cfg",
|
||||
"type": "FLOAT",
|
||||
"widget": {"name": "cfg"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "sampler_name",
|
||||
"type": "COMBO",
|
||||
"widget": {"name": "sampler_name"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "scheduler",
|
||||
"type": "COMBO",
|
||||
"widget": {"name": "scheduler"},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "denoise",
|
||||
"type": "FLOAT",
|
||||
"widget": {"name": "denoise"},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "LATENT",
|
||||
"type": "LATENT",
|
||||
"slot_index": 0,
|
||||
"links": [7]
|
||||
}
|
||||
],
|
||||
"properties": {"Node name for S&R": "KSampler"},
|
||||
"widgets_values": [42, "randomize", 4, 1.0, "euler", "normal", 1.0]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "VAELoader",
|
||||
"pos": [1000, 550],
|
||||
"size": [250, 58],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "vae_name",
|
||||
"type": "COMBO",
|
||||
"widget": {"name": "vae_name"},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "VAE",
|
||||
"type": "VAE",
|
||||
"slot_index": 0,
|
||||
"links": [8]
|
||||
}
|
||||
],
|
||||
"properties": {"Node name for S&R": "VAELoader"},
|
||||
"widgets_values": ["ae.safetensors"]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "VAEDecode",
|
||||
"pos": [1400, 300],
|
||||
"size": [200, 46],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "samples",
|
||||
"type": "LATENT",
|
||||
"link": 7
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 8
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"slot_index": 0,
|
||||
"links": [9]
|
||||
}
|
||||
],
|
||||
"properties": {"Node name for S&R": "VAEDecode"},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "SaveImage",
|
||||
"pos": [1650, 250],
|
||||
"size": [300, 270],
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 9
|
||||
},
|
||||
{
|
||||
"name": "filename_prefix",
|
||||
"type": "STRING",
|
||||
"widget": {"name": "filename_prefix"},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {"Node name for S&R": "SaveImage"},
|
||||
"widgets_values": ["BC250"]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 0, 2, 1, "CLIP"],
|
||||
[2, 1, 0, 3, 1, "CLIP"],
|
||||
[3, 4, 0, 6, 0, "MODEL"],
|
||||
[4, 2, 0, 6, 1, "CONDITIONING"],
|
||||
[5, 3, 0, 6, 2, "CONDITIONING"],
|
||||
[6, 5, 0, 6, 3, "LATENT"],
|
||||
[7, 6, 0, 8, 0, "LATENT"],
|
||||
[8, 7, 0, 8, 1, "VAE"],
|
||||
[9, 8, 0, 9, 0, "IMAGE"]
|
||||
],
|
||||
"groups": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "BC-250 Z-Image-Turbo (GGUF)",
|
||||
"bounding": [70, 100, 560, 580],
|
||||
"color": "#3f789e",
|
||||
"font_size": 24,
|
||||
"flags": {}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Sampling",
|
||||
"bounding": [960, 130, 380, 520],
|
||||
"color": "#8A8",
|
||||
"font_size": 24,
|
||||
"flags": {}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Decode & Save",
|
||||
"bounding": [1360, 230, 620, 180],
|
||||
"color": "#A88",
|
||||
"font_size": 24,
|
||||
"flags": {}
|
||||
}
|
||||
],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.8,
|
||||
"offset": [50, -50]
|
||||
}
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
curl -s http://localhost:9090/ | grep -oP 'data-page="[^"]*"'
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
echo "=== MEMORY ==="
|
||||
free -h
|
||||
echo
|
||||
echo "=== GPU VRAM ==="
|
||||
cat /sys/class/drm/card*/device/mem_info_vram_total 2>/dev/null || echo "no sysfs"
|
||||
echo
|
||||
echo "=== TOP MEM PROCS ==="
|
||||
ps aux --sort=-%mem | head -10
|
||||
echo
|
||||
echo "=== DOCKER ==="
|
||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" 2>&1
|
||||
echo
|
||||
echo "=== DOCKER STATS ==="
|
||||
docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" 2>&1
|
||||
echo
|
||||
echo "=== QWEN3-TTS MODELS ==="
|
||||
ls -lh ~/sudx-ai/models/qwen3-tts/ 2>&1
|
||||
echo
|
||||
echo "=== DISK ==="
|
||||
df -h /
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check registered Flask routes in dashboard"""
|
||||
import sys
|
||||
sys.path.insert(0, "/opt/dashboard")
|
||||
# Read the app.py source and exec it, but stop before app.run
|
||||
src = open("/opt/dashboard/app.py").read()
|
||||
# Find app.run and cut before it
|
||||
idx = src.find("app.run(")
|
||||
if idx > 0:
|
||||
exec(compile(src[:idx], "app.py", "exec"))
|
||||
else:
|
||||
exec(compile(src, "app.py", "exec"))
|
||||
|
||||
for rule in sorted(app.url_map.iter_rules(), key=lambda r: str(r)):
|
||||
if "tts" in str(rule) or "proxy" in str(rule):
|
||||
print(f"{rule.methods} {rule}")
|
||||
@@ -0,0 +1,9 @@
|
||||
import json
|
||||
d = json.load(open("/models/qwen3-tts/preset_speakers/vivian.json"))
|
||||
print("keys:", list(d.keys()))
|
||||
print("spk_emb len:", len(d.get("spk_emb", [])))
|
||||
codes = d.get("codes", [])
|
||||
print("codes shape:", len(codes), "x", len(codes[0]) if codes else 0)
|
||||
text_ids = d.get("text_ids", [])
|
||||
print("text_ids len:", len(text_ids))
|
||||
print("text:", d.get("text", "")[:80])
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Debug proxy_tts 400 error"""
|
||||
import sys, traceback
|
||||
sys.path.insert(0, "/opt/dashboard")
|
||||
|
||||
# Read app source
|
||||
src = open("/opt/dashboard/app.py").read()
|
||||
idx = src.find("app.run(")
|
||||
exec(compile(src[:idx], "app.py", "exec"))
|
||||
|
||||
# Use Flask test client
|
||||
with app.test_client() as c:
|
||||
print("=== Test 1: POST with JSON ===")
|
||||
r = c.post("/api/proxy/tts", json={"input":"Hello test","voice":"vivian","language":"en","speed":1.0})
|
||||
print(f"Status: {r.status_code}")
|
||||
if r.status_code != 200:
|
||||
print(f"Body: {r.data[:500]}")
|
||||
else:
|
||||
print(f"Content-Type: {r.content_type}, Size: {len(r.data)}")
|
||||
|
||||
print("\n=== Test 2: POST with raw JSON string ===")
|
||||
r = c.post("/api/proxy/tts", data='{"input":"Hello"}', content_type="application/json")
|
||||
print(f"Status: {r.status_code}")
|
||||
if r.status_code != 200:
|
||||
print(f"Body: {r.data[:500]}")
|
||||
else:
|
||||
print(f"Content-Type: {r.content_type}, Size: {len(r.data)}")
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Diagnose qwen3-tts engine init - run inside container"""
|
||||
import sys, os, time, traceback
|
||||
os.chdir("/models")
|
||||
sys.path.insert(0, "/opt/qwen3-tts")
|
||||
os.environ["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
print("=== Step 1: Assets + Tokenizer ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.assets import AssetsManager
|
||||
from tokenizers import Tokenizer
|
||||
assets = AssetsManager("qwen3-tts")
|
||||
tok = Tokenizer.from_file("/models/qwen3-tts/tokenizer.json")
|
||||
print(f" OK: assets loaded, vocab_size={tok.get_vocab_size()}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== Step 2: Codec + Speaker Encoders ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.encoder import CodecEncoder, SpeakerEncoder
|
||||
ce = CodecEncoder("/models/qwen3-tts/qwen3_tts_codec_encoder.fp16.onnx")
|
||||
se = SpeakerEncoder("/models/qwen3-tts/qwen3_tts_speaker_encoder.fp16.onnx")
|
||||
print(f" OK: codec_encoder + speaker_encoder loaded", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== Step 3: DecoderProxy ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.decoder import DecoderProxy
|
||||
t0 = time.time()
|
||||
dec = DecoderProxy("/models/qwen3-tts/qwen3_tts_decoder.fp16.onnx", onnx_provider="CPUExecutionProvider", chunk_size=2048)
|
||||
print(f" DecoderProxy created in {time.time()-t0:.2f}s", flush=True)
|
||||
print(f" Waiting for ready (20s timeout)...", flush=True)
|
||||
ready = dec.wait_until_ready(timeout=20)
|
||||
print(f" ready={ready}, states={getattr(dec, 'ready_states', 'N/A')}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== Step 4: GGUF / llama.cpp ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference import llama
|
||||
print(f" llama module loaded: {dir(llama)}", flush=True)
|
||||
t_path = "qwen3-tts/qwen3_tts_talker.q5_k.gguf"
|
||||
p_path = "qwen3-tts/qwen3_tts_predictor.q8_0.gguf"
|
||||
print(f" Loading talker from {t_path}...", flush=True)
|
||||
t0 = time.time()
|
||||
talker = llama.LlamaModel(t_path, n_gpu_layers=-1)
|
||||
print(f" Talker loaded in {time.time()-t0:.2f}s", flush=True)
|
||||
print(f" Loading predictor from {p_path}...", flush=True)
|
||||
t0 = time.time()
|
||||
predictor = llama.LlamaModel(p_path, n_gpu_layers=-1)
|
||||
print(f" Predictor loaded in {time.time()-t0:.2f}s", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== Step 5: Full TTSEngine ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference import TTSEngine
|
||||
t0 = time.time()
|
||||
engine = TTSEngine(model_dir="qwen3-tts", onnx_provider="CPUExecutionProvider")
|
||||
print(f" Engine created in {time.time()-t0:.2f}s, ready={engine.ready}", flush=True)
|
||||
if engine.ready:
|
||||
stream = engine.create_stream(n_ctx=2048)
|
||||
print(f" stream={stream}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== DONE ===", flush=True)
|
||||
@@ -0,0 +1,18 @@
|
||||
import traceback, sys, os
|
||||
os.chdir("/models")
|
||||
sys.path.insert(0, "/opt/qwen3-tts")
|
||||
try:
|
||||
print("Importing proxy.DecoderProxy...", flush=True)
|
||||
from qwen3_tts_gguf.inference.proxy import DecoderProxy
|
||||
print(f"OK: {DecoderProxy}", flush=True)
|
||||
except Exception as e:
|
||||
print(f"FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
try:
|
||||
print("\nImporting engine directly...", flush=True)
|
||||
from qwen3_tts_gguf.inference.engine import TTSEngine
|
||||
print(f"OK: {TTSEngine}", flush=True)
|
||||
except Exception as e:
|
||||
print(f"FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Test DecoderProxy from proxy.py and full engine init with exception details"""
|
||||
import sys, os, time, traceback
|
||||
os.chdir("/models")
|
||||
sys.path.insert(0, "/opt/qwen3-tts")
|
||||
os.environ["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
print("=== Test DecoderProxy from proxy.py ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.proxy import DecoderProxy
|
||||
t0 = time.time()
|
||||
dec = DecoderProxy(
|
||||
"/models/qwen3-tts/qwen3_tts_decoder.fp16.onnx",
|
||||
onnx_provider="CPUExecutionProvider",
|
||||
chunk_size=2048
|
||||
)
|
||||
print(f" Constructor OK: {time.time()-t0:.2f}s", flush=True)
|
||||
print(f" Waiting for ready (25s)...", flush=True)
|
||||
ready = dec.wait_until_ready(timeout=25)
|
||||
print(f" ready={ready}", flush=True)
|
||||
if hasattr(dec, 'ready_states'):
|
||||
print(f" states: {dec.ready_states}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== Test GGUF loading ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference import llama
|
||||
t0 = time.time()
|
||||
talker = llama.LlamaModel("qwen3-tts/qwen3_tts_talker.q5_k.gguf", n_gpu_layers=-1)
|
||||
print(f" Talker OK: {time.time()-t0:.2f}s", flush=True)
|
||||
predictor = llama.LlamaModel("qwen3-tts/qwen3_tts_predictor.q8_0.gguf", n_gpu_layers=-1)
|
||||
print(f" Predictor OK", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== Full Engine (verbose, catch exception) ===", flush=True)
|
||||
try:
|
||||
# Monkey-patch to see the actual exception
|
||||
import qwen3_tts_gguf.inference.engine as eng_mod
|
||||
orig_init = eng_mod.TTSEngine.__init__
|
||||
def patched_init(self, *args, **kwargs):
|
||||
try:
|
||||
orig_init(self, *args, **kwargs)
|
||||
except Exception as e:
|
||||
print(f" !! Engine __init__ exception: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
raise
|
||||
eng_mod.TTSEngine.__init__ = patched_init
|
||||
|
||||
from qwen3_tts_gguf.inference import TTSEngine
|
||||
t0 = time.time()
|
||||
engine = TTSEngine(model_dir="qwen3-tts", onnx_provider="CPUExecutionProvider")
|
||||
elapsed = time.time() - t0
|
||||
print(f" Engine: ready={engine.ready}, took {elapsed:.2f}s", flush=True)
|
||||
print(f" has talker_model: {hasattr(engine, 'talker_model')}", flush=True)
|
||||
print(f" has predictor_model: {hasattr(engine, 'predictor_model')}", flush=True)
|
||||
print(f" has decoder: {hasattr(engine, 'decoder')}", flush=True)
|
||||
if hasattr(engine, 'decoder'):
|
||||
print(f" decoder type: {type(engine.decoder)}", flush=True)
|
||||
if hasattr(engine.decoder, 'ready_states'):
|
||||
print(f" decoder states: {engine.decoder.ready_states}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== DONE ===", flush=True)
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Diagnose TTSEngine init inside qwen3-tts container."""
|
||||
import sys, os, traceback
|
||||
|
||||
MODEL_DIR = "/models/qwen3-tts"
|
||||
|
||||
print("=== Step 1: Check model dir ===")
|
||||
if os.path.isdir(MODEL_DIR):
|
||||
for f in sorted(os.listdir(MODEL_DIR)):
|
||||
fp = os.path.join(MODEL_DIR, f)
|
||||
if os.path.isfile(fp):
|
||||
print(f" {f} ({os.path.getsize(fp):,} bytes)")
|
||||
else:
|
||||
print(f" {f}/")
|
||||
else:
|
||||
print(f" ERROR: {MODEL_DIR} does not exist!")
|
||||
sys.exit(1)
|
||||
|
||||
print("\n=== Step 2: Import qwen3_tts_gguf ===")
|
||||
try:
|
||||
import qwen3_tts_gguf
|
||||
print(f" OK: {qwen3_tts_gguf.__file__}")
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}")
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
print("\n=== Step 3: Import TTSEngine ===")
|
||||
try:
|
||||
from qwen3_tts_gguf.inference import TTSEngine
|
||||
print(f" OK: {TTSEngine}")
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}")
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
print("\n=== Step 4: Check inference bin dir ===")
|
||||
bin_dir = os.path.join(os.path.dirname(qwen3_tts_gguf.__file__), "inference", "bin")
|
||||
if os.path.isdir(bin_dir):
|
||||
for f in sorted(os.listdir(bin_dir)):
|
||||
print(f" {f}")
|
||||
else:
|
||||
print(f" WARN: {bin_dir} does not exist")
|
||||
|
||||
print("\n=== Step 5: Init TTSEngine ===")
|
||||
try:
|
||||
engine = TTSEngine(model_dir=MODEL_DIR)
|
||||
print(f" OK: engine={engine}")
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}")
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
print("\n=== DONE: Engine initialized successfully ===")
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check if server.py init_engine actually ran and what happened."""
|
||||
import subprocess, sys
|
||||
|
||||
# Check if there's a running python process and what it looks like
|
||||
result = subprocess.run(["ps", "aux"], capture_output=True, text=True)
|
||||
for line in result.stdout.split("\n"):
|
||||
if "python" in line.lower():
|
||||
print(line)
|
||||
|
||||
print("\n=== Test: replicate server.py init_engine exactly ===")
|
||||
import os
|
||||
MODEL_DIR = "/models/qwen3-tts"
|
||||
engine = None
|
||||
|
||||
def init_engine():
|
||||
global engine, MODEL_DIR
|
||||
print(f" init_engine called, MODEL_DIR={MODEL_DIR}")
|
||||
try:
|
||||
from qwen3_tts_gguf.inference import TTSEngine
|
||||
print(f" TTSEngine imported: {TTSEngine}")
|
||||
engine = TTSEngine(model_dir=MODEL_DIR)
|
||||
print(f" engine created: {engine}")
|
||||
print(f" engine is None: {engine is None}")
|
||||
print(f" bool(engine): {bool(engine)}")
|
||||
except Exception as e:
|
||||
print(f" EXCEPTION in init_engine: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
init_engine()
|
||||
print(f"\n=== Result: engine={engine}, bool(engine)={bool(engine) if engine else 'N/A (None)'} ===")
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deep diagnostic: replicate exact server.py init_engine() flow."""
|
||||
import os, sys
|
||||
from pathlib import Path
|
||||
|
||||
MODEL_DIR = "/models/qwen3-tts"
|
||||
|
||||
print(f"cwd = {Path.cwd()}")
|
||||
print(f"MODEL_DIR = {MODEL_DIR}")
|
||||
|
||||
# Replicate TTSEngine.__init__ path logic
|
||||
project_root = Path.cwd() # /home/ttsuser (WORKDIR)
|
||||
model_dir = project_root / MODEL_DIR
|
||||
print(f"project_root = {project_root}")
|
||||
print(f"model_dir (resolved) = {model_dir}")
|
||||
|
||||
paths = {
|
||||
"talker_gguf": model_dir / "qwen3_tts_talker.q5_k.gguf",
|
||||
"predictor_gguf": model_dir / "qwen3_tts_predictor.q8_0.gguf",
|
||||
"decoder_onnx": model_dir / "qwen3_tts_decoder.fp16.onnx",
|
||||
"codec_enc_onnx": model_dir / "qwen3_tts_codec_encoder.fp16.onnx",
|
||||
"spk_enc_onnx": model_dir / "qwen3_tts_speaker_encoder.fp16.onnx",
|
||||
"tokenizer": model_dir / "tokenizer.json",
|
||||
}
|
||||
|
||||
for name, p in paths.items():
|
||||
print(f" {name}: {p} -> exists={p.exists()}")
|
||||
|
||||
# Check the missing files check
|
||||
missing = [name for name, p in paths.items()
|
||||
if name in ["talker_gguf", "predictor_gguf", "decoder_onnx", "tokenizer"]
|
||||
and not p.exists()]
|
||||
print(f"\nmissing = {missing}")
|
||||
|
||||
if not missing:
|
||||
print("\n=== Testing relative_to ===")
|
||||
for name in ["talker_gguf", "predictor_gguf"]:
|
||||
try:
|
||||
rel = paths[name].relative_to(project_root).as_posix()
|
||||
print(f" {name} relative = {rel}")
|
||||
except ValueError as e:
|
||||
print(f" {name} relative_to FAILED: {e}")
|
||||
|
||||
print("\n=== Testing with chdir to parent ===")
|
||||
parent = os.path.dirname(MODEL_DIR) # /models
|
||||
basename = os.path.basename(MODEL_DIR) # qwen3-tts
|
||||
print(f" parent={parent}, basename={basename}")
|
||||
os.chdir(parent)
|
||||
print(f" new cwd = {Path.cwd()}")
|
||||
new_root = Path.cwd()
|
||||
new_model = new_root / basename
|
||||
for name in ["talker_gguf", "predictor_gguf"]:
|
||||
p = new_model / paths[name].name
|
||||
try:
|
||||
rel = p.relative_to(new_root).as_posix()
|
||||
print(f" {name} relative = {rel} (exists={p.exists()})")
|
||||
except ValueError as e:
|
||||
print(f" {name} relative_to FAILED: {e}")
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replicate exact server.py init_engine with chdir fix, max verbosity."""
|
||||
import os, sys, traceback
|
||||
sys.stdout = sys.stderr # ensure all output goes to same stream
|
||||
from pathlib import Path
|
||||
|
||||
MODEL_DIR = "/models/qwen3-tts"
|
||||
parent = os.path.dirname(os.path.abspath(MODEL_DIR))
|
||||
basename = os.path.basename(MODEL_DIR)
|
||||
|
||||
print(f"[diag] chdir to {parent}")
|
||||
os.chdir(parent)
|
||||
print(f"[diag] cwd now = {Path.cwd()}")
|
||||
print(f"[diag] basename = {basename}")
|
||||
|
||||
# Check all files with expected names
|
||||
model_path = Path.cwd() / basename
|
||||
for f in sorted(model_path.iterdir()):
|
||||
print(f" {f.name} {'(link)' if f.is_symlink() else ''}")
|
||||
|
||||
print(f"\n[diag] Importing TTSEngine...")
|
||||
from qwen3_tts_gguf.inference import TTSEngine
|
||||
|
||||
print(f"[diag] Creating TTSEngine(model_dir={basename!r}, onnx_provider='CPUExecutionProvider')...")
|
||||
try:
|
||||
engine = TTSEngine(model_dir=basename, onnx_provider="CPUExecutionProvider")
|
||||
print(f"\n[diag] engine.ready = {engine.ready}")
|
||||
print(f"[diag] bool(engine) = {bool(engine)}")
|
||||
if hasattr(engine, 'decoder'):
|
||||
print(f"[diag] decoder = {engine.decoder}")
|
||||
if hasattr(engine.decoder, 'ready_states'):
|
||||
print(f"[diag] decoder.ready_states = {engine.decoder.ready_states}")
|
||||
except Exception as e:
|
||||
print(f"[diag] EXCEPTION: {e}")
|
||||
traceback.print_exc()
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deep engine init diagnostic — catch every failure point."""
|
||||
import os, sys, traceback, logging
|
||||
from pathlib import Path
|
||||
|
||||
# Configure ALL loggers to console
|
||||
logging.basicConfig(level=logging.DEBUG, stream=sys.stderr, format='%(name)s %(levelname)s %(message)s')
|
||||
|
||||
MODEL_DIR = "/models/qwen3-tts"
|
||||
parent = os.path.dirname(os.path.abspath(MODEL_DIR))
|
||||
basename = os.path.basename(MODEL_DIR)
|
||||
os.chdir(parent)
|
||||
print(f"cwd={Path.cwd()}, basename={basename}", flush=True)
|
||||
|
||||
project_root = Path.cwd()
|
||||
model_dir = project_root / basename
|
||||
print(f"model_dir={model_dir}", flush=True)
|
||||
|
||||
# Step 1: AssetsManager
|
||||
print("\n=== Step 1: AssetsManager ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.assets import AssetsManager
|
||||
assets = AssetsManager(str(model_dir))
|
||||
print(f" OK: assets={assets}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
# Step 2: Tokenizer
|
||||
print("\n=== Step 2: Tokenizer ===", flush=True)
|
||||
try:
|
||||
from tokenizers import Tokenizer
|
||||
tok = Tokenizer.from_file(str(model_dir / "tokenizer.json"))
|
||||
print(f" OK: tokenizer loaded", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
# Step 3: CodecEncoder
|
||||
print("\n=== Step 3: CodecEncoder ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.encoder import CodecEncoder
|
||||
codec = CodecEncoder(str(model_dir / "qwen3_tts_codec_encoder.fp16.onnx"))
|
||||
print(f" OK: codec_encoder={codec}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
# Step 4: SpeakerEncoder
|
||||
print("\n=== Step 4: SpeakerEncoder ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.encoder import SpeakerEncoder
|
||||
spk = SpeakerEncoder(str(model_dir / "qwen3_tts_speaker_encoder.fp16.onnx"))
|
||||
print(f" OK: speaker_encoder={spk}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
# Step 5: DecoderProxy
|
||||
print("\n=== Step 5: DecoderProxy ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference.proxy import DecoderProxy
|
||||
decoder = DecoderProxy(str(model_dir / "qwen3_tts_decoder.fp16.onnx"), onnx_provider="CPUExecutionProvider", chunk_size=12)
|
||||
print(f" OK: decoder={decoder}", flush=True)
|
||||
print(" Waiting for decoder ready (timeout=10)...", flush=True)
|
||||
is_ready = decoder.wait_until_ready(timeout=10)
|
||||
print(f" decoder ready={is_ready}", flush=True)
|
||||
if hasattr(decoder, 'ready_states'):
|
||||
print(f" ready_states={decoder.ready_states}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
# Step 6: LlamaModel
|
||||
print("\n=== Step 6: LlamaModel (GGUF) ===", flush=True)
|
||||
try:
|
||||
from qwen3_tts_gguf.inference import llama
|
||||
t_path = (model_dir / "qwen3_tts_talker.q5_k.gguf").relative_to(project_root).as_posix()
|
||||
p_path = (model_dir / "qwen3_tts_predictor.q8_0.gguf").relative_to(project_root).as_posix()
|
||||
print(f" talker_path={t_path}", flush=True)
|
||||
print(f" predictor_path={p_path}", flush=True)
|
||||
talker = llama.LlamaModel(t_path, n_gpu_layers=-1)
|
||||
print(f" OK: talker={talker}", flush=True)
|
||||
predictor = llama.LlamaModel(p_path, n_gpu_layers=-1)
|
||||
print(f" OK: predictor={predictor}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" FAIL: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n=== DONE ===", flush=True)
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dump tensor info from qwen3_assets.gguf and extract embeddings to npy."""
|
||||
import sys, os, struct
|
||||
import numpy as np
|
||||
|
||||
GGUF_PATH = "/models/qwen3-tts/qwen3_assets.gguf"
|
||||
OUT_DIR = "/tmp/embeddings"
|
||||
|
||||
# Try using the gguf library
|
||||
try:
|
||||
from gguf import GGUFReader
|
||||
print("Using gguf library GGUFReader")
|
||||
reader = GGUFReader(GGUF_PATH)
|
||||
print(f"Tensors in {GGUF_PATH}:")
|
||||
for i, tensor in enumerate(reader.tensors):
|
||||
print(f" [{i}] name={tensor.name}, shape={tensor.shape}, type={tensor.tensor_type}")
|
||||
|
||||
# Name mapping: GGUF tensor name -> npy filename
|
||||
NAME_MAP = {
|
||||
"text_embd": "text_embedding_projected.npy",
|
||||
"proj.weight": "proj_weight.npy",
|
||||
"proj.bias": "proj_bias.npy",
|
||||
}
|
||||
for j in range(16):
|
||||
NAME_MAP[f"codec_embd.{j}"] = f"codec_embedding_{j}.npy"
|
||||
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
|
||||
for tensor in reader.tensors:
|
||||
name = tensor.name
|
||||
outname = NAME_MAP.get(name, name.replace("/", "_").replace(".", "_") + ".npy")
|
||||
|
||||
# tensor.data is a numpy array (may be quantized view)
|
||||
data = tensor.data
|
||||
print(f" Processing: {name} -> {outname}, raw shape={data.shape}, dtype={data.dtype}")
|
||||
|
||||
# For Q8_0: block size 32, each block = 2 bytes scale + 32 bytes ints
|
||||
# The gguf library should dequantize automatically via .data
|
||||
# If dtype is already float, use as-is. Otherwise cast.
|
||||
if data.dtype in (np.float32, np.float64):
|
||||
arr = data.astype(np.float32)
|
||||
elif data.dtype == np.float16:
|
||||
arr = data.astype(np.float32)
|
||||
else:
|
||||
# Quantized — try dequantizing manually for Q8_0
|
||||
print(f" WARNING: dtype={data.dtype}, attempting Q8_0 dequant for shape {tensor.shape}")
|
||||
target_shape = list(tensor.shape)
|
||||
# Q8_0: block_size=32, each block: 1 fp16 scale + 32 int8
|
||||
n_elements = 1
|
||||
for d in target_shape:
|
||||
n_elements *= d
|
||||
n_blocks = n_elements // 32
|
||||
raw = data.tobytes()
|
||||
# Each Q8_0 block: 2 bytes (fp16 scale) + 32 bytes (int8 quants) = 34 bytes
|
||||
block_size = 34
|
||||
if len(raw) == n_blocks * block_size:
|
||||
scales = np.zeros(n_blocks, dtype=np.float32)
|
||||
quants = np.zeros(n_elements, dtype=np.float32)
|
||||
for bi in range(n_blocks):
|
||||
offset = bi * block_size
|
||||
s = np.frombuffer(raw[offset:offset+2], dtype=np.float16)[0]
|
||||
scales[bi] = float(s)
|
||||
qs = np.frombuffer(raw[offset+2:offset+block_size], dtype=np.int8)
|
||||
quants[bi*32:(bi+1)*32] = qs.astype(np.float32) * float(s)
|
||||
arr = quants.reshape(target_shape)
|
||||
else:
|
||||
print(f" ERROR: Cannot dequantize, raw_bytes={len(raw)}, expected={n_blocks * block_size}")
|
||||
arr = data.astype(np.float32) if data.dtype.kind == 'f' else None
|
||||
if arr is None:
|
||||
print(f" SKIPPING tensor {name}")
|
||||
continue
|
||||
|
||||
outpath = os.path.join(OUT_DIR, outname)
|
||||
np.save(outpath, arr)
|
||||
print(f" Saved: {outpath} shape={arr.shape} dtype={arr.dtype}")
|
||||
|
||||
print(f"\nDone! Files in {OUT_DIR}:")
|
||||
for f in sorted(os.listdir(OUT_DIR)):
|
||||
sz = os.path.getsize(os.path.join(OUT_DIR, f))
|
||||
print(f" {f} ({sz} bytes)")
|
||||
|
||||
except ImportError:
|
||||
print("gguf library not available, trying manual parse...")
|
||||
# Minimal GGUF tensor listing
|
||||
with open(GGUF_PATH, "rb") as f:
|
||||
magic = f.read(4)
|
||||
print(f"Magic: {magic}")
|
||||
version = struct.unpack("<I", f.read(4))[0]
|
||||
print(f"Version: {version}")
|
||||
n_tensors = struct.unpack("<Q", f.read(8))[0]
|
||||
n_kv = struct.unpack("<Q", f.read(8))[0]
|
||||
print(f"Tensors: {n_tensors}, KV pairs: {n_kv}")
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Qwen3-TTS HTTP server — OpenAI-compatible /v1/audio/speech endpoint."""
|
||||
import argparse, io, os, sys
|
||||
from flask import Flask, request, send_file, jsonify
|
||||
|
||||
app = Flask(__name__)
|
||||
MODEL_DIR = "/models/qwen3-tts"
|
||||
engine = None
|
||||
|
||||
SPEAKERS = ["Vivian","Serena","uncle_fu","Dylan","Eric","Ryan","Aiden","ono_anna","Sohee"]
|
||||
LANGUAGES = ["english","chinese","japanese","korean","german","spanish","french","russian","italian","portuguese"]
|
||||
|
||||
def init_engine():
|
||||
global engine
|
||||
# TTSEngine uses Path.cwd() as project_root and builds relative paths from it.
|
||||
# model_dir must be a subdirectory of cwd for relative_to() to work.
|
||||
parent = os.path.dirname(os.path.abspath(MODEL_DIR))
|
||||
basename = os.path.basename(MODEL_DIR)
|
||||
os.chdir(parent)
|
||||
from qwen3_tts_gguf.inference import TTSEngine
|
||||
engine = TTSEngine(model_dir=basename, onnx_provider="CPUExecutionProvider")
|
||||
if not engine.ready:
|
||||
print(f"[server] WARNING: engine created but not ready", flush=True)
|
||||
|
||||
@app.route("/health")
|
||||
def health():
|
||||
ready = engine is not None and engine.ready
|
||||
return jsonify({"status": "ok" if ready else "loading", "speakers": SPEAKERS, "languages": LANGUAGES})
|
||||
|
||||
@app.route("/v1/audio/speech", methods=["POST"])
|
||||
def speech():
|
||||
if engine is None or not engine.ready:
|
||||
return jsonify({"error": "engine not ready"}), 503
|
||||
data = request.get_json(force=True)
|
||||
text = data.get("input", "")
|
||||
if not text:
|
||||
return jsonify({"error": "missing input"}), 400
|
||||
speaker = data.get("voice", "Vivian")
|
||||
if speaker not in SPEAKERS:
|
||||
speaker = "Vivian"
|
||||
language = data.get("language", "english")
|
||||
if language not in LANGUAGES:
|
||||
language = "english"
|
||||
instruct = data.get("instruct", "")
|
||||
try:
|
||||
import numpy as np, wave as _wave
|
||||
from qwen3_tts_gguf.inference import TTSConfig
|
||||
stream = engine.create_stream(n_ctx=2048)
|
||||
if stream is None:
|
||||
return jsonify({"error": "failed to create stream"}), 500
|
||||
cfg = TTSConfig()
|
||||
result = stream.custom(text=text, speaker=speaker, language=language,
|
||||
instruct=instruct, config=cfg)
|
||||
stream.join()
|
||||
if result is None:
|
||||
return jsonify({"error": "synthesis returned None"}), 500
|
||||
audio = result.audio if hasattr(result, 'audio') and result.audio is not None else None
|
||||
if audio is None:
|
||||
tmp = os.path.join("/tmp", "tts_out.wav")
|
||||
result.save(tmp)
|
||||
import soundfile as sf
|
||||
audio, _ = sf.read(tmp, dtype='float32')
|
||||
os.remove(tmp)
|
||||
buf = io.BytesIO()
|
||||
with _wave.open(buf, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(24000)
|
||||
pcm = (np.clip(audio, -1.0, 1.0) * 32767).astype(np.int16).tobytes()
|
||||
wf.writeframes(pcm)
|
||||
buf.seek(0)
|
||||
return send_file(buf, mimetype="audio/wav", download_name="speech.wav")
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--host", default="0.0.0.0")
|
||||
p.add_argument("--port", type=int, default=8072)
|
||||
p.add_argument("--model-dir", default=MODEL_DIR)
|
||||
a = p.parse_args()
|
||||
MODEL_DIR = a.model_dir
|
||||
init_engine()
|
||||
app.run(host=a.host, port=a.port)
|
||||
@@ -0,0 +1,6 @@
|
||||
import urllib.request
|
||||
try:
|
||||
r = urllib.request.urlopen("http://qwen3-tts:8072/health", timeout=3)
|
||||
print("OK:", r.read().decode())
|
||||
except Exception as e:
|
||||
print("FAIL:", e)
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
curl -s -X POST http://localhost:9090/api/model/load \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model_id":"qwen3-4b"}'
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Qwen3-TTS HTTP server — OpenAI-compatible speech endpoint with voice training."""
|
||||
import argparse, io, json, os, re, sys, tempfile, time
|
||||
from pathlib import Path
|
||||
from flask import Flask, request, send_file, jsonify
|
||||
|
||||
app = Flask(__name__)
|
||||
MODEL_DIR = "/models/qwen3-tts"
|
||||
CUSTOM_VOICES_DIR = "/models/qwen3-tts/custom_speakers"
|
||||
engine = None
|
||||
|
||||
PRESET_SPEAKERS = ["Vivian","Serena","uncle_fu","Dylan","Eric","Ryan","Aiden","ono_anna","Sohee"]
|
||||
LANGUAGES = ["english","chinese","japanese","korean","german","spanish","french","russian","italian","portuguese"]
|
||||
SAFE_NAME_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}$')
|
||||
|
||||
# ── Bootstrap helpers ──
|
||||
|
||||
def ensure_embeddings(model_dir):
|
||||
emb_dir = os.path.join(model_dir, "embeddings")
|
||||
if os.path.exists(os.path.join(emb_dir, "text_embedding_projected.npy")):
|
||||
return
|
||||
gguf_path = os.path.join(model_dir, "qwen3_assets.gguf")
|
||||
if not os.path.exists(gguf_path):
|
||||
print(f"[server] WARNING: {gguf_path} not found", flush=True)
|
||||
return
|
||||
print(f"[server] Extracting embeddings from GGUF...", flush=True)
|
||||
import numpy as np
|
||||
from gguf import GGUFReader
|
||||
NAME_MAP = {"text_embd": "text_embedding_projected.npy",
|
||||
"proj.weight": "proj_weight.npy", "proj.bias": "proj_bias.npy"}
|
||||
for j in range(16):
|
||||
NAME_MAP[f"codec_embd.{j}"] = f"codec_embedding_{j}.npy"
|
||||
reader = GGUFReader(gguf_path)
|
||||
os.makedirs(emb_dir, exist_ok=True)
|
||||
for tensor in reader.tensors:
|
||||
outname = NAME_MAP.get(tensor.name)
|
||||
if outname is None:
|
||||
continue
|
||||
data = tensor.data
|
||||
shape = list(tensor.shape)
|
||||
n_el = 1
|
||||
for d in shape:
|
||||
n_el *= d
|
||||
n_blocks = n_el // 32
|
||||
raw = data.tobytes()
|
||||
block_sz = 34
|
||||
if len(raw) == n_blocks * block_sz:
|
||||
quants = np.zeros(n_el, dtype=np.float32)
|
||||
for bi in range(n_blocks):
|
||||
off = bi * block_sz
|
||||
s = float(np.frombuffer(raw[off:off+2], dtype=np.float16)[0])
|
||||
qs = np.frombuffer(raw[off+2:off+block_sz], dtype=np.int8)
|
||||
quants[bi*32:(bi+1)*32] = qs.astype(np.float32) * s
|
||||
arr = quants.reshape(shape)
|
||||
else:
|
||||
arr = np.array(data, dtype=np.float32).reshape(shape)
|
||||
np.save(os.path.join(emb_dir, outname), arr)
|
||||
print(f"[server] Embeddings extracted", flush=True)
|
||||
|
||||
def ensure_symlinks(model_dir):
|
||||
links = {"qwen3_tts_talker.q5_k.gguf": "qwen3_tts_talker.gguf",
|
||||
"qwen3_tts_predictor.q8_0.gguf": "qwen3_tts_predictor.gguf",
|
||||
"qwen3_tts_decoder.fp16.onnx": "qwen3_tts_decoder.onnx",
|
||||
"qwen3_tts_codec_encoder.fp16.onnx": "qwen3_tts_codec_encoder.onnx",
|
||||
"qwen3_tts_speaker_encoder.fp16.onnx": "qwen3_tts_speaker_encoder.onnx"}
|
||||
for link_name, target in links.items():
|
||||
lp = os.path.join(model_dir, link_name)
|
||||
tp = os.path.join(model_dir, target)
|
||||
if not os.path.exists(lp) and os.path.exists(tp):
|
||||
os.symlink(target, lp)
|
||||
|
||||
def init_engine():
|
||||
global engine
|
||||
parent = os.path.dirname(os.path.abspath(MODEL_DIR))
|
||||
basename = os.path.basename(MODEL_DIR)
|
||||
ensure_embeddings(MODEL_DIR)
|
||||
ensure_symlinks(MODEL_DIR)
|
||||
os.makedirs(CUSTOM_VOICES_DIR, exist_ok=True)
|
||||
os.chdir(parent)
|
||||
from qwen3_tts_gguf.inference import TTSEngine
|
||||
engine = TTSEngine(model_dir=basename, onnx_provider="CPUExecutionProvider")
|
||||
if not engine.ready:
|
||||
print(f"[server] WARNING: engine created but not ready", flush=True)
|
||||
|
||||
# ── Voice management helpers ──
|
||||
|
||||
def list_all_speakers():
|
||||
presets = []
|
||||
for name in PRESET_SPEAKERS:
|
||||
presets.append({"name": name, "type": "preset"})
|
||||
customs = []
|
||||
if os.path.isdir(CUSTOM_VOICES_DIR):
|
||||
for f in sorted(os.listdir(CUSTOM_VOICES_DIR)):
|
||||
if f.endswith(".json"):
|
||||
vname = f[:-5]
|
||||
meta = _load_voice_meta(vname)
|
||||
customs.append({"name": vname, "type": "custom",
|
||||
"description": meta.get("description", ""),
|
||||
"created": meta.get("created", "")})
|
||||
return presets + customs
|
||||
|
||||
def _load_voice_meta(name):
|
||||
path = os.path.join(CUSTOM_VOICES_DIR, f"{name}.json")
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return {"description": data.get("description", ""),
|
||||
"created": data.get("created", ""),
|
||||
"ref_text": data.get("text", ""),
|
||||
"duration_hint": data.get("duration_hint", 0)}
|
||||
|
||||
def _voice_exists(name):
|
||||
return os.path.exists(os.path.join(CUSTOM_VOICES_DIR, f"{name}.json"))
|
||||
|
||||
def _is_custom_voice(name):
|
||||
return name not in [s.lower() for s in PRESET_SPEAKERS] and _voice_exists(name)
|
||||
|
||||
def _audio_to_wav(buf):
|
||||
import numpy as np, wave as _wave
|
||||
buf_bytes = buf.tobytes() if hasattr(buf, 'tobytes') else buf
|
||||
return buf_bytes
|
||||
|
||||
# ── Audio rendering helper ──
|
||||
|
||||
def render_audio(audio_np):
|
||||
import numpy as np, wave as _wave
|
||||
buf = io.BytesIO()
|
||||
with _wave.open(buf, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(24000)
|
||||
pcm = (np.clip(audio_np, -1.0, 1.0) * 32767).astype(np.int16).tobytes()
|
||||
wf.writeframes(pcm)
|
||||
buf.seek(0)
|
||||
return buf
|
||||
|
||||
# ── Routes ──
|
||||
|
||||
@app.route("/health")
|
||||
def health():
|
||||
ready = engine is not None and engine.ready
|
||||
speakers = [s["name"] for s in list_all_speakers()]
|
||||
return jsonify({"status": "ok" if ready else "loading",
|
||||
"speakers": speakers, "languages": LANGUAGES})
|
||||
|
||||
@app.route("/v1/voices", methods=["GET"])
|
||||
def voices_list():
|
||||
return jsonify({"voices": list_all_speakers()})
|
||||
|
||||
@app.route("/v1/voices/<name>", methods=["GET"])
|
||||
def voices_get(name):
|
||||
name = name.lower()
|
||||
if name in [s.lower() for s in PRESET_SPEAKERS]:
|
||||
return jsonify({"name": name, "type": "preset"})
|
||||
if not _voice_exists(name):
|
||||
return jsonify({"error": "voice not found"}), 404
|
||||
meta = _load_voice_meta(name)
|
||||
return jsonify({"name": name, "type": "custom", **meta})
|
||||
|
||||
@app.route("/v1/voices/<name>", methods=["DELETE"])
|
||||
def voices_delete(name):
|
||||
name = name.lower()
|
||||
if name in [s.lower() for s in PRESET_SPEAKERS]:
|
||||
return jsonify({"error": "cannot delete preset voice"}), 400
|
||||
path = os.path.join(CUSTOM_VOICES_DIR, f"{name}.json")
|
||||
if not os.path.exists(path):
|
||||
return jsonify({"error": "voice not found"}), 404
|
||||
os.remove(path)
|
||||
return jsonify({"deleted": name})
|
||||
|
||||
@app.route("/v1/voices/train", methods=["POST"])
|
||||
def voices_train():
|
||||
if engine is None or not engine.ready:
|
||||
return jsonify({"error": "engine not ready"}), 503
|
||||
|
||||
if "audio" not in request.files:
|
||||
return jsonify({"error": "missing 'audio' file in multipart form"}), 400
|
||||
|
||||
name = request.form.get("name", "").strip().lower()
|
||||
if not name or not SAFE_NAME_RE.match(name):
|
||||
return jsonify({"error": "invalid name (a-z, 0-9, _, - ; max 64 chars)"}), 400
|
||||
if name in [s.lower() for s in PRESET_SPEAKERS]:
|
||||
return jsonify({"error": "name conflicts with preset speaker"}), 400
|
||||
|
||||
ref_text = request.form.get("text", "").strip()
|
||||
description = request.form.get("description", "").strip()
|
||||
language = request.form.get("language", "english").strip().lower()
|
||||
if language not in LANGUAGES:
|
||||
language = "english"
|
||||
|
||||
audio_file = request.files["audio"]
|
||||
allowed_ext = {".wav", ".mp3", ".flac", ".m4a", ".opus", ".ogg"}
|
||||
ext = os.path.splitext(audio_file.filename or "upload.wav")[1].lower()
|
||||
if ext not in allowed_ext:
|
||||
return jsonify({"error": f"unsupported format: {ext}"}), 400
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
from qwen3_tts_gguf.inference.utils.audio import load_audio
|
||||
from qwen3_tts_gguf.inference import TTSConfig
|
||||
from qwen3_tts_gguf.inference.schema.result import TTSResult
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp:
|
||||
audio_file.save(tmp)
|
||||
tmp_path = tmp.name
|
||||
|
||||
samples = load_audio(tmp_path)
|
||||
os.unlink(tmp_path)
|
||||
|
||||
if samples is None or len(samples) < 2400:
|
||||
return jsonify({"error": "audio too short (min 0.1s at 24kHz)"}), 400
|
||||
|
||||
duration = len(samples) / 24000.0
|
||||
if duration > 30.0:
|
||||
samples = samples[:int(30.0 * 24000)]
|
||||
duration = 30.0
|
||||
|
||||
codes = engine.codec_encoder.encode(samples)
|
||||
spk_emb = engine.speaker_encoder.encode(samples)
|
||||
|
||||
text_ids = engine.tokenizer.encode(ref_text).ids if ref_text else []
|
||||
|
||||
result = TTSResult(
|
||||
text=ref_text,
|
||||
text_ids=text_ids,
|
||||
codes=codes,
|
||||
spk_emb=spk_emb,
|
||||
audio=samples
|
||||
)
|
||||
|
||||
voice_data = {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"language": language,
|
||||
"created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"duration_hint": round(duration, 2),
|
||||
"text": ref_text,
|
||||
"text_ids": text_ids,
|
||||
"codes": codes.tolist(),
|
||||
"spk_emb": result.spk_emb.tolist(),
|
||||
}
|
||||
|
||||
out_path = os.path.join(CUSTOM_VOICES_DIR, f"{name}.json")
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(voice_data, f, ensure_ascii=False)
|
||||
|
||||
preview_audio = None
|
||||
if ref_text:
|
||||
stream = engine.create_stream(n_ctx=2048)
|
||||
if stream is not None:
|
||||
stream.set_voice(result)
|
||||
clone_result = stream.clone(text=ref_text, language=language, config=TTSConfig())
|
||||
stream.join()
|
||||
if clone_result and clone_result.audio is not None:
|
||||
preview_audio = clone_result.audio
|
||||
|
||||
resp = {"name": name, "type": "custom", "description": description,
|
||||
"duration": round(duration, 2), "spk_emb_dim": len(spk_emb),
|
||||
"codes_frames": len(codes)}
|
||||
|
||||
if preview_audio is not None:
|
||||
wav_buf = render_audio(preview_audio)
|
||||
resp_json = json.dumps(resp)
|
||||
return send_file(wav_buf, mimetype="audio/wav", download_name=f"{name}_preview.wav",
|
||||
as_attachment=False), 200, {"X-Voice-Info": resp_json}
|
||||
|
||||
return jsonify(resp), 201
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route("/v1/audio/speech", methods=["POST"])
|
||||
def speech():
|
||||
if engine is None or not engine.ready:
|
||||
return jsonify({"error": "engine not ready"}), 503
|
||||
data = request.get_json(force=True)
|
||||
text = data.get("input", "")
|
||||
if not text:
|
||||
return jsonify({"error": "missing input"}), 400
|
||||
|
||||
voice_name = data.get("voice", "Vivian")
|
||||
language = data.get("language", "english")
|
||||
if language not in LANGUAGES:
|
||||
language = "english"
|
||||
instruct = data.get("instruct", "")
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
from qwen3_tts_gguf.inference import TTSConfig
|
||||
from qwen3_tts_gguf.inference.schema.result import TTSResult
|
||||
|
||||
stream = engine.create_stream(n_ctx=2048)
|
||||
if stream is None:
|
||||
return jsonify({"error": "failed to create stream"}), 500
|
||||
|
||||
cfg = TTSConfig()
|
||||
voice_json = os.path.join(CUSTOM_VOICES_DIR, f"{voice_name.lower()}.json")
|
||||
|
||||
if os.path.exists(voice_json):
|
||||
with open(voice_json, "r", encoding="utf-8") as f:
|
||||
vdata = json.load(f)
|
||||
spk_emb = np.array(vdata["spk_emb"], dtype=np.float32)
|
||||
codes = np.array(vdata["codes"], dtype=np.int64)
|
||||
anchor = TTSResult(
|
||||
text=vdata.get("text", ""),
|
||||
text_ids=vdata.get("text_ids", []),
|
||||
codes=codes,
|
||||
spk_emb=spk_emb
|
||||
)
|
||||
stream.set_voice(anchor)
|
||||
result = stream.clone(text=text, language=language, config=cfg)
|
||||
else:
|
||||
speaker = voice_name
|
||||
if speaker not in PRESET_SPEAKERS:
|
||||
speaker = "Vivian"
|
||||
result = stream.custom(text=text, speaker=speaker, language=language,
|
||||
instruct=instruct, config=cfg)
|
||||
|
||||
stream.join()
|
||||
if result is None:
|
||||
return jsonify({"error": "synthesis returned None"}), 500
|
||||
|
||||
audio = result.audio if hasattr(result, 'audio') and result.audio is not None else None
|
||||
if audio is None:
|
||||
tmp = os.path.join("/tmp", "tts_out.wav")
|
||||
result.save(tmp)
|
||||
import soundfile as sf
|
||||
audio, _ = sf.read(tmp, dtype='float32')
|
||||
os.remove(tmp)
|
||||
|
||||
return send_file(render_audio(audio), mimetype="audio/wav", download_name="speech.wav")
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--host", default="0.0.0.0")
|
||||
p.add_argument("--port", type=int, default=8072)
|
||||
p.add_argument("--model-dir", default=MODEL_DIR)
|
||||
a = p.parse_args()
|
||||
MODEL_DIR = a.model_dir
|
||||
CUSTOM_VOICES_DIR = os.path.join(MODEL_DIR, "custom_speakers")
|
||||
init_engine()
|
||||
app.run(host=a.host, port=a.port)
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Patch speaker.py in-place to be headless-safe.
|
||||
Replaces the sd.OutputStream block with a dummy loop that just sends READY.
|
||||
"""
|
||||
import os
|
||||
|
||||
SPEAKER_PATH = "/opt/qwen3-tts/qwen3_tts_gguf/inference/workers/speaker.py"
|
||||
|
||||
with open(SPEAKER_PATH, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
# Replace the try block at the end of speaker_worker_proc
|
||||
old = ''' try:
|
||||
with sd.OutputStream(samplerate=sample_rate, channels=1, callback=audio_callback, blocksize=2048):
|
||||
# 握手
|
||||
if result_queue:
|
||||
result_queue.put(SpeakerResponse(msg_type="READY"))
|
||||
|
||||
while True:
|
||||
time.sleep(0.2)
|
||||
if state.get("stop"): break
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"❌ [SpeakerWorker] 异常: {e}")'''
|
||||
|
||||
new = ''' try:
|
||||
with sd.OutputStream(samplerate=sample_rate, channels=1, callback=audio_callback, blocksize=2048):
|
||||
# 握手
|
||||
if result_queue:
|
||||
result_queue.put(SpeakerResponse(msg_type="READY"))
|
||||
|
||||
while True:
|
||||
time.sleep(0.2)
|
||||
if state.get("stop"): break
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"⚠️ [SpeakerWorker] No audio device, running headless: {e}")
|
||||
# Headless fallback: send READY and drain queue without playback
|
||||
if result_queue:
|
||||
result_queue.put(SpeakerResponse(msg_type="READY"))
|
||||
while not state.get("stop"):
|
||||
try:
|
||||
command = play_queue.get(timeout=0.5)
|
||||
handle_command(command, state)
|
||||
except Exception:
|
||||
pass'''
|
||||
|
||||
if old in content:
|
||||
content = content.replace(old, new)
|
||||
with open(SPEAKER_PATH, "w") as f:
|
||||
f.write(content)
|
||||
print("OK: speaker.py patched with headless fallback")
|
||||
else:
|
||||
print("WARNING: exact match not found, trying simplified patch...")
|
||||
# Try to find and patch just the except Exception block
|
||||
if '❌ [SpeakerWorker] 异常' in content:
|
||||
content = content.replace(
|
||||
'print(f"❌ [SpeakerWorker] 异常: {e}")',
|
||||
'''print(f"⚠️ [SpeakerWorker] No audio device, running headless: {e}")
|
||||
# Headless fallback: send READY and drain queue without playback
|
||||
if result_queue:
|
||||
result_queue.put(SpeakerResponse(msg_type="READY"))
|
||||
while not state.get("stop"):
|
||||
try:
|
||||
command = play_queue.get(timeout=0.5)
|
||||
handle_command(command, state)
|
||||
except Exception:
|
||||
pass'''
|
||||
)
|
||||
with open(SPEAKER_PATH, "w") as f:
|
||||
f.write(content)
|
||||
print("OK: speaker.py patched (simplified)")
|
||||
else:
|
||||
print("ERROR: Could not find patch target in speaker.py")
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
echo "=== inference/__init__.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/__init__.py
|
||||
echo ""
|
||||
echo "=== engine.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/engine.py
|
||||
echo ""
|
||||
echo "=== stream.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/stream.py
|
||||
echo ""
|
||||
echo "=== config.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/config.py
|
||||
echo ""
|
||||
echo "=== schema/result.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/schema/result.py
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
echo "=== inference/__init__.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/__init__.py
|
||||
echo ""
|
||||
echo "=== engine.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/engine.py
|
||||
@@ -0,0 +1,114 @@
|
||||
import time
|
||||
import queue
|
||||
import numpy as np
|
||||
try:
|
||||
import sounddevice as sd
|
||||
except (ImportError, OSError):
|
||||
sd = None
|
||||
from ..schema.protocol import SpeakerRequest, SpeakerResponse
|
||||
|
||||
def handle_command(cmd: SpeakerRequest, state: dict):
|
||||
if cmd is None or cmd.msg_type == "EXIT":
|
||||
state["stop"] = True
|
||||
return
|
||||
if cmd.msg_type == "STOP":
|
||||
state["current_data"] = np.zeros((0, 1), dtype=np.float32)
|
||||
state["started"] = False
|
||||
return
|
||||
if cmd.msg_type == "PAUSE":
|
||||
state["paused"] = True
|
||||
return
|
||||
if cmd.msg_type == "CONTINUE":
|
||||
state["paused"] = False
|
||||
return
|
||||
if cmd.msg_type == "AUDIO":
|
||||
if cmd.audio is not None and len(cmd.audio) > 0:
|
||||
state["current_data"] = np.concatenate(
|
||||
[state["current_data"], cmd.audio.reshape(-1, 1).astype(np.float32)],
|
||||
axis=0
|
||||
)
|
||||
|
||||
def sync_playback_status(state: dict, result_queue):
|
||||
if result_queue is None: return
|
||||
if state.get("paused", False):
|
||||
target = "PAUSED"
|
||||
elif state.get("started", False):
|
||||
target = "PLAYING"
|
||||
else:
|
||||
target = "IDLE"
|
||||
if target == state["playback_state"]:
|
||||
return
|
||||
msg_map = {"PAUSED": "PAUSED", "PLAYING": "STARTED", "IDLE": "FINISHED"}
|
||||
result_queue.put(SpeakerResponse(msg_type=msg_map[target]))
|
||||
state["playback_state"] = target
|
||||
|
||||
def fill_audio(outdata, frames, state: dict):
|
||||
if state.get("paused", False):
|
||||
outdata.fill(0)
|
||||
return
|
||||
if not state["started"]:
|
||||
if len(state["current_data"]) >= state["threshold"]:
|
||||
state["started"] = True
|
||||
else:
|
||||
outdata.fill(0)
|
||||
return
|
||||
avail = len(state["current_data"])
|
||||
to_copy = min(avail, frames)
|
||||
if to_copy > 0:
|
||||
outdata[:to_copy] = state["current_data"][:to_copy]
|
||||
state["current_data"] = state["current_data"][to_copy:]
|
||||
if to_copy < frames:
|
||||
outdata[to_copy:].fill(0)
|
||||
state["started"] = False
|
||||
|
||||
def speaker_worker_proc(play_queue, result_queue=None, sample_rate=24000):
|
||||
state = {
|
||||
"current_data": np.zeros((0, 1), dtype=np.float32),
|
||||
"started": False,
|
||||
"threshold": 1200,
|
||||
"stop": False,
|
||||
"paused": False,
|
||||
"playback_state": "IDLE"
|
||||
}
|
||||
|
||||
def audio_callback(outdata, frames, time_info, status):
|
||||
while True:
|
||||
try:
|
||||
command = play_queue.get_nowait()
|
||||
handle_command(command, state)
|
||||
except queue.Empty:
|
||||
break
|
||||
fill_audio(outdata, frames, state)
|
||||
sync_playback_status(state, result_queue)
|
||||
|
||||
if sd is None:
|
||||
# Headless mode: no audio device available
|
||||
if result_queue:
|
||||
result_queue.put(SpeakerResponse(msg_type="READY"))
|
||||
while not state.get("stop"):
|
||||
try:
|
||||
command = play_queue.get(timeout=0.5)
|
||||
handle_command(command, state)
|
||||
except queue.Empty:
|
||||
pass
|
||||
return
|
||||
|
||||
try:
|
||||
with sd.OutputStream(samplerate=sample_rate, channels=1, callback=audio_callback, blocksize=2048):
|
||||
if result_queue:
|
||||
result_queue.put(SpeakerResponse(msg_type="READY"))
|
||||
while True:
|
||||
time.sleep(0.2)
|
||||
if state.get("stop"): break
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"⚠️ [SpeakerWorker] No audio device, running headless: {e}")
|
||||
if result_queue:
|
||||
result_queue.put(SpeakerResponse(msg_type="READY"))
|
||||
while not state.get("stop"):
|
||||
try:
|
||||
command = play_queue.get(timeout=0.5)
|
||||
handle_command(command, state)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
echo "=== ACE-Step Speed Test ==="
|
||||
echo "Starting generation: 10s audio, 15 steps..."
|
||||
START=$(date +%s)
|
||||
|
||||
curl -s -X POST http://localhost:8076/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"prompt":"upbeat electronic dance music","lyrics":"[verse]\nLa la la\n[chorus]\nDance all night","duration":10,"ace_steps":15,"cfg_scale":5.0,"seed":42}' \
|
||||
-o /tmp/ace_result.json
|
||||
|
||||
END=$(date +%s)
|
||||
ELAPSED=$((END - START))
|
||||
|
||||
echo "Total time: ${ELAPSED}s"
|
||||
echo ""
|
||||
echo "Response keys:"
|
||||
python3 -c "import json; d=json.load(open('/tmp/ace_result.json')); print('Keys:', list(d.keys())); print('Duration field:', d.get('duration_seconds','N/A')); print('Steps:', d.get('ace_steps','N/A'))" 2>/dev/null || echo "Could not parse JSON response"
|
||||
|
||||
echo ""
|
||||
echo "Container logs (last 10 lines):"
|
||||
docker logs ace-step --tail 10 2>&1
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# Simulate browser: chat stream + TTS call
|
||||
echo "=== Chat Stream Test ==="
|
||||
START=$(date +%s)
|
||||
# Send chat request and capture response
|
||||
curl -s --max-time 30 \
|
||||
http://localhost:9090/api/proxy/chat \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"messages":[{"role":"user","content":"Say hi in one sentence."}],"stream":true}' \
|
||||
-o /tmp/chat_stream.txt 2>&1
|
||||
EXIT=$?
|
||||
END=$(date +%s)
|
||||
echo "curl exit: $EXIT, took: $((END-START))s"
|
||||
echo "Last 5 lines of stream:"
|
||||
tail -5 /tmp/chat_stream.txt
|
||||
echo ""
|
||||
echo "Has [DONE]:"
|
||||
grep -c "DONE" /tmp/chat_stream.txt
|
||||
echo ""
|
||||
|
||||
echo "=== Now TTS call ==="
|
||||
curl -s -w '\nHTTP: %{http_code} Size: %{size_download}\n' \
|
||||
-o /tmp/tts_after_chat.wav \
|
||||
http://localhost:9090/api/proxy/tts \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"input":"Hello there!","voice":"Vivian","language":"en","speed":1.0}'
|
||||
|
||||
echo "=== Sidecar health ==="
|
||||
curl -s http://localhost:9090/api/tts/sidecar/health
|
||||
echo ""
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
curl -s -o /tmp/clone_test.wav \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Hallo, ich bin eine geklonte Stimme. Das ist ziemlich cool.","voice":"meine_stimme","language":"german"}' \
|
||||
http://localhost:8072/v1/audio/speech
|
||||
echo "Clone WAV:"
|
||||
ls -la /tmp/clone_test.wav
|
||||
file /tmp/clone_test.wav
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
echo "=== BEFORE LOAD ==="
|
||||
docker ps --format "{{.Names}} {{.Status}}" | sort
|
||||
echo
|
||||
echo "=== Loading qwen3-4b ==="
|
||||
curl -s -X POST http://localhost:9090/api/model/load \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model_id":"qwen3-4b"}'
|
||||
echo
|
||||
|
||||
# Poll until done
|
||||
for i in $(seq 1 30); do
|
||||
sleep 2
|
||||
busy=$(curl -s http://localhost:9090/api/operation/log | python3 -c "import sys,json; print(json.load(sys.stdin)['busy'])")
|
||||
if [ "$busy" = "False" ]; then
|
||||
echo "=== LOAD COMPLETE ==="
|
||||
break
|
||||
fi
|
||||
echo " waiting... ($((i*2))s)"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== AFTER LOAD ==="
|
||||
docker ps --format "{{.Names}} {{.Status}}" | sort
|
||||
echo
|
||||
echo "=== OP LOG ==="
|
||||
curl -s http://localhost:9090/api/operation/log | python3 -c "import sys,json; [print(l) for l in json.load(sys.stdin)['log']]"
|
||||
echo
|
||||
echo "=== TTS HEALTH ==="
|
||||
curl -s http://localhost:8072/health
|
||||
echo
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
echo "=== Test 1: Chat only ==="
|
||||
curl -s -X POST http://localhost:9090/api/proxy/chat \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"messages":[{"role":"user","content":"Say hello in one sentence."}],"stream":false}' | head -c 500
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "=== Test 2: TTS sidecar health ==="
|
||||
curl -s http://localhost:9090/api/tts/sidecar/health
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "=== Test 3: Chat+Speak pipeline ==="
|
||||
curl -s -X POST http://localhost:9090/api/chat/speak \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"messages":[{"role":"user","content":"Say hello in one short sentence."}],"tts_voice":"vivian","tts_language":"en"}' | python3 -c "
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
print('Text:', d.get('text','')[:200])
|
||||
print('Audio bytes:', len(d.get('audio_b64','')) if d.get('audio_b64') else 'NONE')
|
||||
print('Timings:', d.get('timings',{}))
|
||||
if d.get('error'): print('ERROR:', d['error'])
|
||||
"
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "=== Test 4: Memory usage ==="
|
||||
free -h | head -2
|
||||
echo
|
||||
docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}"
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# Test proxy_tts from host
|
||||
|
||||
echo "=== Test 1: curl with JSON ==="
|
||||
curl -s -w '\nHTTP: %{http_code} Size: %{size_download}\n' \
|
||||
-o /tmp/proxy_test1.wav \
|
||||
http://localhost:9090/api/proxy/tts \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"input":"Hello, this is a quick test.","voice":"Vivian","language":"en","speed":1.0}'
|
||||
|
||||
echo "=== Content of response ==="
|
||||
file /tmp/proxy_test1.wav 2>/dev/null || echo "No file"
|
||||
head -c 100 /tmp/proxy_test1.wav 2>/dev/null | xxd | head -3
|
||||
|
||||
echo ""
|
||||
echo "=== Test 2: curl with --data-raw ==="
|
||||
curl -s -w '\nHTTP: %{http_code} Size: %{size_download}\n' \
|
||||
-o /tmp/proxy_test2.wav \
|
||||
http://localhost:9090/api/proxy/tts \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data-raw '{"input":"test"}'
|
||||
|
||||
echo "=== Debug: dashboard logs ==="
|
||||
docker logs sudx-dashboard 2>&1 | tail -5
|
||||
@@ -0,0 +1,8 @@
|
||||
from app import app
|
||||
c = app.test_client()
|
||||
r = c.get("/api/voices")
|
||||
print("STATUS:", r.status_code)
|
||||
print("DATA:", r.data[:300])
|
||||
print("---")
|
||||
r2 = c.get("/api/system")
|
||||
print("SYSTEM STATUS:", r2.status_code)
|
||||
@@ -0,0 +1,21 @@
|
||||
from app import app
|
||||
import werkzeug
|
||||
|
||||
# Check URL map
|
||||
for rule in app.url_map.iter_rules():
|
||||
if 'voices' in rule.rule:
|
||||
print(f"Rule: {rule.rule}, Methods: {rule.methods}, Endpoint: {rule.endpoint}")
|
||||
|
||||
# Try to manually resolve
|
||||
adapter = app.url_map.bind('')
|
||||
try:
|
||||
endpoint, values = adapter.match('/api/voices', method='GET')
|
||||
print(f"\nMatched: endpoint={endpoint}, values={values}")
|
||||
except Exception as e:
|
||||
print(f"\nMatch FAILED: {e}")
|
||||
|
||||
# Check the actual view function
|
||||
print("\nView functions with 'voice':")
|
||||
for name, func in app.view_functions.items():
|
||||
if 'voice' in name.lower():
|
||||
print(f" {name}: {func}")
|
||||
@@ -0,0 +1,28 @@
|
||||
from app import app
|
||||
|
||||
with app.test_request_context('/api/voices'):
|
||||
try:
|
||||
from app import api_voices
|
||||
result = api_voices()
|
||||
print("RESULT TYPE:", type(result))
|
||||
if isinstance(result, tuple):
|
||||
print("STATUS:", result[1] if len(result) > 1 else "no status")
|
||||
print("CONTENT:", result[0][:200] if result[0] else "empty")
|
||||
else:
|
||||
print("RESULT:", result)
|
||||
except Exception as e:
|
||||
print(f"EXCEPTION: {type(e).__name__}: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Also test with test client but with more detail
|
||||
print("\n--- Test Client ---")
|
||||
c = app.test_client()
|
||||
r = c.get('/api/voices')
|
||||
print(f"Status: {r.status_code}")
|
||||
print(f"Headers: {dict(r.headers)}")
|
||||
|
||||
# Test a working endpoint for comparison
|
||||
print("\n--- /api/runtime ---")
|
||||
r2 = c.get('/api/runtime')
|
||||
print(f"Status: {r2.status_code}")
|
||||
@@ -0,0 +1,4 @@
|
||||
import sounddevice
|
||||
print("sounddevice OK")
|
||||
from qwen3_tts_gguf.inference.workers import decoder_worker_proc
|
||||
print("decoder worker import OK")
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
curl -s -o /tmp/test_speech.wav \
|
||||
-w "HTTP %{http_code} Size: %{size_download}\n" \
|
||||
-X POST http://localhost:8072/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Hello World, this is a test.","voice":"Vivian","language":"english"}'
|
||||
|
||||
if [ -f /tmp/test_speech.wav ]; then
|
||||
file /tmp/test_speech.wav
|
||||
ls -la /tmp/test_speech.wav
|
||||
fi
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
# Test streaming SSE from chat proxy
|
||||
echo "=== Stream test ==="
|
||||
timeout 15 curl -sN \
|
||||
http://localhost:9090/api/proxy/chat \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"messages":[{"role":"user","content":"Say hi in one word"}],"stream":true}' \
|
||||
2>&1 | tee /tmp/stream_test.txt
|
||||
|
||||
echo ""
|
||||
echo "=== Stream output size ==="
|
||||
wc -c /tmp/stream_test.txt
|
||||
echo "=== Has DONE ==="
|
||||
grep -c DONE /tmp/stream_test.txt
|
||||
echo "=== Last 3 lines ==="
|
||||
tail -3 /tmp/stream_test.txt
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
curl -s -w "\nHTTP %{http_code}\n" \
|
||||
-o /tmp/test_speech.wav \
|
||||
http://localhost:8072/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Hello World"}' 2>&1
|
||||
ls -la /tmp/test_speech.wav 2>&1
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
# TTS synthesis test
|
||||
curl -v -o /tmp/test_tts.wav \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Hello world, this is a test of the text to speech engine.","voice":"Vivian"}' \
|
||||
http://localhost:8072/v1/audio/speech 2>&1
|
||||
|
||||
echo "---"
|
||||
if [ -f /tmp/test_tts.wav ]; then
|
||||
ls -la /tmp/test_tts.wav
|
||||
file /tmp/test_tts.wav 2>/dev/null || echo "(file cmd not found)"
|
||||
python3 -c "
|
||||
import struct
|
||||
with open('/tmp/test_tts.wav','rb') as f:
|
||||
hdr = f.read(44)
|
||||
if hdr[:4] == b'RIFF':
|
||||
sz = struct.unpack('<I', hdr[4:8])[0]
|
||||
fmt = hdr[8:12]
|
||||
ch = struct.unpack('<H', hdr[22:24])[0]
|
||||
sr = struct.unpack('<I', hdr[24:28])[0]
|
||||
print(f'WAV: size={sz+8}, channels={ch}, sample_rate={sr}')
|
||||
else:
|
||||
print(f'Not WAV, first bytes: {hdr[:20]}')
|
||||
" 2>&1
|
||||
else
|
||||
echo "No output file created"
|
||||
fi
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# 1. Generate a reference audio from Vivian
|
||||
echo "[1] Generating reference audio from Vivian..."
|
||||
curl -s -o /tmp/vivian_ref.wav \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Dies ist ein Test der Sprachsynthese. Meine Stimme sollte geklont werden koennen.","voice":"Vivian","language":"german"}' \
|
||||
http://localhost:8072/v1/audio/speech
|
||||
echo "Generated /tmp/vivian_ref.wav: $(ls -la /tmp/vivian_ref.wav 2>&1)"
|
||||
|
||||
# 2. Train a custom voice from that audio
|
||||
echo ""
|
||||
echo "[2] Training custom voice 'meine_stimme' from reference audio..."
|
||||
curl -v -X POST http://localhost:8072/v1/voices/train \
|
||||
-F "audio=@/tmp/vivian_ref.wav" \
|
||||
-F "name=meine_stimme" \
|
||||
-F "text=Dies ist ein Test der Sprachsynthese." \
|
||||
-F "description=Vivian clone test" \
|
||||
-F "language=german" \
|
||||
2>&1
|
||||
|
||||
# 3. List voices
|
||||
echo ""
|
||||
echo "[3] Listing all voices..."
|
||||
curl -s http://localhost:8072/v1/voices 2>&1
|
||||
|
||||
# 4. Generate speech with the cloned voice
|
||||
echo ""
|
||||
echo "[4] Generating speech with cloned voice..."
|
||||
curl -s -o /tmp/clone_test.wav \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Hallo, ich bin eine geklonte Stimme. Das ist ziemlich cool.","voice":"meine_stimme","language":"german"}' \
|
||||
http://localhost:8072/v1/audio/speech
|
||||
echo "Generated /tmp/clone_test.wav: $(ls -la /tmp/clone_test.wav 2>&1)"
|
||||
|
||||
# 5. Voice details
|
||||
echo ""
|
||||
echo "[5] Voice details..."
|
||||
curl -s http://localhost:8072/v1/voices/meine_stimme 2>&1
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
echo "=== All containers ==="
|
||||
docker ps --format "{{.Names}} {{.Status}}" | sort
|
||||
|
||||
echo
|
||||
echo "=== TTS sidecar health ==="
|
||||
curl -s http://localhost:9090/api/tts/sidecar/health
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "=== Chat + Speak pipeline ==="
|
||||
result=$(curl -s -X POST http://localhost:9090/api/chat/speak \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"messages":[{"role":"user","content":"Say one sentence about the weather."}],"tts_voice":"vivian","tts_language":"en"}')
|
||||
|
||||
echo "$result" | python3 -c "
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
print('Text:', d.get('text','')[:300])
|
||||
audio = d.get('audio_b64','')
|
||||
print('Audio b64 len:', len(audio) if audio else 'NONE')
|
||||
if d.get('error'): print('ERROR:', d['error'])
|
||||
print('Timings:', d.get('timings',{}))
|
||||
"
|
||||
|
||||
echo
|
||||
echo "=== Containers still alive? ==="
|
||||
docker ps --format "{{.Names}} {{.Status}}" | sort
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
echo "=== TTS Proxy Test ==="
|
||||
curl -s -w '\nHTTP: %{http_code} Size: %{size_download}\n' \
|
||||
-o /tmp/proxy_verify.wav \
|
||||
http://localhost:9090/api/proxy/tts \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"input":"This is a verification test.","voice":"Vivian","language":"en","speed":1.0}'
|
||||
|
||||
echo "=== File check ==="
|
||||
file /tmp/proxy_verify.wav 2>/dev/null
|
||||
ls -la /tmp/proxy_verify.wav 2>/dev/null
|
||||
|
||||
echo "=== New speakText in HTML ==="
|
||||
curl -s http://localhost:9090/ | grep -c 'Generating audio'
|
||||
curl -s http://localhost:9090/ | grep -c 'TTS not ready'
|
||||
curl -s http://localhost:9090/ | grep -c 'targetDiv'
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
# Add timing debug prints to load_diffusion_model_state_dict in sd.py
|
||||
|
||||
SD_PY="/home/fabian/ComfyUI/comfy/sd.py"
|
||||
|
||||
# Check if already patched
|
||||
if grep -q 'BC250_DEBUG' "$SD_PY"; then
|
||||
echo "Already has debug prints"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Find the model_type FLOW line and add debug before get_model call
|
||||
# The sequence is roughly:
|
||||
# 1. logging.info("model_type ...")
|
||||
# 2. model = model_config.get_model(new_sd, "")
|
||||
# 3. model.load_model_weights(...)
|
||||
# 4. model.to(offload_device) or model_patcher creation
|
||||
|
||||
# Add debug after "model_type" print
|
||||
python3 -c "
|
||||
import re
|
||||
|
||||
with open('$SD_PY', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find 'model_type FLOW' or similar logging line and surrounding code
|
||||
# Add timing around get_model, load_model_weights, etc.
|
||||
target = 'model = model_config.get_model(new_sd, \"\")'
|
||||
if target in content:
|
||||
replacement = '''import time as _t; _ts = _t.time(); logging.warning(\"[BC250_DEBUG] Creating model skeleton...\") # BC250_DEBUG
|
||||
model = model_config.get_model(new_sd, \"\")
|
||||
logging.warning(f\"[BC250_DEBUG] Model skeleton created in {_t.time()-_ts:.1f}s\") # BC250_DEBUG'''
|
||||
content = content.replace(target, replacement, 1)
|
||||
|
||||
target2 = 'model.load_model_weights(new_sd, prefix)'
|
||||
if target2 in content:
|
||||
replacement2 = '''logging.warning(\"[BC250_DEBUG] Loading model weights...\") # BC250_DEBUG
|
||||
_ts2 = _t.time()
|
||||
model.load_model_weights(new_sd, prefix)
|
||||
logging.warning(f\"[BC250_DEBUG] Model weights loaded in {_t.time()-_ts2:.1f}s\") # BC250_DEBUG'''
|
||||
content = content.replace(target2, replacement2, 1)
|
||||
|
||||
with open('$SD_PY', 'w') as f:
|
||||
f.write(content)
|
||||
print('Debug prints added')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Verify ==="
|
||||
grep -n 'BC250_DEBUG' "$SD_PY"
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# Add remaining debug timing around load_model_weights and model.to()
|
||||
SD_PY="/home/fabian/ComfyUI/comfy/sd.py"
|
||||
|
||||
# Add before model.to(offload_device)
|
||||
python3 -c "
|
||||
with open('$SD_PY', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
old = ''' if not model_management.is_device_cpu(offload_device):
|
||||
model.to(offload_device)
|
||||
model.load_model_weights(new_sd, \"\", assign=model_patcher.is_dynamic())'''
|
||||
|
||||
new = ''' if not model_management.is_device_cpu(offload_device):
|
||||
logging.warning(f\"[BC250_DEBUG] Moving model to {offload_device}...\") # BC250_DEBUG
|
||||
_td = _t.time()
|
||||
model.to(offload_device)
|
||||
logging.warning(f\"[BC250_DEBUG] Model moved in {_t.time()-_td:.1f}s\") # BC250_DEBUG
|
||||
logging.warning(\"[BC250_DEBUG] Loading model weights (453 GGUF tensors)...\") # BC250_DEBUG
|
||||
_tw = _t.time()
|
||||
model.load_model_weights(new_sd, \"\", assign=model_patcher.is_dynamic())
|
||||
logging.warning(f\"[BC250_DEBUG] Model weights loaded in {_t.time()-_tw:.1f}s\") # BC250_DEBUG'''
|
||||
|
||||
content = content.replace(old, new, 1)
|
||||
with open('$SD_PY', 'w') as f:
|
||||
f.write(content)
|
||||
print('Done')
|
||||
"
|
||||
|
||||
echo "=== Verify ==="
|
||||
grep -n 'BC250_DEBUG' "$SD_PY"
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
pid=$(pgrep -f "python.*main.py" | head -1)
|
||||
echo "PID: $pid"
|
||||
|
||||
# Check ALL threads for running state
|
||||
echo "=== All thread states ==="
|
||||
for tid in $(ls /proc/$pid/task/ 2>/dev/null); do
|
||||
stat=$(cat /proc/$pid/task/$tid/stat 2>/dev/null)
|
||||
state=$(echo "$stat" | awk '{print $3}')
|
||||
cpu=$(echo "$stat" | awk '{print $14+$15}') # utime+stime
|
||||
wchan=$(cat /proc/$pid/task/$tid/wchan 2>/dev/null)
|
||||
if [ "$state" = "R" ] || [ "$cpu" -gt 1000 ] 2>/dev/null; then
|
||||
echo " ** TID $tid: state=$state cpu=$cpu wchan=$wchan **"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Running threads only ==="
|
||||
for tid in $(ls /proc/$pid/task/ 2>/dev/null); do
|
||||
state=$(cat /proc/$pid/task/$tid/stat 2>/dev/null | awk '{print $3}')
|
||||
if [ "$state" = "R" ]; then
|
||||
echo " RUNNING: TID $tid"
|
||||
cat /proc/$pid/task/$tid/wchan 2>/dev/null
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Top CPU threads (last column is cumulative CPU ticks) ==="
|
||||
for tid in $(ls /proc/$pid/task/ 2>/dev/null); do
|
||||
stat=$(cat /proc/$pid/task/$tid/stat 2>/dev/null)
|
||||
utime=$(echo "$stat" | awk '{print $14}')
|
||||
stime=$(echo "$stat" | awk '{print $15}')
|
||||
total=$((utime + stime))
|
||||
state=$(echo "$stat" | awk '{print $3}')
|
||||
if [ "$total" -gt 100 ]; then
|
||||
echo " TID $tid: state=$state total_ticks=$total"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== py-spy dump (venv) ==="
|
||||
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope > /dev/null 2>&1
|
||||
/home/fabian/ComfyUI/venv/bin/py-spy dump --pid $pid 2>&1 | head -80
|
||||
|
||||
echo ""
|
||||
echo "=== VRAM check ==="
|
||||
rocm-smi --showmeminfo vram 2>/dev/null | grep -E "Used|Total"
|
||||
@@ -0,0 +1,95 @@
|
||||
import torch
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
import gc
|
||||
|
||||
print(f'Device: {torch.cuda.get_device_name(0)}')
|
||||
print(f'Total Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB')
|
||||
print()
|
||||
|
||||
def mem_info():
|
||||
alloc = torch.cuda.memory_allocated() / 1e6
|
||||
reserved = torch.cuda.memory_reserved() / 1e6
|
||||
return f'alloc={alloc:.0f}MB, reserved={reserved:.0f}MB'
|
||||
|
||||
def test_attention(heads, seq_len, head_dim, dtype=torch.float32):
|
||||
label = f'Attn h={heads} s={seq_len} d={head_dim} {"fp32" if dtype==torch.float32 else "fp16"}'
|
||||
print(f'=== {label} ===', flush=True)
|
||||
|
||||
# Calculate memory needed
|
||||
attn_size = heads * seq_len * seq_len * (4 if dtype==torch.float32 else 2)
|
||||
qk_size = 2 * heads * seq_len * head_dim * (4 if dtype==torch.float32 else 2)
|
||||
total_est = (attn_size + qk_size) / 1e6
|
||||
print(f' Est memory: {total_est:.0f}MB ({mem_info()})', flush=True)
|
||||
|
||||
try:
|
||||
q = torch.randn(1, heads, seq_len, head_dim, device='cuda', dtype=dtype)
|
||||
k = torch.randn(1, heads, seq_len, head_dim, device='cuda', dtype=dtype)
|
||||
print(f' Q/K allocated ({mem_info()})', flush=True)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
t = time.time()
|
||||
scores = torch.matmul(q, k.transpose(-2, -1))
|
||||
torch.cuda.synchronize()
|
||||
elapsed = time.time() - t
|
||||
|
||||
print(f' OK: {elapsed:.3f}s, scores shape={list(scores.shape)} ({mem_info()})', flush=True)
|
||||
del q, k, scores
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f' FAIL: {e}', flush=True)
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
return False
|
||||
|
||||
# Progressive attention scaling
|
||||
test_attention(4, 512, 64)
|
||||
test_attention(8, 1024, 64)
|
||||
test_attention(8, 1024, 128)
|
||||
test_attention(16, 2048, 128)
|
||||
test_attention(24, 2048, 128)
|
||||
test_attention(24, 4096, 128) # full Lumina2 scale!
|
||||
|
||||
# If full scale fails in fp32, try fp16
|
||||
print()
|
||||
print('=== fp16 ATTENTION TESTS ===', flush=True)
|
||||
test_attention(24, 4096, 128, torch.float16)
|
||||
|
||||
# Test split attention approach (process in chunks)
|
||||
print()
|
||||
print('=== SPLIT ATTENTION (simulate ComfyUI split attn) ===', flush=True)
|
||||
try:
|
||||
heads = 24
|
||||
seq = 4096
|
||||
hd = 128
|
||||
chunk = 512 # process 512 tokens at a time
|
||||
|
||||
q = torch.randn(1, heads, seq, hd, device='cuda', dtype=torch.float32)
|
||||
k = torch.randn(1, heads, seq, hd, device='cuda', dtype=torch.float32)
|
||||
v = torch.randn(1, heads, seq, hd, device='cuda', dtype=torch.float32)
|
||||
out = torch.zeros(1, heads, seq, hd, device='cuda', dtype=torch.float32)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
t = time.time()
|
||||
for i in range(0, seq, chunk):
|
||||
q_chunk = q[:, :, i:i+chunk, :]
|
||||
scores = torch.matmul(q_chunk, k.transpose(-2, -1))
|
||||
attn = torch.softmax(scores, dim=-1)
|
||||
out[:, :, i:i+chunk, :] = torch.matmul(attn, v)
|
||||
del scores, attn
|
||||
torch.cuda.synchronize()
|
||||
elapsed = time.time() - t
|
||||
print(f' Split attention OK: {elapsed:.3f}s', flush=True)
|
||||
del q, k, v, out
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
except Exception as e:
|
||||
print(f' Split attention FAIL: {e}', flush=True)
|
||||
|
||||
print()
|
||||
print('=== DONE ===', flush=True)
|
||||
torch.cuda.synchronize()
|
||||
os._exit(0)
|
||||
@@ -0,0 +1,631 @@
|
||||
"""
|
||||
BC-250 gfx1010 Comprehensive Monkey-Patch v10
|
||||
1. Replaces torch.softmax with manual implementation (VGPR overflow fix)
|
||||
2. Replaces SDPA with manual implementation
|
||||
3. Patches GGUF cast_bias_weight to dequant on CPU (avoids GPU page-fault hangs)
|
||||
4. Pre-clones mmap'd tensor data before GPU transfer (XNACK workaround)
|
||||
5. Pre-warms GPU context and caching allocator
|
||||
6. Forces text encoder to CPU (memory constraint)
|
||||
7. VAE decode on CPU float32 (bypasses GPU managed memory issues)
|
||||
8. Sets torch threads to all CPU cores (faster CPU ops + VAE decode)
|
||||
9. Caches VAE model on CPU (avoids reload each generation)
|
||||
10. Startup preloading: submits warmup prompt to preload all models on boot
|
||||
|
||||
v10 changes:
|
||||
- Background warmup thread submits 64x64 @ 1 step prompt after server starts
|
||||
- All models (CLIP, UNET, VAE) preloaded before user interaction
|
||||
- Models configurable via BC250_PRELOAD_* env vars
|
||||
|
||||
BC-250 APU / gfx1010: XNACK disabled, no GPU page fault handling.
|
||||
GPU copy shader hangs on non-resident pages (mmap'd or swapped).
|
||||
Place in ComfyUI root and import as first line of main.py.
|
||||
"""
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import os
|
||||
import sys
|
||||
import gc
|
||||
import logging
|
||||
import threading
|
||||
import json
|
||||
import time as _time
|
||||
import threading
|
||||
import json
|
||||
import time as _time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# === THREAD CONFIGURATION ===
|
||||
# BC-250 has 12 threads (6C/12T Zen2). Use all for CPU-heavy work (VAE, CLIP, dequant).
|
||||
_NUM_THREADS = int(os.environ.get("BC250_NUM_THREADS", str(os.cpu_count() or 12)))
|
||||
torch.set_num_threads(_NUM_THREADS)
|
||||
# Note: set_num_interop_threads must be called before any parallel op, skip to avoid deadlock
|
||||
logger.warning(f"[BC-250] Torch threads: intra-op={_NUM_THREADS}")
|
||||
|
||||
SAFE_SOFTMAX_THRESHOLD = int(os.environ.get("BC250_SOFTMAX_THRESHOLD", "4096"))
|
||||
|
||||
_original_softmax = torch.nn.functional.softmax
|
||||
_original_tensor_softmax = torch.Tensor.softmax
|
||||
_original_sdpa = torch.nn.functional.scaled_dot_product_attention
|
||||
|
||||
# === MMAP PRE-CLONE PATCH ===
|
||||
_original_module_apply = torch.nn.Module._apply
|
||||
|
||||
def _bc250_safe_apply(self, fn, recurse=True):
|
||||
"""Pre-clone mmap'd CPU tensor data before GPU transfer to avoid XNACK hangs."""
|
||||
for key, param in self._parameters.items():
|
||||
if param is not None and param.device.type == 'cpu':
|
||||
param.data = param.data.clone()
|
||||
for key, buf in self._buffers.items():
|
||||
if buf is not None and buf.device.type == 'cpu':
|
||||
self._buffers[key] = buf.clone()
|
||||
return _original_module_apply(self, fn, recurse)
|
||||
|
||||
# === SOFTMAX PATCH ===
|
||||
|
||||
def _safe_softmax_impl(input, dim=-1):
|
||||
x_max = input.max(dim=dim, keepdim=True).values
|
||||
exp_x = torch.exp(input - x_max)
|
||||
return exp_x / exp_x.sum(dim=dim, keepdim=True)
|
||||
|
||||
def patched_softmax(input, dim=None, _stacklevel=3, dtype=None):
|
||||
if dim is None:
|
||||
dim = -1
|
||||
if dtype is not None:
|
||||
input = input.to(dtype)
|
||||
if input.is_cuda and input.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual F.softmax triggered: shape={list(input.shape)}, dim={dim}, threshold={SAFE_SOFTMAX_THRESHOLD}")
|
||||
patched_softmax._logged = True
|
||||
return _safe_softmax_impl(input, dim)
|
||||
return _original_softmax(input, dim=dim)
|
||||
|
||||
def patched_tensor_softmax(self, dim=-1, dtype=None):
|
||||
if dtype is not None:
|
||||
self = self.to(dtype)
|
||||
if self.is_cuda and self.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_tensor_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual softmax triggered: shape={list(self.shape)}, dim={dim}, threshold={SAFE_SOFTMAX_THRESHOLD}")
|
||||
patched_tensor_softmax._logged = True
|
||||
return _safe_softmax_impl(self, dim)
|
||||
return _original_tensor_softmax(self, dim=dim)
|
||||
|
||||
def _safe_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
L, S = query.size(-2), key.size(-2)
|
||||
if scale is None:
|
||||
scale = query.size(-1) ** -0.5
|
||||
attn_weight = torch.matmul(query, key.transpose(-2, -1)) * scale
|
||||
if is_causal:
|
||||
causal_mask = torch.triu(torch.ones(L, S, dtype=torch.bool, device=query.device), diagonal=1)
|
||||
attn_weight = attn_weight.masked_fill(causal_mask, float('-inf'))
|
||||
if attn_mask is not None:
|
||||
if attn_mask.dtype == torch.bool:
|
||||
attn_weight = attn_weight.masked_fill(~attn_mask, float('-inf'))
|
||||
else:
|
||||
attn_weight = attn_weight + attn_mask
|
||||
attn_weight = _safe_softmax_impl(attn_weight, dim=-1)
|
||||
if dropout_p > 0.0:
|
||||
attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p)
|
||||
return torch.matmul(attn_weight, value)
|
||||
|
||||
def patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
S = key.size(-2)
|
||||
if query.is_cuda and S > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_sdpa, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual SDPA triggered: Q={list(query.shape)}, K={list(key.shape)}, S={S}, threshold={SAFE_SOFTMAX_THRESHOLD}")
|
||||
patched_sdpa._logged = True
|
||||
return _safe_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
return _original_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
|
||||
# === GGUF CPU-DEQUANT PATCH (cast_bias_weight override) ===
|
||||
_gguf_patched = False
|
||||
|
||||
def _try_patch_gguf():
|
||||
"""Patch GGMLLayer.cast_bias_weight to dequant on CPU, send floats to GPU."""
|
||||
global _gguf_patched
|
||||
if _gguf_patched:
|
||||
return True
|
||||
|
||||
ops_mod = None
|
||||
dequant_mod = None
|
||||
for name, mod in sys.modules.items():
|
||||
if mod is None:
|
||||
continue
|
||||
if name.endswith('.ops') and 'GGUF' in name:
|
||||
ops_mod = mod
|
||||
if name.endswith('.dequant') and 'GGUF' in name:
|
||||
dequant_mod = mod
|
||||
|
||||
if ops_mod is None or dequant_mod is None:
|
||||
return False
|
||||
|
||||
GGMLLayer = getattr(ops_mod, 'GGMLLayer', None)
|
||||
is_quantized_fn = getattr(dequant_mod, 'is_quantized', None)
|
||||
if GGMLLayer is None or is_quantized_fn is None:
|
||||
return False
|
||||
|
||||
def _bc250_cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None):
|
||||
"""Dequant on CPU, send float results to GPU.
|
||||
|
||||
Cannot use .to(device) on quantized GGUF tensors from mmap'd files
|
||||
(GPU copy shader hangs on non-resident pages, XNACK disabled).
|
||||
Dequant to float on CPU, then transfer dequantized float to GPU.
|
||||
"""
|
||||
import comfy.model_management
|
||||
import comfy.ops
|
||||
|
||||
if input is not None:
|
||||
if dtype is None:
|
||||
dtype = getattr(input, "dtype", torch.float32)
|
||||
if bias_dtype is None:
|
||||
bias_dtype = dtype
|
||||
if device is None:
|
||||
device = input.device
|
||||
|
||||
non_blocking = comfy.model_management.device_supports_non_blocking(device)
|
||||
|
||||
bias = None
|
||||
if s.bias is not None:
|
||||
if is_quantized_fn(s.bias):
|
||||
bias = s.get_weight(s.bias, bias_dtype)
|
||||
else:
|
||||
bias = s.get_weight(s.bias.to(device), bias_dtype)
|
||||
bias = comfy.ops.cast_to(bias, bias_dtype, device, non_blocking=non_blocking, copy=False)
|
||||
|
||||
if is_quantized_fn(s.weight):
|
||||
weight = s.get_weight(s.weight, dtype)
|
||||
else:
|
||||
weight = s.get_weight(s.weight.to(device), dtype)
|
||||
weight = comfy.ops.cast_to(weight, dtype, device, non_blocking=non_blocking, copy=False)
|
||||
return weight, bias
|
||||
|
||||
GGMLLayer.cast_bias_weight = _bc250_cast_bias_weight
|
||||
|
||||
_gguf_patched = True
|
||||
logger.warning("[BC-250] GGUF cast_bias_weight patched (CPU dequant)")
|
||||
return True
|
||||
|
||||
# === IMPORT HOOK for deferred GGUF patching ===
|
||||
|
||||
class _GGUFImportWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if 'GGUF' in fullname and ('dequant' in fullname or 'ops' in fullname):
|
||||
return self
|
||||
return None
|
||||
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
|
||||
if _try_patch_gguf():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === TEXT ENCODER CPU PATCH ===
|
||||
_te_patched = False
|
||||
|
||||
def _try_patch_text_encoder_device():
|
||||
global _te_patched
|
||||
if _te_patched:
|
||||
return True
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm is None:
|
||||
return False
|
||||
mm.text_encoder_device = lambda: torch.device("cpu")
|
||||
mm.text_encoder_offload_device = lambda: torch.device("cpu")
|
||||
|
||||
_te_patched = True
|
||||
logger.warning("[BC-250] Text encoder forced to CPU (memory constraint)")
|
||||
return True
|
||||
|
||||
# === VAE CPU FLOAT32 DECODE PATCH ===
|
||||
# Decode VAE on CPU using float32 (not fp16). fp16 on CPU is emulated (10x slower).
|
||||
# Cannot use GPU because UNet managed memory blocks new GPU allocations (XNACK disabled).
|
||||
# 320MB VAE at float32 = 640MB RAM. For 256x256: ~2-3 min on 12-thread CPU.
|
||||
|
||||
_vae_patched = False
|
||||
_vae_cached = False # Track whether VAE is already loaded to CPU float32
|
||||
|
||||
def _try_patch_vae_cpu():
|
||||
"""Patch comfy.sd.VAE to decode on CPU with float32, with persistent caching."""
|
||||
global _vae_patched
|
||||
if _vae_patched:
|
||||
return True
|
||||
|
||||
sd_mod = sys.modules.get('comfy.sd')
|
||||
if sd_mod is None:
|
||||
return False
|
||||
|
||||
VAE = getattr(sd_mod, 'VAE', None)
|
||||
if VAE is None:
|
||||
return False
|
||||
|
||||
_original_vae_encode = getattr(VAE, 'encode', None)
|
||||
|
||||
def _ensure_vae_on_cpu_f32(self):
|
||||
"""Move VAE to CPU float32 once, then keep it cached."""
|
||||
global _vae_cached
|
||||
if not _vae_cached or next(self.first_stage_model.parameters()).dtype != torch.float32:
|
||||
logger.warning("[BC-250] Loading VAE to CPU float32 (will stay cached)")
|
||||
self.first_stage_model.to(torch.float32).to(torch.device("cpu"))
|
||||
self.first_stage_model.eval()
|
||||
_vae_cached = True
|
||||
# Prevent ComfyUI model_management from offloading the VAE
|
||||
self.disable_offload = True
|
||||
|
||||
def _bc250_vae_decode(self, samples_in, vae_options={}):
|
||||
"""CPU float32 VAE decode — bypasses GPU managed memory entirely.
|
||||
|
||||
The UNet (5032MB managed memory) blocks new GPU allocations
|
||||
when its pages are swapped by the OS (XNACK disabled on gfx1010).
|
||||
float32 on CPU is ~5x faster than fp16 (which requires emulation).
|
||||
VAE stays cached on CPU after first load — no re-conversion needed.
|
||||
"""
|
||||
import time
|
||||
t0 = time.time()
|
||||
logger.warning("[BC-250] VAE decode: CPU float32 (cached)")
|
||||
|
||||
self.throw_exception_if_invalid()
|
||||
|
||||
if self.latent_dim == 2 and samples_in.ndim == 5:
|
||||
samples_in = samples_in[:, :, 0]
|
||||
|
||||
cpu = torch.device("cpu")
|
||||
_ensure_vae_on_cpu_f32(self)
|
||||
|
||||
pixel_samples = None
|
||||
with torch.no_grad():
|
||||
for x in range(samples_in.shape[0]):
|
||||
sample = samples_in[x:x+1].to(torch.float32)
|
||||
decoded = self.first_stage_model.decode(sample, **vae_options)
|
||||
# Squeeze temporal dim for 3D video autoencoders (single image)
|
||||
if decoded.ndim == 5:
|
||||
decoded = decoded[:, :, 0]
|
||||
out = self.process_output(decoded.float())
|
||||
if pixel_samples is None:
|
||||
pixel_samples = torch.empty(
|
||||
(samples_in.shape[0],) + tuple(out.shape[1:]),
|
||||
device=cpu
|
||||
)
|
||||
pixel_samples[x:x+1] = out
|
||||
del decoded, sample
|
||||
|
||||
# NCHW → NHWC (same as original ComfyUI VAE.decode line 977)
|
||||
pixel_samples = pixel_samples.movedim(1, -1)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
logger.warning(f"[BC-250] VAE decode complete in {elapsed:.1f}s")
|
||||
return pixel_samples
|
||||
|
||||
VAE.decode = _bc250_vae_decode
|
||||
|
||||
if _original_vae_encode is not None:
|
||||
def _bc250_vae_encode(self, pixel_samples):
|
||||
"""CPU float32 VAE encode (cached)."""
|
||||
import time
|
||||
t0 = time.time()
|
||||
logger.warning("[BC-250] VAE encode: CPU float32 (cached)")
|
||||
self.throw_exception_if_invalid()
|
||||
|
||||
_ensure_vae_on_cpu_f32(self)
|
||||
|
||||
with torch.no_grad():
|
||||
pixels_in = self.process_input(pixel_samples).to(torch.float32)
|
||||
result = self.first_stage_model.encode(pixels_in).float()
|
||||
|
||||
elapsed = time.time() - t0
|
||||
logger.warning(f"[BC-250] VAE encode complete in {elapsed:.1f}s")
|
||||
return result
|
||||
|
||||
VAE.encode = _bc250_vae_encode
|
||||
|
||||
_vae_patched = True
|
||||
logger.warning("[BC-250] VAE patched: CPU float32 decode/encode with caching (bypass GPU managed memory)")
|
||||
return True
|
||||
|
||||
class _SDModuleWatcher:
|
||||
"""Patches comfy.sd.VAE after it's imported."""
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.sd':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_vae_cpu():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
class _ModelMgmtWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.model_management':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_text_encoder_device():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === GPU MEMORY CLEANUP HOOK ===
|
||||
# Patch model_management.load_models_gpu to clean up before loading
|
||||
|
||||
_load_patched = False
|
||||
|
||||
def _try_patch_load_models():
|
||||
"""Add GPU memory cleanup before model loading."""
|
||||
global _load_patched
|
||||
if _load_patched:
|
||||
return True
|
||||
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm is None:
|
||||
return False
|
||||
|
||||
_original_load = getattr(mm, 'load_models_gpu', None)
|
||||
if _original_load is None:
|
||||
return False
|
||||
|
||||
def _bc250_load_models_gpu(models, *args, **kwargs):
|
||||
"""Clean GPU cache before loading models to prevent memory pressure hangs."""
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
return _original_load(models, *args, **kwargs)
|
||||
|
||||
mm.load_models_gpu = _bc250_load_models_gpu
|
||||
_load_patched = True
|
||||
logger.warning("[BC-250] GPU memory cleanup hook installed (load_models_gpu)")
|
||||
return True
|
||||
|
||||
# === STARTUP PRELOAD ===
|
||||
|
||||
_PRELOAD_CLIP = os.environ.get("BC250_PRELOAD_CLIP", "Qwen_3_4b-Q8_0.gguf")
|
||||
_PRELOAD_UNET = os.environ.get("BC250_PRELOAD_UNET", "z_image_turbo-Q5_K_S.gguf")
|
||||
_PRELOAD_VAE = os.environ.get("BC250_PRELOAD_VAE", "ae.safetensors")
|
||||
_PRELOAD_PORT = int(os.environ.get("BC250_PRELOAD_PORT", "8188"))
|
||||
_PRELOAD_ENABLED = os.environ.get("BC250_PRELOAD", "1") == "1"
|
||||
|
||||
def _preload_models():
|
||||
"""Background thread: wait for ComfyUI server, then submit a warmup prompt."""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
url = f"http://127.0.0.1:{_PRELOAD_PORT}"
|
||||
|
||||
# Wait for server to be ready (max 120s)
|
||||
logger.warning("[BC-250] Preload: waiting for ComfyUI server...")
|
||||
for _ in range(240):
|
||||
try:
|
||||
urllib.request.urlopen(f"{url}/api/system_stats", timeout=2)
|
||||
break
|
||||
except (urllib.error.URLError, OSError, ConnectionRefusedError):
|
||||
_time.sleep(0.5)
|
||||
else:
|
||||
logger.warning("[BC-250] Preload: server not ready after 120s, skipping")
|
||||
return
|
||||
|
||||
logger.warning("[BC-250] Preload: server ready, submitting warmup prompt...")
|
||||
|
||||
warmup = {
|
||||
"1": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": _PRELOAD_CLIP, "type": "lumina2"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "warmup", "clip": ["1", 0]}},
|
||||
"3": {"class_type": "CLIPTextEncode", "inputs": {"text": "", "clip": ["1", 0]}},
|
||||
"4": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": _PRELOAD_UNET}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 64, "height": 64, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"seed": 1, "steps": 1, "cfg": 1.0, "sampler_name": "euler",
|
||||
"scheduler": "normal", "denoise": 1.0,
|
||||
"model": ["4", 0], "positive": ["2", 0], "negative": ["3", 0], "latent_image": ["5", 0]
|
||||
}},
|
||||
"7": {"class_type": "VAELoader", "inputs": {"vae_name": _PRELOAD_VAE}},
|
||||
"8": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["7", 0]}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "_warmup", "images": ["8", 0]}}
|
||||
}
|
||||
|
||||
payload = json.dumps({"prompt": warmup}).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{url}/api/prompt",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
data = json.loads(resp.read())
|
||||
prompt_id = data.get("prompt_id", "unknown")
|
||||
logger.warning(f"[BC-250] Preload: warmup prompt queued (id={prompt_id})")
|
||||
|
||||
# Wait for completion (max 5min)
|
||||
for _ in range(300):
|
||||
_time.sleep(1)
|
||||
try:
|
||||
hist_resp = urllib.request.urlopen(f"{url}/api/history/{prompt_id}", timeout=5)
|
||||
hist = json.loads(hist_resp.read())
|
||||
if prompt_id in hist:
|
||||
logger.warning("[BC-250] Preload: all models loaded and cached. Ready for user prompts.")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("[BC-250] Preload: warmup timed out after 5min")
|
||||
except Exception as e:
|
||||
logger.warning(f"[BC-250] Preload: warmup failed: {e}")
|
||||
|
||||
|
||||
def _start_preload_thread():
|
||||
if not _PRELOAD_ENABLED:
|
||||
logger.warning("[BC-250] Preload: disabled (BC250_PRELOAD=0)")
|
||||
return
|
||||
t = threading.Thread(target=_preload_models, daemon=True, name="BC250-Preload")
|
||||
t.start()
|
||||
logger.warning("[BC-250] Preload: background warmup thread started")
|
||||
|
||||
# === STARTUP PRELOAD ===
|
||||
|
||||
_PRELOAD_CLIP = os.environ.get("BC250_PRELOAD_CLIP", "Qwen_3_4b-Q8_0.gguf")
|
||||
_PRELOAD_UNET = os.environ.get("BC250_PRELOAD_UNET", "z_image_turbo-Q5_K_S.gguf")
|
||||
_PRELOAD_VAE = os.environ.get("BC250_PRELOAD_VAE", "ae.safetensors")
|
||||
_PRELOAD_PORT = int(os.environ.get("BC250_PRELOAD_PORT", "8188"))
|
||||
_PRELOAD_ENABLED = os.environ.get("BC250_PRELOAD", "1") == "1"
|
||||
|
||||
def _preload_models():
|
||||
"""Background thread: wait for ComfyUI server, then submit a warmup prompt."""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
url = f"http://127.0.0.1:{_PRELOAD_PORT}"
|
||||
|
||||
# Wait for server to be ready (max 120s)
|
||||
logger.warning("[BC-250] Preload: waiting for ComfyUI server...")
|
||||
for _ in range(240):
|
||||
try:
|
||||
urllib.request.urlopen(f"{url}/api/system_stats", timeout=2)
|
||||
break
|
||||
except (urllib.error.URLError, OSError, ConnectionRefusedError):
|
||||
_time.sleep(0.5)
|
||||
else:
|
||||
logger.warning("[BC-250] Preload: server not ready after 120s, skipping")
|
||||
return
|
||||
|
||||
logger.warning("[BC-250] Preload: server ready, submitting warmup prompt...")
|
||||
|
||||
warmup = {
|
||||
"1": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": _PRELOAD_CLIP, "type": "lumina2"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "warmup", "clip": ["1", 0]}},
|
||||
"3": {"class_type": "CLIPTextEncode", "inputs": {"text": "", "clip": ["1", 0]}},
|
||||
"4": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": _PRELOAD_UNET}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 64, "height": 64, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"seed": 1, "steps": 1, "cfg": 1.0, "sampler_name": "euler",
|
||||
"scheduler": "normal", "denoise": 1.0,
|
||||
"model": ["4", 0], "positive": ["2", 0], "negative": ["3", 0], "latent_image": ["5", 0]
|
||||
}},
|
||||
"7": {"class_type": "VAELoader", "inputs": {"vae_name": _PRELOAD_VAE}},
|
||||
"8": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["7", 0]}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "_warmup", "images": ["8", 0]}}
|
||||
}
|
||||
|
||||
payload = json.dumps({"prompt": warmup}).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{url}/api/prompt",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
data = json.loads(resp.read())
|
||||
prompt_id = data.get("prompt_id", "unknown")
|
||||
logger.warning(f"[BC-250] Preload: warmup prompt queued (id={prompt_id})")
|
||||
|
||||
# Wait for completion (max 5min)
|
||||
for _ in range(300):
|
||||
_time.sleep(1)
|
||||
try:
|
||||
hist_resp = urllib.request.urlopen(f"{url}/api/history/{prompt_id}", timeout=5)
|
||||
hist = json.loads(hist_resp.read())
|
||||
if prompt_id in hist:
|
||||
logger.warning("[BC-250] Preload: all models loaded and cached. Ready for user prompts.")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("[BC-250] Preload: warmup timed out after 5min")
|
||||
except Exception as e:
|
||||
logger.warning(f"[BC-250] Preload: warmup failed: {e}")
|
||||
|
||||
|
||||
def _start_preload_thread():
|
||||
if not _PRELOAD_ENABLED:
|
||||
logger.warning("[BC-250] Preload: disabled (BC250_PRELOAD=0)")
|
||||
return
|
||||
t = threading.Thread(target=_preload_models, daemon=True, name="BC250-Preload")
|
||||
t.start()
|
||||
logger.warning("[BC-250] Preload: background warmup thread started")
|
||||
|
||||
# === INSTALL ===
|
||||
|
||||
def _prewarm_gpu():
|
||||
try:
|
||||
if not torch.cuda.is_available():
|
||||
return
|
||||
dummy = torch.zeros(1, device='cuda')
|
||||
_ = dummy + 1
|
||||
torch.cuda.synchronize()
|
||||
del dummy
|
||||
torch.cuda.empty_cache()
|
||||
logger.warning("[BC-250] GPU pre-warmed (context + allocator ready)")
|
||||
except Exception as e:
|
||||
logger.warning(f"[BC-250] GPU pre-warm failed: {e}")
|
||||
|
||||
|
||||
def install():
|
||||
# Mmap pre-clone patch
|
||||
torch.nn.Module._apply = _bc250_safe_apply
|
||||
logger.warning("[BC-250] Mmap pre-clone patch installed (XNACK workaround)")
|
||||
|
||||
# Softmax patches
|
||||
torch.nn.functional.softmax = patched_softmax
|
||||
torch.Tensor.softmax = patched_tensor_softmax
|
||||
torch.nn.functional.scaled_dot_product_attention = patched_sdpa
|
||||
logger.warning(f"[BC-250] Softmax monkey-patch installed (threshold={SAFE_SOFTMAX_THRESHOLD})")
|
||||
|
||||
# GGUF deferred cast_bias_weight patch
|
||||
sys.meta_path.insert(0, _GGUFImportWatcher())
|
||||
logger.warning("[BC-250] GGUF CPU-dequant hook registered (cast_bias_weight)")
|
||||
|
||||
# Text encoder CPU patch
|
||||
sys.meta_path.insert(0, _ModelMgmtWatcher())
|
||||
|
||||
# VAE CPU-only patch
|
||||
sys.meta_path.insert(0, _SDModuleWatcher())
|
||||
|
||||
# Try immediate patches if modules already loaded
|
||||
_try_patch_gguf()
|
||||
_try_patch_text_encoder_device()
|
||||
_try_patch_vae_cpu()
|
||||
_try_patch_load_models()
|
||||
|
||||
# Pre-warm GPU
|
||||
_prewarm_gpu()
|
||||
|
||||
# Start background preload thread
|
||||
_start_preload_thread()
|
||||
|
||||
install()
|
||||
@@ -0,0 +1,539 @@
|
||||
"""
|
||||
BC-250 gfx1010 Comprehensive Monkey-Patch v11
|
||||
1. Softmax: manual impl for dim > threshold (VGPR overflow fix)
|
||||
2. SDPA: manual impl for large sequences
|
||||
3. GGUF: GPU dequant with weight cache (eliminates per-step dequant)
|
||||
4. Mmap: pre-clones non-GGUF tensor data before GPU transfer
|
||||
5. GPU: pre-warms context and caching allocator
|
||||
6. CLIP: forces text encoder to CPU (memory constraint)
|
||||
7. VAE: GPU fp16 decode with persistent caching (shared memory APU)
|
||||
8. Threads: all CPU cores for intra-op parallelism
|
||||
9. Preload: background warmup prompt on server start
|
||||
|
||||
NOTE: mlockall REMOVED — on APU with shared memory, pinning 10GB of mmap'd
|
||||
GGUF files leaves no room for GPU GTT allocations → OOM kill.
|
||||
The kernel page cache handles this correctly without mlockall.
|
||||
|
||||
v11 changes vs v10:
|
||||
- GPU dequant instead of CPU (GGUF dequant ops are pure PyTorch, run on GPU)
|
||||
- Weight cache: dequanted fp16 weights cached per-layer, reused across steps
|
||||
- mlockall() to pin process memory in RAM (no zram/swap penalty)
|
||||
- Removed duplicate preload section
|
||||
- Clean rewrite
|
||||
|
||||
BC-250 APU / gfx1010: XNACK disabled, no GPU page fault handling.
|
||||
Place in ComfyUI root and import as first line of main.py.
|
||||
"""
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import os
|
||||
import sys
|
||||
import gc
|
||||
import logging
|
||||
import threading
|
||||
import json
|
||||
import time as _time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# === THREAD CONFIGURATION ===
|
||||
_NUM_THREADS = int(os.environ.get("BC250_NUM_THREADS", str(os.cpu_count() or 12)))
|
||||
torch.set_num_threads(_NUM_THREADS)
|
||||
logger.warning(f"[BC-250] Torch threads: intra-op={_NUM_THREADS}")
|
||||
|
||||
SAFE_SOFTMAX_THRESHOLD = int(os.environ.get("BC250_SOFTMAX_THRESHOLD", "4096"))
|
||||
|
||||
_original_softmax = torch.nn.functional.softmax
|
||||
_original_tensor_softmax = torch.Tensor.softmax
|
||||
_original_sdpa = torch.nn.functional.scaled_dot_product_attention
|
||||
|
||||
# === MMAP PRE-CLONE PATCH ===
|
||||
_original_module_apply = torch.nn.Module._apply
|
||||
|
||||
def _bc250_safe_apply(self, fn, recurse=True):
|
||||
"""Pre-clone mmap'd CPU tensor data before GPU transfer (XNACK workaround).
|
||||
Note: GGMLTensor.clone() returns self, so GGUF weights are unaffected.
|
||||
They're handled by GGMLTensor.to() which preserves metadata."""
|
||||
for key, param in self._parameters.items():
|
||||
if param is not None and param.device.type == 'cpu':
|
||||
param.data = param.data.clone()
|
||||
for key, buf in self._buffers.items():
|
||||
if buf is not None and buf.device.type == 'cpu':
|
||||
self._buffers[key] = buf.clone()
|
||||
return _original_module_apply(self, fn, recurse)
|
||||
|
||||
# === SOFTMAX PATCH ===
|
||||
|
||||
def _safe_softmax_impl(input, dim=-1):
|
||||
x_max = input.max(dim=dim, keepdim=True).values
|
||||
exp_x = torch.exp(input - x_max)
|
||||
return exp_x / exp_x.sum(dim=dim, keepdim=True)
|
||||
|
||||
def patched_softmax(input, dim=None, _stacklevel=3, dtype=None):
|
||||
if dim is None:
|
||||
dim = -1
|
||||
if dtype is not None:
|
||||
input = input.to(dtype)
|
||||
if input.is_cuda and input.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual F.softmax: shape={list(input.shape)}, dim={dim}")
|
||||
patched_softmax._logged = True
|
||||
return _safe_softmax_impl(input, dim)
|
||||
return _original_softmax(input, dim=dim)
|
||||
|
||||
def patched_tensor_softmax(self, dim=-1, dtype=None):
|
||||
if dtype is not None:
|
||||
self = self.to(dtype)
|
||||
if self.is_cuda and self.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_tensor_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual softmax: shape={list(self.shape)}, dim={dim}")
|
||||
patched_tensor_softmax._logged = True
|
||||
return _safe_softmax_impl(self, dim)
|
||||
return _original_tensor_softmax(self, dim=dim)
|
||||
|
||||
def _safe_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
L, S = query.size(-2), key.size(-2)
|
||||
if scale is None:
|
||||
scale = query.size(-1) ** -0.5
|
||||
attn_weight = torch.matmul(query, key.transpose(-2, -1)) * scale
|
||||
if is_causal:
|
||||
causal_mask = torch.triu(torch.ones(L, S, dtype=torch.bool, device=query.device), diagonal=1)
|
||||
attn_weight = attn_weight.masked_fill(causal_mask, float('-inf'))
|
||||
if attn_mask is not None:
|
||||
if attn_mask.dtype == torch.bool:
|
||||
attn_weight = attn_weight.masked_fill(~attn_mask, float('-inf'))
|
||||
else:
|
||||
attn_weight = attn_weight + attn_mask
|
||||
attn_weight = _safe_softmax_impl(attn_weight, dim=-1)
|
||||
if dropout_p > 0.0:
|
||||
attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p)
|
||||
return torch.matmul(attn_weight, value)
|
||||
|
||||
def patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
S = key.size(-2)
|
||||
if query.is_cuda and S > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_sdpa, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual SDPA: Q={list(query.shape)}, S={S}")
|
||||
patched_sdpa._logged = True
|
||||
return _safe_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
return _original_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
|
||||
# === GGUF WEIGHT CACHE + GPU DEQUANT ===
|
||||
_gguf_patched = False
|
||||
_weight_cache = {}
|
||||
_weight_cache_bytes = 0
|
||||
_WEIGHT_CACHE_MB = int(os.environ.get("BC250_WEIGHT_CACHE_MB", "0"))
|
||||
|
||||
def _try_patch_gguf():
|
||||
"""Patch GGMLLayer.cast_bias_weight: GPU dequant + weight caching."""
|
||||
global _gguf_patched
|
||||
if _gguf_patched:
|
||||
return True
|
||||
|
||||
ops_mod = None
|
||||
dequant_mod = None
|
||||
for name, mod in sys.modules.items():
|
||||
if mod is None:
|
||||
continue
|
||||
if name.endswith('.ops') and 'GGUF' in name:
|
||||
ops_mod = mod
|
||||
if name.endswith('.dequant') and 'GGUF' in name:
|
||||
dequant_mod = mod
|
||||
|
||||
if ops_mod is None or dequant_mod is None:
|
||||
return False
|
||||
|
||||
GGMLLayer = getattr(ops_mod, 'GGMLLayer', None)
|
||||
is_quantized_fn = getattr(dequant_mod, 'is_quantized', None)
|
||||
if GGMLLayer is None or is_quantized_fn is None:
|
||||
return False
|
||||
|
||||
_original_cast = getattr(GGMLLayer, 'cast_bias_weight', None)
|
||||
cache_budget = _WEIGHT_CACHE_MB * 1024 * 1024
|
||||
|
||||
def _bc250_cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None):
|
||||
"""GPU dequant with optional weight caching.
|
||||
|
||||
With --highvram, GGUF weights are already on GPU. Dequant happens
|
||||
via PyTorch tensor ops on GPU (parallel) instead of CPU (sequential).
|
||||
If weight cache is enabled (BC250_WEIGHT_CACHE_MB > 0), dequanted
|
||||
weights are cached per-layer to eliminate dequant on steps 2+.
|
||||
"""
|
||||
global _weight_cache_bytes
|
||||
import comfy.model_management
|
||||
import comfy.ops
|
||||
|
||||
if input is not None:
|
||||
if dtype is None:
|
||||
dtype = getattr(input, "dtype", torch.float32)
|
||||
if bias_dtype is None:
|
||||
bias_dtype = dtype
|
||||
if device is None:
|
||||
device = input.device
|
||||
|
||||
non_blocking = comfy.model_management.device_supports_non_blocking(device)
|
||||
|
||||
# Check weight cache
|
||||
if cache_budget > 0:
|
||||
cache_key = id(s)
|
||||
cached = _weight_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
# Bias
|
||||
bias = None
|
||||
if s.bias is not None:
|
||||
bias = s.get_weight(s.bias.to(device), bias_dtype)
|
||||
bias = comfy.ops.cast_to(bias, bias_dtype, device, non_blocking=non_blocking, copy=False)
|
||||
|
||||
# Weight: .to(device) moves GGMLTensor to GPU, get_weight dequants on GPU
|
||||
weight = s.get_weight(s.weight.to(device), dtype)
|
||||
weight = comfy.ops.cast_to(weight, dtype, device, non_blocking=non_blocking, copy=False)
|
||||
|
||||
# Cache if within budget
|
||||
if cache_budget > 0:
|
||||
entry_bytes = weight.nelement() * weight.element_size()
|
||||
if bias is not None:
|
||||
entry_bytes += bias.nelement() * bias.element_size()
|
||||
if _weight_cache_bytes + entry_bytes <= cache_budget:
|
||||
_weight_cache[cache_key] = (weight, bias)
|
||||
_weight_cache_bytes += entry_bytes
|
||||
|
||||
return weight, bias
|
||||
|
||||
GGMLLayer.cast_bias_weight = _bc250_cast_bias_weight
|
||||
|
||||
_gguf_patched = True
|
||||
cache_str = f", weight cache={_WEIGHT_CACHE_MB}MB" if cache_budget > 0 else ""
|
||||
logger.warning(f"[BC-250] GGUF patched: GPU dequant{cache_str}")
|
||||
return True
|
||||
|
||||
# === IMPORT HOOK for deferred GGUF patching ===
|
||||
|
||||
class _GGUFImportWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if 'GGUF' in fullname and ('dequant' in fullname or 'ops' in fullname):
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_gguf():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === TEXT ENCODER CPU PATCH ===
|
||||
_te_patched = False
|
||||
|
||||
def _try_patch_text_encoder_device():
|
||||
global _te_patched
|
||||
if _te_patched:
|
||||
return True
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm is None:
|
||||
return False
|
||||
mm.text_encoder_device = lambda: torch.device("cpu")
|
||||
mm.text_encoder_offload_device = lambda: torch.device("cpu")
|
||||
_te_patched = True
|
||||
logger.warning("[BC-250] Text encoder forced to CPU")
|
||||
return True
|
||||
|
||||
class _ModelMgmtWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.model_management':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_text_encoder_device():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === VAE GPU FP16 WITH CACHING ===
|
||||
# BC-250 = APU with shared memory. GPU VRAM = CPU RAM = same physical pool.
|
||||
# No OOM risk from "using VRAM" — it's all the same 16GB.
|
||||
# GPU fp16 VAE is ~10x faster than CPU float32.
|
||||
_vae_patched = False
|
||||
_vae_cached = False
|
||||
|
||||
def _try_patch_vae_gpu():
|
||||
global _vae_patched
|
||||
if _vae_patched:
|
||||
return True
|
||||
|
||||
sd_mod = sys.modules.get('comfy.sd')
|
||||
if sd_mod is None:
|
||||
return False
|
||||
|
||||
VAE = getattr(sd_mod, 'VAE', None)
|
||||
if VAE is None:
|
||||
return False
|
||||
|
||||
def _ensure_vae_on_gpu_f16(self):
|
||||
"""Move VAE to GPU fp16 once, keep it cached. Shared memory = no OOM risk."""
|
||||
global _vae_cached
|
||||
gpu = torch.device("cuda")
|
||||
try:
|
||||
p = next(self.first_stage_model.parameters())
|
||||
already_ready = _vae_cached and p.device.type == 'cuda' and p.dtype == torch.float16
|
||||
except StopIteration:
|
||||
already_ready = False
|
||||
if not already_ready:
|
||||
logger.warning("[BC-250] Loading VAE to GPU fp16 (shared memory, will stay cached)")
|
||||
# Bypass _bc250_safe_apply (mmap pre-clone) — VAE is safetensors, not GGUF
|
||||
old_apply = torch.nn.Module._apply
|
||||
torch.nn.Module._apply = _original_module_apply
|
||||
try:
|
||||
self.first_stage_model.half().cuda()
|
||||
finally:
|
||||
torch.nn.Module._apply = old_apply
|
||||
self.first_stage_model.eval()
|
||||
_vae_cached = True
|
||||
self.disable_offload = True
|
||||
|
||||
def _bc250_vae_decode(self, samples_in, vae_options={}):
|
||||
t0 = _time.time()
|
||||
self.throw_exception_if_invalid()
|
||||
|
||||
if self.latent_dim == 2 and samples_in.ndim == 5:
|
||||
samples_in = samples_in[:, :, 0]
|
||||
|
||||
# Free GPU memory from UNET before loading VAE
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm:
|
||||
mm.unload_all_models()
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
_ensure_vae_on_gpu_f16(self)
|
||||
|
||||
pixel_samples = None
|
||||
with torch.no_grad():
|
||||
for x in range(samples_in.shape[0]):
|
||||
sample = samples_in[x:x+1].to(torch.float16).cuda()
|
||||
decoded = self.first_stage_model.decode(sample, **vae_options)
|
||||
if decoded.ndim == 5:
|
||||
decoded = decoded[:, :, 0]
|
||||
out = self.process_output(decoded.float().cpu())
|
||||
if pixel_samples is None:
|
||||
pixel_samples = torch.empty(
|
||||
(samples_in.shape[0],) + tuple(out.shape[1:]), device='cpu'
|
||||
)
|
||||
pixel_samples[x:x+1] = out
|
||||
del decoded, sample
|
||||
|
||||
pixel_samples = pixel_samples.movedim(1, -1)
|
||||
elapsed = _time.time() - t0
|
||||
logger.warning(f"[BC-250] VAE decode (GPU fp16): {elapsed:.1f}s")
|
||||
return pixel_samples
|
||||
|
||||
def _bc250_vae_encode(self, pixel_samples):
|
||||
t0 = _time.time()
|
||||
self.throw_exception_if_invalid()
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm:
|
||||
mm.unload_all_models()
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
_ensure_vae_on_gpu_f16(self)
|
||||
with torch.no_grad():
|
||||
pixels_in = self.process_input(pixel_samples).to(torch.float16).cuda()
|
||||
result = self.first_stage_model.encode(pixels_in).float().cpu()
|
||||
logger.warning(f"[BC-250] VAE encode (GPU fp16): {_time.time() - t0:.1f}s")
|
||||
return result
|
||||
|
||||
VAE.decode = _bc250_vae_decode
|
||||
VAE.encode = _bc250_vae_encode
|
||||
_vae_patched = True
|
||||
logger.warning("[BC-250] VAE patched: GPU fp16 (shared memory = zero OOM risk)")
|
||||
return True
|
||||
|
||||
class _SDModuleWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.sd':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_vae_gpu():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === GPU MEMORY CLEANUP HOOK ===
|
||||
_load_patched = False
|
||||
|
||||
def _try_patch_load_models():
|
||||
global _load_patched
|
||||
if _load_patched:
|
||||
return True
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm is None:
|
||||
return False
|
||||
_original_load = getattr(mm, 'load_models_gpu', None)
|
||||
if _original_load is None:
|
||||
return False
|
||||
|
||||
def _bc250_load_models_gpu(models, *args, **kwargs):
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
return _original_load(models, *args, **kwargs)
|
||||
|
||||
mm.load_models_gpu = _bc250_load_models_gpu
|
||||
_load_patched = True
|
||||
logger.warning("[BC-250] GPU memory cleanup hook installed")
|
||||
return True
|
||||
|
||||
# === STARTUP PRELOAD ===
|
||||
_PRELOAD_CLIP = os.environ.get("BC250_PRELOAD_CLIP", "Qwen_3_4b-Q8_0.gguf")
|
||||
_PRELOAD_UNET = os.environ.get("BC250_PRELOAD_UNET", "z_image_turbo-Q5_K_S.gguf")
|
||||
_PRELOAD_VAE = os.environ.get("BC250_PRELOAD_VAE", "ae.safetensors")
|
||||
_PRELOAD_PORT = int(os.environ.get("BC250_PRELOAD_PORT", "8188"))
|
||||
_PRELOAD_ENABLED = os.environ.get("BC250_PRELOAD", "1") == "1"
|
||||
|
||||
def _preload_models():
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
url = f"http://127.0.0.1:{_PRELOAD_PORT}"
|
||||
|
||||
logger.warning("[BC-250] Preload: waiting for server...")
|
||||
for _ in range(240):
|
||||
try:
|
||||
urllib.request.urlopen(f"{url}/api/system_stats", timeout=2)
|
||||
break
|
||||
except (urllib.error.URLError, OSError, ConnectionRefusedError):
|
||||
_time.sleep(0.5)
|
||||
else:
|
||||
logger.warning("[BC-250] Preload: server not ready after 120s, skip")
|
||||
return
|
||||
|
||||
logger.warning("[BC-250] Preload: server ready, submitting warmup...")
|
||||
|
||||
warmup = {
|
||||
"1": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": _PRELOAD_CLIP, "type": "lumina2"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "warmup", "clip": ["1", 0]}},
|
||||
"3": {"class_type": "CLIPTextEncode", "inputs": {"text": "", "clip": ["1", 0]}},
|
||||
"4": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": _PRELOAD_UNET}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 64, "height": 64, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"seed": 1, "steps": 1, "cfg": 1.0, "sampler_name": "euler",
|
||||
"scheduler": "normal", "denoise": 1.0,
|
||||
"model": ["4", 0], "positive": ["2", 0], "negative": ["3", 0], "latent_image": ["5", 0]
|
||||
}},
|
||||
"7": {"class_type": "VAELoader", "inputs": {"vae_name": _PRELOAD_VAE}},
|
||||
"8": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["7", 0]}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "_warmup", "images": ["8", 0]}}
|
||||
}
|
||||
|
||||
payload = json.dumps({"prompt": warmup}).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{url}/api/prompt", data=payload,
|
||||
headers={"Content-Type": "application/json"}, method="POST"
|
||||
)
|
||||
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
data = json.loads(resp.read())
|
||||
prompt_id = data.get("prompt_id", "unknown")
|
||||
logger.warning(f"[BC-250] Preload: warmup queued (id={prompt_id})")
|
||||
|
||||
for _ in range(300):
|
||||
_time.sleep(1)
|
||||
try:
|
||||
hist_resp = urllib.request.urlopen(f"{url}/api/history/{prompt_id}", timeout=5)
|
||||
hist = json.loads(hist_resp.read())
|
||||
if prompt_id in hist:
|
||||
logger.warning("[BC-250] Preload: all models cached. Ready.")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("[BC-250] Preload: warmup timed out (5min)")
|
||||
except Exception as e:
|
||||
logger.warning(f"[BC-250] Preload failed: {e}")
|
||||
|
||||
|
||||
def _start_preload_thread():
|
||||
if not _PRELOAD_ENABLED:
|
||||
logger.warning("[BC-250] Preload: disabled (BC250_PRELOAD=0)")
|
||||
return
|
||||
t = threading.Thread(target=_preload_models, daemon=True, name="BC250-Preload")
|
||||
t.start()
|
||||
logger.warning("[BC-250] Preload: background thread started")
|
||||
|
||||
# === INSTALL ===
|
||||
|
||||
def _prewarm_gpu():
|
||||
try:
|
||||
if not torch.cuda.is_available():
|
||||
return
|
||||
dummy = torch.zeros(1, device='cuda')
|
||||
_ = dummy + 1
|
||||
torch.cuda.synchronize()
|
||||
del dummy
|
||||
torch.cuda.empty_cache()
|
||||
logger.warning("[BC-250] GPU pre-warmed")
|
||||
except Exception as e:
|
||||
logger.warning(f"[BC-250] GPU pre-warm failed: {e}")
|
||||
|
||||
|
||||
def install():
|
||||
# Mmap pre-clone patch
|
||||
torch.nn.Module._apply = _bc250_safe_apply
|
||||
logger.warning("[BC-250] Mmap pre-clone patch installed")
|
||||
|
||||
# Softmax patches
|
||||
torch.nn.functional.softmax = patched_softmax
|
||||
torch.Tensor.softmax = patched_tensor_softmax
|
||||
torch.nn.functional.scaled_dot_product_attention = patched_sdpa
|
||||
logger.warning(f"[BC-250] Softmax patch installed (threshold={SAFE_SOFTMAX_THRESHOLD})")
|
||||
|
||||
# Deferred patches via import hooks
|
||||
sys.meta_path.insert(0, _GGUFImportWatcher())
|
||||
sys.meta_path.insert(0, _ModelMgmtWatcher())
|
||||
sys.meta_path.insert(0, _SDModuleWatcher())
|
||||
|
||||
# Try immediate patches
|
||||
_try_patch_gguf()
|
||||
_try_patch_text_encoder_device()
|
||||
_try_patch_vae_gpu()
|
||||
_try_patch_load_models()
|
||||
|
||||
_prewarm_gpu()
|
||||
_start_preload_thread()
|
||||
|
||||
install()
|
||||
@@ -0,0 +1,646 @@
|
||||
"""
|
||||
BC-250 gfx1010 Comprehensive Monkey-Patch v17
|
||||
|
||||
1. BF16 KILL: gfx1010 has NO native bf16 — force f16 everywhere
|
||||
2. Softmax: manual impl for dim > threshold (VGPR overflow fix)
|
||||
3. SDPA: manual impl for large sequences
|
||||
4. GGUF: GGMLTensor.to() patched — quantized weights ALWAYS stay on CPU
|
||||
CPU dequant → f16 → GPU transfer per layer (gfx1010 GPU can't dequant)
|
||||
5. Mmap: pre-clones non-GGUF tensor data before GPU transfer
|
||||
6. GPU: pre-warms context and caching allocator
|
||||
7. CLIP: forces text encoder to CPU (memory constraint)
|
||||
8. VAE: CPU f32 decode cached in RAM (faster than GPU on this APU)
|
||||
9. Threads: all CPU cores for intra-op parallelism
|
||||
10. Rope: force rope() to CPU — gfx1010 has NO native float64
|
||||
11. Non-blocking disabled: gfx1010 without SDMA hangs on async copies
|
||||
|
||||
v14: rope() → CPU (gfx1010 has no float64 HW)
|
||||
v15: GGUF dequant → CPU (gfx1010 GPU hangs on Q5_K bitwise ops)
|
||||
v16: GGMLTensor.to() patched to keep quantized weights on CPU
|
||||
v17: cast_to → direct .to(non_blocking=False), warmup removed
|
||||
- non_blocking=True hangs on gfx1010 (no SDMA, async HIP copy broken)
|
||||
- replaced empty_like+copy_ with direct .to() for CPU→GPU transfer
|
||||
- device_supports_non_blocking → always False for this device
|
||||
|
||||
BC-250 APU / gfx1010: XNACK disabled, no GPU page fault handling.
|
||||
Place in ComfyUI root and import as first line of main.py.
|
||||
"""
|
||||
import os
|
||||
os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import sys
|
||||
import gc
|
||||
import logging
|
||||
import time as _time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# === THREAD CONFIGURATION ===
|
||||
_NUM_THREADS = int(os.environ.get("BC250_NUM_THREADS", str(os.cpu_count() or 12)))
|
||||
torch.set_num_threads(_NUM_THREADS)
|
||||
logger.warning(f"[BC-250] Torch threads: intra-op={_NUM_THREADS}")
|
||||
|
||||
# Disable torch._dynamo — gfx1010 doesn't benefit, compilation overhead is massive
|
||||
try:
|
||||
torch._dynamo.config.suppress_errors = True
|
||||
logger.warning("[BC-250] torch._dynamo: TORCHDYNAMO_DISABLE=1 + suppress_errors")
|
||||
except Exception:
|
||||
logger.warning("[BC-250] torch._dynamo: TORCHDYNAMO_DISABLE=1 (env only)")
|
||||
|
||||
|
||||
SAFE_SOFTMAX_THRESHOLD = int(os.environ.get("BC250_SOFTMAX_THRESHOLD", "4096"))
|
||||
|
||||
_original_softmax = torch.nn.functional.softmax
|
||||
_original_tensor_softmax = torch.Tensor.softmax
|
||||
_original_sdpa = torch.nn.functional.scaled_dot_product_attention
|
||||
|
||||
# === MMAP PRE-CLONE + BF16 KILL PATCH ===
|
||||
_original_module_apply = torch.nn.Module._apply
|
||||
|
||||
def _is_ggml_tensor(t):
|
||||
"""Check if tensor is a GGMLTensor (has GGUF quantization metadata)."""
|
||||
return hasattr(t, 'tensor_type')
|
||||
|
||||
# === GGML TENSOR CPU LOCK ===
|
||||
# Patched later when GGUF module loads (_try_patch_gguf).
|
||||
# GGMLTensor.to() is monkey-patched so quantized weights NEVER leave CPU.
|
||||
# This prevents both: GPU dequant hang AND GPU→CPU transfer hang.
|
||||
|
||||
def _bc250_safe_apply(self, fn, recurse=True):
|
||||
"""Pre-clone mmap'd CPU tensor data before GPU transfer (XNACK workaround).
|
||||
Converts ALL bf16 → f16 (gfx1010 has no native bf16 — including GGML BF16).
|
||||
BF16 GGML tensors are dequantized to f32→f16, becoming regular tensors.
|
||||
Note: GGMLTensor.clone() returns self, so quantized GGUF weights are unaffected."""
|
||||
for key, param in self._parameters.items():
|
||||
if param is None:
|
||||
continue
|
||||
# Clone CPU data for XNACK workaround (skip GGML: clone() returns self)
|
||||
if param.device.type == 'cpu' and not _is_ggml_tensor(param.data):
|
||||
param.data = param.data.clone()
|
||||
# gfx1010: no native bf16. Convert ALL bf16 → f16 (including GGML BF16)
|
||||
if param.data.dtype == torch.bfloat16:
|
||||
param.data = param.data.float().half()
|
||||
for key, buf in self._buffers.items():
|
||||
if buf is None:
|
||||
continue
|
||||
if buf.device.type == 'cpu' and not _is_ggml_tensor(buf):
|
||||
buf = buf.clone()
|
||||
if buf.dtype == torch.bfloat16:
|
||||
buf = buf.float().half()
|
||||
self._buffers[key] = buf
|
||||
return _original_module_apply(self, fn, recurse)
|
||||
|
||||
# === SOFTMAX PATCH ===
|
||||
|
||||
def _safe_softmax_impl(input, dim=-1):
|
||||
x_max = input.max(dim=dim, keepdim=True).values
|
||||
exp_x = torch.exp(input - x_max)
|
||||
return exp_x / exp_x.sum(dim=dim, keepdim=True)
|
||||
|
||||
def patched_softmax(input, dim=None, _stacklevel=3, dtype=None):
|
||||
if dim is None:
|
||||
dim = -1
|
||||
if dtype is not None:
|
||||
input = input.to(dtype)
|
||||
if input.is_cuda and input.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual F.softmax: shape={list(input.shape)}, dim={dim}")
|
||||
patched_softmax._logged = True
|
||||
return _safe_softmax_impl(input, dim)
|
||||
return _original_softmax(input, dim=dim)
|
||||
|
||||
def patched_tensor_softmax(self, dim=-1, dtype=None):
|
||||
if dtype is not None:
|
||||
self = self.to(dtype)
|
||||
if self.is_cuda and self.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_tensor_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual softmax: shape={list(self.shape)}, dim={dim}")
|
||||
patched_tensor_softmax._logged = True
|
||||
return _safe_softmax_impl(self, dim)
|
||||
return _original_tensor_softmax(self, dim=dim)
|
||||
|
||||
def _safe_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
L, S = query.size(-2), key.size(-2)
|
||||
if scale is None:
|
||||
scale = query.size(-1) ** -0.5
|
||||
attn_weight = torch.matmul(query, key.transpose(-2, -1)) * scale
|
||||
if is_causal:
|
||||
causal_mask = torch.triu(torch.ones(L, S, dtype=torch.bool, device=query.device), diagonal=1)
|
||||
attn_weight = attn_weight.masked_fill(causal_mask, float('-inf'))
|
||||
if attn_mask is not None:
|
||||
if attn_mask.dtype == torch.bool:
|
||||
attn_weight = attn_weight.masked_fill(~attn_mask, float('-inf'))
|
||||
else:
|
||||
attn_weight = attn_weight + attn_mask
|
||||
attn_weight = _safe_softmax_impl(attn_weight, dim=-1)
|
||||
if dropout_p > 0.0:
|
||||
attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p)
|
||||
return torch.matmul(attn_weight, value)
|
||||
|
||||
def patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
S = key.size(-2)
|
||||
if query.is_cuda and S > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_sdpa, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual SDPA: Q={list(query.shape)}, S={S}")
|
||||
patched_sdpa._logged = True
|
||||
return _safe_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
return _original_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
|
||||
# === GGUF CPU DEQUANT + GGMLTensor CPU LOCK ===
|
||||
_gguf_patched = False
|
||||
|
||||
def _try_patch_gguf():
|
||||
"""Patch GGUF for BC-250:
|
||||
1. GGMLTensor.to() → keeps quantized weights on CPU (ignore device arg)
|
||||
2. cast_bias_weight → CPU dequant, then transfer f16 result to GPU
|
||||
|
||||
gfx1010 GPU cannot dequantize Q5_K (bitwise ops hang).
|
||||
And once weights are on GPU, transferring back to CPU also hangs.
|
||||
Only safe path: weights stay CPU → dequant on CPU → f16 to GPU."""
|
||||
global _gguf_patched
|
||||
if _gguf_patched:
|
||||
return True
|
||||
|
||||
ops_mod = None
|
||||
dequant_mod = None
|
||||
for name, mod in sys.modules.items():
|
||||
if mod is None:
|
||||
continue
|
||||
if name.endswith('.ops') and 'GGUF' in name:
|
||||
ops_mod = mod
|
||||
if name.endswith('.dequant') and 'GGUF' in name:
|
||||
dequant_mod = mod
|
||||
|
||||
if ops_mod is None or dequant_mod is None:
|
||||
return False
|
||||
|
||||
GGMLLayer = getattr(ops_mod, 'GGMLLayer', None)
|
||||
GGMLTensor = getattr(ops_mod, 'GGMLTensor', None)
|
||||
is_quantized_fn = getattr(dequant_mod, 'is_quantized', None)
|
||||
if GGMLLayer is None or GGMLTensor is None or is_quantized_fn is None:
|
||||
return False
|
||||
|
||||
torch_compiler_disable = getattr(ops_mod, 'torch_compiler_disable', None)
|
||||
|
||||
# === PATCH 1: GGMLTensor.to() — keep quantized on CPU ===
|
||||
_original_ggml_to = GGMLTensor.to
|
||||
|
||||
def _bc250_ggml_to(self, *args, **kwargs):
|
||||
"""Intercept .to() calls: keep quantized weights on CPU.
|
||||
Only allow dtype changes, block device changes to CUDA.
|
||||
This prevents load_models_gpu from moving GGUF weights to GPU."""
|
||||
# Check if this is a quantized tensor (has tensor_type metadata)
|
||||
if hasattr(self, 'tensor_type') and self.tensor_type is not None:
|
||||
# Parse the .to() call to extract device, dtype, non_blocking
|
||||
# Common patterns from nn.Module._apply:
|
||||
# t.to(device, dtype, non_blocking) — 3 positional
|
||||
# t.to(device) — 1 positional
|
||||
# t.to(dtype) — 1 positional (dtype)
|
||||
# t.to(device=..., dtype=..., non_blocking=...) — kwargs
|
||||
parsed_device = kwargs.get('device', None)
|
||||
parsed_dtype = kwargs.get('dtype', None)
|
||||
parsed_nb = kwargs.get('non_blocking', False)
|
||||
parsed_mem_fmt = kwargs.get('memory_format', None)
|
||||
|
||||
for a in args:
|
||||
if isinstance(a, torch.device):
|
||||
parsed_device = a
|
||||
elif isinstance(a, str):
|
||||
try:
|
||||
parsed_device = torch.device(a)
|
||||
except Exception:
|
||||
pass
|
||||
elif isinstance(a, torch.dtype):
|
||||
parsed_dtype = a
|
||||
elif isinstance(a, bool):
|
||||
parsed_nb = a
|
||||
elif a is None:
|
||||
# dtype=None from Module.to() convert function
|
||||
pass
|
||||
|
||||
# Block CUDA transfer for quantized weights — stay on CPU
|
||||
if parsed_device is not None and parsed_device.type == 'cuda':
|
||||
# Reconstruct call without device, keeping dtype/non_blocking
|
||||
remap_kwargs = {}
|
||||
if parsed_dtype is not None:
|
||||
remap_kwargs['dtype'] = parsed_dtype
|
||||
if parsed_nb:
|
||||
remap_kwargs['non_blocking'] = parsed_nb
|
||||
if parsed_mem_fmt is not None:
|
||||
remap_kwargs['memory_format'] = parsed_mem_fmt
|
||||
if remap_kwargs:
|
||||
return _original_ggml_to(self, **remap_kwargs)
|
||||
return self # No-op: was just a device move
|
||||
|
||||
return _original_ggml_to(self, *args, **kwargs)
|
||||
|
||||
GGMLTensor.to = _bc250_ggml_to
|
||||
logger.warning("[BC-250] GGMLTensor.to() patched: quantized weights locked to CPU")
|
||||
|
||||
# === PATCH 2: cast_bias_weight — CPU dequant + GPU transfer ===
|
||||
_fwd_count = [0, 0.0]
|
||||
|
||||
def _bc250_cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None):
|
||||
"""CPU dequant → GPU transfer. Weights are guaranteed CPU (GGMLTensor.to patched).
|
||||
Dequant on CPU via get_weight(), transfer f16 to GPU with synchronous .to().
|
||||
gfx1010 without SDMA cannot do async copies — non_blocking=False always."""
|
||||
if _fwd_count[0] == 0:
|
||||
_fwd_count[1] = _time.time()
|
||||
_fwd_count[0] += 1
|
||||
|
||||
if input is not None:
|
||||
if dtype is None:
|
||||
dtype = getattr(input, "dtype", torch.float32)
|
||||
if bias_dtype is None:
|
||||
bias_dtype = dtype
|
||||
if device is None:
|
||||
device = input.device
|
||||
|
||||
# gfx1010: never dequant to bf16
|
||||
if dtype == torch.bfloat16:
|
||||
dtype = torch.float16
|
||||
if bias_dtype == torch.bfloat16:
|
||||
bias_dtype = torch.float16
|
||||
|
||||
is_first = _fwd_count[0] <= 3 or 499 <= _fwd_count[0] <= 505
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: device={device}, w_type={type(s.weight).__name__}, w_dev={s.weight.device}, has_tt={hasattr(s.weight, 'tensor_type')}")
|
||||
|
||||
bias = None
|
||||
if s.bias is not None:
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: get_weight(bias)...")
|
||||
bias = s.get_weight(s.bias, bias_dtype)
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: bias got, type={type(bias).__name__}, dev={bias.device}, dt={bias.dtype}")
|
||||
if type(bias) is not torch.Tensor:
|
||||
bias = bias.as_subclass(torch.Tensor)
|
||||
if bias.device != device or bias.dtype != bias_dtype:
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: bias.to({device})...")
|
||||
bias = bias.to(device=device, dtype=bias_dtype, non_blocking=False)
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: bias transferred")
|
||||
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: get_weight(weight)...")
|
||||
weight = s.get_weight(s.weight, dtype)
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: weight got, type={type(weight).__name__}, shape={list(weight.shape)}, dev={weight.device}, dt={weight.dtype}")
|
||||
if type(weight) is not torch.Tensor:
|
||||
weight = weight.as_subclass(torch.Tensor)
|
||||
if weight.device != device or weight.dtype != dtype:
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: weight.to({device}, {dtype})...")
|
||||
weight = weight.to(device=device, dtype=dtype, non_blocking=False)
|
||||
if is_first:
|
||||
logger.warning(f"[BC-250] CBW#{_fwd_count[0]}: weight transferred")
|
||||
|
||||
if _fwd_count[0] % 100 == 0:
|
||||
elapsed = _time.time() - _fwd_count[1]
|
||||
logger.warning(f"[BC-250] Layer {_fwd_count[0]}, elapsed {elapsed:.1f}s")
|
||||
|
||||
# Log all CUDA transfers to find the exact hang point
|
||||
if device is not None and hasattr(device, 'type') and device.type == 'cuda':
|
||||
logger.warning(f"[BC-250] CUDA#{_fwd_count[0]}: {list(weight.shape)} {weight.dtype} done")
|
||||
|
||||
return weight, bias
|
||||
|
||||
if torch_compiler_disable is not None:
|
||||
_bc250_cast_bias_weight = torch_compiler_disable()(_bc250_cast_bias_weight)
|
||||
|
||||
GGMLLayer.cast_bias_weight = _bc250_cast_bias_weight
|
||||
_gguf_patched = True
|
||||
logger.warning("[BC-250] GGUF patched: CPU dequant + GGMLTensor CPU-locked")
|
||||
return True
|
||||
|
||||
class _GGUFImportWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if 'GGUF' in fullname and ('dequant' in fullname or 'ops' in fullname):
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_gguf():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === MODEL MANAGEMENT PATCHES (text encoder CPU + bf16 kill + load hook) ===
|
||||
_mm_patched = False
|
||||
|
||||
def _try_patch_model_management():
|
||||
"""Patches comfy.model_management:
|
||||
- Text encoder → CPU
|
||||
- should_use_bf16 → always False
|
||||
- unet_dtype → never returns bf16
|
||||
- load_models_gpu → cleanup + post-load bf16→f16
|
||||
"""
|
||||
global _mm_patched
|
||||
if _mm_patched:
|
||||
return True
|
||||
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm is None:
|
||||
return False
|
||||
|
||||
# Text encoder on CPU
|
||||
mm.text_encoder_device = lambda: torch.device("cpu")
|
||||
mm.text_encoder_offload_device = lambda: torch.device("cpu")
|
||||
logger.warning("[BC-250] Text encoder forced to CPU")
|
||||
|
||||
# Kill bf16 globally — gfx1010 has no native bf16
|
||||
mm.should_use_bf16 = lambda *a, **kw: False
|
||||
logger.warning("[BC-250] should_use_bf16 → always False (gfx1010)")
|
||||
|
||||
# Force non_blocking=False — gfx1010 without SDMA hangs on async HIP copies
|
||||
mm.device_supports_non_blocking = lambda *a, **kw: False
|
||||
logger.warning("[BC-250] device_supports_non_blocking → always False (no SDMA)")
|
||||
|
||||
_original_unet_dtype = mm.unet_dtype
|
||||
def _bc250_unet_dtype(*args, **kwargs):
|
||||
return torch.float16 # gfx1010: always f16 (2× faster than f32, no bf16 HW)
|
||||
mm.unet_dtype = _bc250_unet_dtype
|
||||
logger.warning("[BC-250] unet_dtype patched: always f16")
|
||||
|
||||
# Ensure fp16 is recognized as available
|
||||
mm.should_use_fp16 = lambda *a, **kw: True
|
||||
logger.warning("[BC-250] should_use_fp16 → always True")
|
||||
|
||||
# Load hook: cleanup + post-load bf16 → f16 conversion
|
||||
_original_load = getattr(mm, 'load_models_gpu', None)
|
||||
if _original_load is not None:
|
||||
def _bc250_load_models_gpu(models, *args, **kwargs):
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
result = _original_load(models, *args, **kwargs)
|
||||
# Post-load: convert ALL remaining bf16 params/buffers to f16
|
||||
for m in models:
|
||||
real_model = getattr(m, 'model', None)
|
||||
if real_model is None:
|
||||
continue
|
||||
converted = 0
|
||||
for p in real_model.parameters():
|
||||
if p.dtype == torch.bfloat16:
|
||||
p.data = p.data.float().half()
|
||||
converted += 1
|
||||
for name, buf in real_model.named_buffers():
|
||||
if buf is not None and buf.dtype == torch.bfloat16:
|
||||
parts = name.split('.')
|
||||
obj = real_model
|
||||
for part in parts[:-1]:
|
||||
obj = getattr(obj, part)
|
||||
setattr(obj, parts[-1], buf.float().half())
|
||||
converted += 1
|
||||
if converted > 0:
|
||||
logger.warning(f"[BC-250] Post-load: converted {converted} bf16→f16 params/buffers")
|
||||
return result
|
||||
mm.load_models_gpu = _bc250_load_models_gpu
|
||||
logger.warning("[BC-250] GPU load hook installed (cleanup + bf16 kill)")
|
||||
|
||||
_mm_patched = True
|
||||
return True
|
||||
|
||||
class _ModelMgmtWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.model_management':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_model_management():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === VAE CPU CACHED ===
|
||||
_vae_patched = False
|
||||
_vae_cached = False
|
||||
|
||||
def _try_patch_vae_gpu():
|
||||
global _vae_patched
|
||||
if _vae_patched:
|
||||
return True
|
||||
|
||||
sd_mod = sys.modules.get('comfy.sd')
|
||||
if sd_mod is None:
|
||||
return False
|
||||
|
||||
VAE = getattr(sd_mod, 'VAE', None)
|
||||
if VAE is None:
|
||||
return False
|
||||
|
||||
def _ensure_vae_cached_cpu(self):
|
||||
"""Keep VAE on CPU in RAM, eval mode. No GPU transfer needed.
|
||||
On BC-250 APU: GPU VAE decode is slower than CPU (24 CUs, no SDMA).
|
||||
CPU has 12 Zen2 threads and direct RAM access — faster for VAE convolutions."""
|
||||
global _vae_cached
|
||||
if not _vae_cached:
|
||||
t0 = _time.time()
|
||||
self.first_stage_model.to(device='cpu', dtype=torch.float32)
|
||||
self.first_stage_model.eval()
|
||||
_vae_cached = True
|
||||
logger.warning(f"[BC-250] VAE cached on CPU (f32) in {_time.time()-t0:.1f}s")
|
||||
self.disable_offload = True
|
||||
|
||||
def _bc250_vae_decode(self, samples_in, vae_options={}):
|
||||
t0 = _time.time()
|
||||
self.throw_exception_if_invalid()
|
||||
|
||||
if self.latent_dim == 2 and samples_in.ndim == 5:
|
||||
samples_in = samples_in[:, :, 0]
|
||||
|
||||
_ensure_vae_cached_cpu(self)
|
||||
|
||||
pixel_samples = None
|
||||
with torch.no_grad():
|
||||
for x in range(samples_in.shape[0]):
|
||||
sample = samples_in[x:x+1].float().cpu()
|
||||
decoded = self.first_stage_model.decode(sample, **vae_options)
|
||||
if decoded.ndim == 5:
|
||||
decoded = decoded[:, :, 0]
|
||||
out = self.process_output(decoded.float())
|
||||
if pixel_samples is None:
|
||||
pixel_samples = torch.empty(
|
||||
(samples_in.shape[0],) + tuple(out.shape[1:]), device='cpu'
|
||||
)
|
||||
pixel_samples[x:x+1] = out
|
||||
del decoded, sample
|
||||
|
||||
pixel_samples = pixel_samples.movedim(1, -1)
|
||||
elapsed = _time.time() - t0
|
||||
logger.warning(f"[BC-250] VAE decode (CPU f32): {elapsed:.1f}s")
|
||||
return pixel_samples
|
||||
|
||||
def _bc250_vae_encode(self, pixel_samples):
|
||||
t0 = _time.time()
|
||||
self.throw_exception_if_invalid()
|
||||
_ensure_vae_cached_cpu(self)
|
||||
with torch.no_grad():
|
||||
pixels_in = self.process_input(pixel_samples).float().cpu()
|
||||
result = self.first_stage_model.encode(pixels_in).float()
|
||||
logger.warning(f"[BC-250] VAE encode (CPU f32): {_time.time() - t0:.1f}s")
|
||||
return result
|
||||
|
||||
VAE.decode = _bc250_vae_decode
|
||||
VAE.encode = _bc250_vae_encode
|
||||
_vae_patched = True
|
||||
logger.warning("[BC-250] VAE patched: CPU f32 decode/encode (cached in RAM)")
|
||||
return True
|
||||
|
||||
class _SDModuleWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.sd':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_vae_gpu():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === ROPE CPU PATCH (gfx1010 has no float64 hardware) ===
|
||||
_rope_patched = False
|
||||
|
||||
def _try_patch_rope():
|
||||
"""Patch rope() in flux/math.py to always compute on CPU.
|
||||
gfx1010 has no native float64 — GPU float64 ops are software-emulated and hang."""
|
||||
global _rope_patched
|
||||
if _rope_patched:
|
||||
return True
|
||||
|
||||
flux_math = sys.modules.get('comfy.ldm.flux.math')
|
||||
if flux_math is None:
|
||||
return False
|
||||
|
||||
_original_rope = getattr(flux_math, 'rope', None)
|
||||
if _original_rope is None:
|
||||
return False
|
||||
|
||||
def _bc250_rope(pos, dim, theta):
|
||||
"""Compute rope on CPU (float64 not supported on gfx1010), then move result to original device."""
|
||||
assert dim % 2 == 0
|
||||
target_device = pos.device
|
||||
device = torch.device("cpu")
|
||||
scale = torch.linspace(0, (dim - 2) / dim, steps=dim // 2, dtype=torch.float64, device=device)
|
||||
omega = 1.0 / (theta ** scale)
|
||||
out = torch.einsum("...n,d->...nd", pos.to(dtype=torch.float32, device=device), omega)
|
||||
from einops import rearrange
|
||||
out = torch.stack([torch.cos(out), -torch.sin(out), torch.sin(out), torch.cos(out)], dim=-1)
|
||||
out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2)
|
||||
return out.to(dtype=torch.float32, device=target_device)
|
||||
|
||||
flux_math.rope = _bc250_rope
|
||||
_rope_patched = True
|
||||
logger.warning("[BC-250] rope() patched: CPU computation (no float64 on gfx1010)")
|
||||
return True
|
||||
|
||||
class _FluxMathWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.ldm.flux.math':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_rope():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === WARMUP REMOVED (v17) ===
|
||||
# First prompt may be slower, subsequent prompts benefit from HIP kernel caches.
|
||||
|
||||
# === INSTALL ===
|
||||
|
||||
def _prewarm_gpu():
|
||||
try:
|
||||
if not torch.cuda.is_available():
|
||||
return
|
||||
# Warm GPU context + allocator
|
||||
dummy = torch.zeros(1, device='cuda')
|
||||
_ = dummy + 1
|
||||
torch.cuda.synchronize()
|
||||
# Warm CPU→GPU copy kernel (COMGR JIT on first transfer)
|
||||
cpu_t = torch.randn(256, 256, dtype=torch.float16)
|
||||
gpu_t = cpu_t.to('cuda')
|
||||
_ = torch.matmul(gpu_t, gpu_t.T)
|
||||
torch.cuda.synchronize()
|
||||
del dummy, cpu_t, gpu_t, _
|
||||
torch.cuda.empty_cache()
|
||||
logger.warning("[BC-250] GPU pre-warmed (context + copy + matmul)")
|
||||
except Exception as e:
|
||||
logger.warning(f"[BC-250] GPU pre-warm failed: {e}")
|
||||
|
||||
|
||||
def install():
|
||||
# Mmap pre-clone + bf16 kill patch
|
||||
torch.nn.Module._apply = _bc250_safe_apply
|
||||
logger.warning("[BC-250] Mmap pre-clone + bf16→f16 patch installed")
|
||||
|
||||
# Softmax patches
|
||||
torch.nn.functional.softmax = patched_softmax
|
||||
torch.Tensor.softmax = patched_tensor_softmax
|
||||
torch.nn.functional.scaled_dot_product_attention = patched_sdpa
|
||||
logger.warning(f"[BC-250] Softmax patch installed (threshold={SAFE_SOFTMAX_THRESHOLD})")
|
||||
|
||||
# Deferred patches via import hooks
|
||||
sys.meta_path.insert(0, _GGUFImportWatcher())
|
||||
sys.meta_path.insert(0, _ModelMgmtWatcher())
|
||||
sys.meta_path.insert(0, _SDModuleWatcher())
|
||||
sys.meta_path.insert(0, _FluxMathWatcher())
|
||||
|
||||
# Try immediate patches
|
||||
_try_patch_gguf()
|
||||
_try_patch_model_management()
|
||||
_try_patch_vae_gpu()
|
||||
_try_patch_rope()
|
||||
|
||||
_prewarm_gpu()
|
||||
logger.warning("[BC-250] v17 ready — no warmup, first prompt may be slow")
|
||||
|
||||
install()
|
||||
@@ -0,0 +1,426 @@
|
||||
"""
|
||||
BC-250 gfx1010 Comprehensive Monkey-Patch v6
|
||||
1. Replaces torch.softmax with manual implementation (VGPR overflow fix)
|
||||
2. Replaces SDPA with manual implementation
|
||||
3. Patches GGUF cast_bias_weight to dequant on CPU (avoids GPU page-fault hangs)
|
||||
4. Pre-clones mmap'd tensor data before GPU transfer (XNACK workaround)
|
||||
5. Pre-warms GPU context and caching allocator
|
||||
6. Forces text encoder to CPU (memory constraint)
|
||||
7. Forces VAE decode on CPU (prevents GPU page-fault hang on safetensors mmap)
|
||||
|
||||
v6 changes: Removed NO_VRAM (made sampling impossibly slow).
|
||||
Instead, VAE is forced to decode on CPU. UNet uses normal lowvram path.
|
||||
Previous LOWVRAM run: 4/4 steps in 21s (5.5s/step). NO_VRAM: stuck at 0/4 for 10+ min.
|
||||
|
||||
BC-250 APU / gfx1010: XNACK disabled, no GPU page fault handling.
|
||||
GPU copy shader hangs on non-resident pages (mmap'd or swapped).
|
||||
Place in ComfyUI root and import as first line of main.py.
|
||||
"""
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import os
|
||||
import sys
|
||||
import gc
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SAFE_SOFTMAX_THRESHOLD = int(os.environ.get("BC250_SOFTMAX_THRESHOLD", "512"))
|
||||
|
||||
_original_softmax = torch.nn.functional.softmax
|
||||
_original_tensor_softmax = torch.Tensor.softmax
|
||||
_original_sdpa = torch.nn.functional.scaled_dot_product_attention
|
||||
|
||||
# === MMAP PRE-CLONE PATCH ===
|
||||
_original_module_apply = torch.nn.Module._apply
|
||||
|
||||
def _bc250_safe_apply(self, fn, recurse=True):
|
||||
"""Pre-clone mmap'd CPU tensor data before GPU transfer to avoid XNACK hangs."""
|
||||
for key, param in self._parameters.items():
|
||||
if param is not None and param.device.type == 'cpu':
|
||||
param.data = param.data.clone()
|
||||
for key, buf in self._buffers.items():
|
||||
if buf is not None and buf.device.type == 'cpu':
|
||||
self._buffers[key] = buf.clone()
|
||||
return _original_module_apply(self, fn, recurse)
|
||||
|
||||
# === SOFTMAX PATCH ===
|
||||
|
||||
def _safe_softmax_impl(input, dim=-1):
|
||||
x_max = input.max(dim=dim, keepdim=True).values
|
||||
exp_x = torch.exp(input - x_max)
|
||||
return exp_x / exp_x.sum(dim=dim, keepdim=True)
|
||||
|
||||
def patched_softmax(input, dim=None, _stacklevel=3, dtype=None):
|
||||
if dim is None:
|
||||
dim = -1
|
||||
if dtype is not None:
|
||||
input = input.to(dtype)
|
||||
if input.is_cuda and input.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
return _safe_softmax_impl(input, dim)
|
||||
return _original_softmax(input, dim=dim)
|
||||
|
||||
def patched_tensor_softmax(self, dim=-1, dtype=None):
|
||||
if dtype is not None:
|
||||
self = self.to(dtype)
|
||||
if self.is_cuda and self.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
return _safe_softmax_impl(self, dim)
|
||||
return _original_tensor_softmax(self, dim=dim)
|
||||
|
||||
def _safe_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
L, S = query.size(-2), key.size(-2)
|
||||
if scale is None:
|
||||
scale = query.size(-1) ** -0.5
|
||||
attn_weight = torch.matmul(query, key.transpose(-2, -1)) * scale
|
||||
if is_causal:
|
||||
causal_mask = torch.triu(torch.ones(L, S, dtype=torch.bool, device=query.device), diagonal=1)
|
||||
attn_weight = attn_weight.masked_fill(causal_mask, float('-inf'))
|
||||
if attn_mask is not None:
|
||||
if attn_mask.dtype == torch.bool:
|
||||
attn_weight = attn_weight.masked_fill(~attn_mask, float('-inf'))
|
||||
else:
|
||||
attn_weight = attn_weight + attn_mask
|
||||
attn_weight = _safe_softmax_impl(attn_weight, dim=-1)
|
||||
if dropout_p > 0.0:
|
||||
attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p)
|
||||
return torch.matmul(attn_weight, value)
|
||||
|
||||
def patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
S = key.size(-2)
|
||||
if query.is_cuda and S > SAFE_SOFTMAX_THRESHOLD:
|
||||
return _safe_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
return _original_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
|
||||
# === GGUF CPU-DEQUANT PATCH (cast_bias_weight override) ===
|
||||
_gguf_patched = False
|
||||
|
||||
def _try_patch_gguf():
|
||||
"""Patch GGMLLayer.cast_bias_weight to dequant on CPU, send floats to GPU."""
|
||||
global _gguf_patched
|
||||
if _gguf_patched:
|
||||
return True
|
||||
|
||||
ops_mod = None
|
||||
dequant_mod = None
|
||||
for name, mod in sys.modules.items():
|
||||
if mod is None:
|
||||
continue
|
||||
if name.endswith('.ops') and 'GGUF' in name:
|
||||
ops_mod = mod
|
||||
if name.endswith('.dequant') and 'GGUF' in name:
|
||||
dequant_mod = mod
|
||||
|
||||
if ops_mod is None or dequant_mod is None:
|
||||
return False
|
||||
|
||||
GGMLLayer = getattr(ops_mod, 'GGMLLayer', None)
|
||||
is_quantized_fn = getattr(dequant_mod, 'is_quantized', None)
|
||||
if GGMLLayer is None or is_quantized_fn is None:
|
||||
return False
|
||||
|
||||
def _bc250_cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None):
|
||||
"""Dequant on CPU, only send float results to GPU."""
|
||||
import comfy.model_management
|
||||
import comfy.ops
|
||||
|
||||
if input is not None:
|
||||
if dtype is None:
|
||||
dtype = getattr(input, "dtype", torch.float32)
|
||||
if bias_dtype is None:
|
||||
bias_dtype = dtype
|
||||
if device is None:
|
||||
device = input.device
|
||||
|
||||
non_blocking = comfy.model_management.device_supports_non_blocking(device)
|
||||
|
||||
bias = None
|
||||
if s.bias is not None:
|
||||
if is_quantized_fn(s.bias):
|
||||
bias = s.get_weight(s.bias, bias_dtype)
|
||||
else:
|
||||
bias = s.get_weight(s.bias.to(device), bias_dtype)
|
||||
bias = comfy.ops.cast_to(bias, bias_dtype, device, non_blocking=non_blocking, copy=False)
|
||||
|
||||
if is_quantized_fn(s.weight):
|
||||
weight = s.get_weight(s.weight, dtype)
|
||||
else:
|
||||
weight = s.get_weight(s.weight.to(device), dtype)
|
||||
weight = comfy.ops.cast_to(weight, dtype, device, non_blocking=non_blocking, copy=False)
|
||||
return weight, bias
|
||||
|
||||
GGMLLayer.cast_bias_weight = _bc250_cast_bias_weight
|
||||
|
||||
_gguf_patched = True
|
||||
logger.warning("[BC-250] GGUF cast_bias_weight patched (CPU dequant, float-only GPU transfer)")
|
||||
return True
|
||||
|
||||
# === IMPORT HOOK for deferred GGUF patching ===
|
||||
|
||||
class _GGUFImportWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if 'GGUF' in fullname and ('dequant' in fullname or 'ops' in fullname):
|
||||
return self
|
||||
return None
|
||||
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
|
||||
if _try_patch_gguf():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === TEXT ENCODER CPU PATCH ===
|
||||
_te_patched = False
|
||||
|
||||
def _try_patch_text_encoder_device():
|
||||
global _te_patched
|
||||
if _te_patched:
|
||||
return True
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm is None:
|
||||
return False
|
||||
mm.text_encoder_device = lambda: torch.device("cpu")
|
||||
mm.text_encoder_offload_device = lambda: torch.device("cpu")
|
||||
|
||||
_te_patched = True
|
||||
logger.warning("[BC-250] Text encoder forced to CPU (memory constraint)")
|
||||
return True
|
||||
|
||||
# === VAE CPU-ONLY PATCH ===
|
||||
# Force VAE to decode on CPU. VAE is only 320MB — fast enough on CPU for small images.
|
||||
# Avoids GPU page-fault hangs from safetensors mmap'd weights on BC-250 (XNACK disabled).
|
||||
|
||||
_vae_patched = False
|
||||
|
||||
def _try_patch_vae_cpu():
|
||||
"""Patch comfy.sd.VAE to decode and encode on CPU only."""
|
||||
global _vae_patched
|
||||
if _vae_patched:
|
||||
return True
|
||||
|
||||
sd_mod = sys.modules.get('comfy.sd')
|
||||
if sd_mod is None:
|
||||
return False
|
||||
|
||||
VAE = getattr(sd_mod, 'VAE', None)
|
||||
if VAE is None:
|
||||
return False
|
||||
|
||||
_original_vae_decode = VAE.decode
|
||||
_original_vae_encode = getattr(VAE, 'encode', None)
|
||||
|
||||
def _bc250_vae_decode(self, samples_in, vae_options={}):
|
||||
"""Force VAE decode on CPU — bypass load_models_gpu entirely.
|
||||
|
||||
Root cause: load_models_gpu tries to unload UNet (5032MB in GPU managed memory)
|
||||
before loading VAE. Unloading reads GPU pages that may be swapped → XNACK hang.
|
||||
Solution: skip load_models_gpu, run VAE inference directly on CPU.
|
||||
"""
|
||||
import comfy.model_management as mm
|
||||
|
||||
logger.warning("[BC-250] VAE decode: CPU-only bypass (skipping load_models_gpu)")
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
# Temporarily no-op load_models_gpu to prevent UNet unload hang
|
||||
_orig_lmg = mm.load_models_gpu
|
||||
mm.load_models_gpu = lambda *a, **kw: None
|
||||
|
||||
# Save and override device to CPU
|
||||
orig_device = getattr(self, 'device', None)
|
||||
orig_output_device = getattr(self, 'output_device', None)
|
||||
self.device = torch.device("cpu")
|
||||
self.output_device = torch.device("cpu")
|
||||
|
||||
try:
|
||||
# Ensure VAE model weights are on CPU
|
||||
if hasattr(self, 'first_stage_model'):
|
||||
self.first_stage_model.to(torch.device("cpu"))
|
||||
self.first_stage_model.eval()
|
||||
|
||||
# Run the original decode (which now skips load_models_gpu)
|
||||
result = _original_vae_decode(self, samples_in, vae_options)
|
||||
if isinstance(result, torch.Tensor):
|
||||
result = result.to(device=torch.device("cpu"))
|
||||
return result
|
||||
finally:
|
||||
# Restore everything
|
||||
mm.load_models_gpu = _orig_lmg
|
||||
if orig_device is not None:
|
||||
self.device = orig_device
|
||||
if orig_output_device is not None:
|
||||
self.output_device = orig_output_device
|
||||
|
||||
VAE.decode = _bc250_vae_decode
|
||||
|
||||
if _original_vae_encode is not None:
|
||||
def _bc250_vae_encode(self, pixel_samples):
|
||||
"""Force VAE encode on CPU — same bypass as decode."""
|
||||
import comfy.model_management as mm
|
||||
logger.warning("[BC-250] VAE encode: CPU-only bypass")
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
_orig_lmg = mm.load_models_gpu
|
||||
mm.load_models_gpu = lambda *a, **kw: None
|
||||
orig_device = getattr(self, 'device', None)
|
||||
orig_output_device = getattr(self, 'output_device', None)
|
||||
self.device = torch.device("cpu")
|
||||
self.output_device = torch.device("cpu")
|
||||
try:
|
||||
if hasattr(self, 'first_stage_model'):
|
||||
self.first_stage_model.to(torch.device("cpu"))
|
||||
self.first_stage_model.eval()
|
||||
pixel_samples = pixel_samples.to(device=torch.device("cpu"), dtype=torch.float32)
|
||||
result = _original_vae_encode(self, pixel_samples)
|
||||
if isinstance(result, torch.Tensor):
|
||||
result = result.to(device=torch.device("cpu"))
|
||||
return result
|
||||
finally:
|
||||
mm.load_models_gpu = _orig_lmg
|
||||
if orig_device is not None:
|
||||
self.device = orig_device
|
||||
if orig_output_device is not None:
|
||||
self.output_device = orig_output_device
|
||||
|
||||
VAE.encode = _bc250_vae_encode
|
||||
|
||||
_vae_patched = True
|
||||
logger.warning("[BC-250] VAE forced to CPU decode/encode (prevents mmap GPU hangs)")
|
||||
return True
|
||||
|
||||
class _SDModuleWatcher:
|
||||
"""Patches comfy.sd.VAE after it's imported."""
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.sd':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_vae_cpu():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
class _ModelMgmtWatcher:
|
||||
def __init__(self):
|
||||
self.done = False
|
||||
def find_module(self, fullname, path=None):
|
||||
if self.done:
|
||||
return None
|
||||
if fullname == 'comfy.model_management':
|
||||
return self
|
||||
return None
|
||||
def load_module(self, fullname):
|
||||
if self in sys.meta_path:
|
||||
sys.meta_path.remove(self)
|
||||
try:
|
||||
import importlib
|
||||
mod = importlib.import_module(fullname)
|
||||
finally:
|
||||
if self not in sys.meta_path:
|
||||
sys.meta_path.insert(0, self)
|
||||
if _try_patch_text_encoder_device():
|
||||
self.done = True
|
||||
return mod
|
||||
|
||||
# === GPU MEMORY CLEANUP HOOK ===
|
||||
# Patch model_management.load_models_gpu to clean up before loading
|
||||
|
||||
_load_patched = False
|
||||
|
||||
def _try_patch_load_models():
|
||||
"""Add GPU memory cleanup before model loading."""
|
||||
global _load_patched
|
||||
if _load_patched:
|
||||
return True
|
||||
|
||||
mm = sys.modules.get('comfy.model_management')
|
||||
if mm is None:
|
||||
return False
|
||||
|
||||
_original_load = getattr(mm, 'load_models_gpu', None)
|
||||
if _original_load is None:
|
||||
return False
|
||||
|
||||
def _bc250_load_models_gpu(models, *args, **kwargs):
|
||||
"""Clean GPU cache before loading models to prevent memory pressure hangs."""
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
return _original_load(models, *args, **kwargs)
|
||||
|
||||
mm.load_models_gpu = _bc250_load_models_gpu
|
||||
_load_patched = True
|
||||
logger.warning("[BC-250] GPU memory cleanup hook installed (load_models_gpu)")
|
||||
return True
|
||||
|
||||
# === INSTALL ===
|
||||
|
||||
def _prewarm_gpu():
|
||||
try:
|
||||
if not torch.cuda.is_available():
|
||||
return
|
||||
dummy = torch.zeros(1, device='cuda')
|
||||
_ = dummy + 1
|
||||
torch.cuda.synchronize()
|
||||
del dummy
|
||||
torch.cuda.empty_cache()
|
||||
logger.warning("[BC-250] GPU pre-warmed (context + allocator ready)")
|
||||
except Exception as e:
|
||||
logger.warning(f"[BC-250] GPU pre-warm failed: {e}")
|
||||
|
||||
|
||||
def install():
|
||||
# Mmap pre-clone patch
|
||||
torch.nn.Module._apply = _bc250_safe_apply
|
||||
logger.warning("[BC-250] Mmap pre-clone patch installed (XNACK workaround)")
|
||||
|
||||
# Softmax patches
|
||||
torch.nn.functional.softmax = patched_softmax
|
||||
torch.Tensor.softmax = patched_tensor_softmax
|
||||
torch.nn.functional.scaled_dot_product_attention = patched_sdpa
|
||||
logger.warning(f"[BC-250] Softmax monkey-patch installed (threshold={SAFE_SOFTMAX_THRESHOLD})")
|
||||
|
||||
# GGUF deferred cast_bias_weight patch
|
||||
sys.meta_path.insert(0, _GGUFImportWatcher())
|
||||
logger.warning("[BC-250] GGUF CPU-dequant hook registered (cast_bias_weight)")
|
||||
|
||||
# Text encoder CPU patch
|
||||
sys.meta_path.insert(0, _ModelMgmtWatcher())
|
||||
|
||||
# VAE CPU-only patch
|
||||
sys.meta_path.insert(0, _SDModuleWatcher())
|
||||
|
||||
# Try immediate patches if modules already loaded
|
||||
_try_patch_gguf()
|
||||
_try_patch_text_encoder_device()
|
||||
_try_patch_vae_cpu()
|
||||
_try_patch_load_models()
|
||||
|
||||
# Pre-warm GPU
|
||||
_prewarm_gpu()
|
||||
|
||||
install()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 349 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Quick 512x512 benchmark: 4 steps, CFG 1.0
|
||||
COMFY="http://localhost:8188"
|
||||
|
||||
WORKFLOW='{
|
||||
"1": {"class_type": "CLIPLoaderGGUF", "inputs": {"clip_name": "Qwen_3_4b-Q8_0.gguf", "type": "lumina2"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"text": "A hyper-realistic spider eating a fly, macro shot, 8K", "clip": ["1", 0]}},
|
||||
"3": {"class_type": "CLIPTextEncode", "inputs": {"text": "blurry, low quality", "clip": ["1", 0]}},
|
||||
"4": {"class_type": "UnetLoaderGGUF", "inputs": {"unet_name": "z_image_turbo-Q5_K_S.gguf"}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "KSampler", "inputs": {
|
||||
"seed": 42, "steps": 4, "cfg": 1.0, "sampler_name": "euler",
|
||||
"scheduler": "normal", "denoise": 1.0,
|
||||
"model": ["4", 0], "positive": ["2", 0], "negative": ["3", 0], "latent_image": ["5", 0]
|
||||
}},
|
||||
"7": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
|
||||
"8": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["7", 0]}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "bench_512_4step", "images": ["8", 0]}}
|
||||
}'
|
||||
|
||||
echo "Submitting 512x512 @ 4 steps, CFG 1.0..."
|
||||
RESP=$(curl -s -X POST "$COMFY/api/prompt" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"prompt\": $WORKFLOW}")
|
||||
echo "$RESP" | python3 -c "import sys,json; print('Prompt ID:', json.load(sys.stdin).get('prompt_id','FAIL'))" 2>/dev/null
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Submit 512x512 spider test via API
|
||||
PORT=8188
|
||||
URL="http://127.0.0.1:$PORT"
|
||||
STEPS=${1:-4}
|
||||
echo "Submitting spider bench: 512x512, $STEPS steps, CFG 1.0"
|
||||
|
||||
curl -s -X POST "$URL/api/prompt" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"prompt\": {
|
||||
\"1\": {\"class_type\": \"CLIPLoaderGGUF\", \"inputs\": {\"clip_name\": \"Qwen_3_4b-Q8_0.gguf\", \"type\": \"lumina2\"}},
|
||||
\"2\": {\"class_type\": \"CLIPTextEncode\", \"inputs\": {\"text\": \"a giant spider made of chrome and neon lights, cyberpunk cityscape background, rain reflections, ultra detailed, 8k\", \"clip\": [\"1\", 0]}},
|
||||
\"3\": {\"class_type\": \"CLIPTextEncode\", \"inputs\": {\"text\": \"\", \"clip\": [\"1\", 0]}},
|
||||
\"4\": {\"class_type\": \"UnetLoaderGGUF\", \"inputs\": {\"unet_name\": \"z_image_turbo-Q5_K_S.gguf\"}},
|
||||
\"5\": {\"class_type\": \"EmptyLatentImage\", \"inputs\": {\"width\": 512, \"height\": 512, \"batch_size\": 1}},
|
||||
\"6\": {\"class_type\": \"KSampler\", \"inputs\": {
|
||||
\"seed\": 42, \"steps\": $STEPS, \"cfg\": 1.0, \"sampler_name\": \"euler\",
|
||||
\"scheduler\": \"normal\", \"denoise\": 1.0,
|
||||
\"model\": [\"4\", 0], \"positive\": [\"2\", 0], \"negative\": [\"3\", 0], \"latent_image\": [\"5\", 0]
|
||||
}},
|
||||
\"7\": {\"class_type\": \"VAELoader\", \"inputs\": {\"vae_name\": \"ae.safetensors\"}},
|
||||
\"8\": {\"class_type\": \"VAEDecode\", \"inputs\": {\"samples\": [\"6\", 0], \"vae\": [\"7\", 0]}},
|
||||
\"9\": {\"class_type\": \"SaveImage\", \"inputs\": {\"filename_prefix\": \"bench_spider\", \"images\": [\"8\", 0]}}
|
||||
}}"
|
||||
echo ""
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Check comgr cache growth
|
||||
echo "=== comgr cache ==="
|
||||
ls -la ~/.cache/comgr/ 2>/dev/null | tail -5
|
||||
echo "Files: $(ls ~/.cache/comgr/ 2>/dev/null | wc -l)"
|
||||
echo "Size: $(du -sh ~/.cache/comgr/ 2>/dev/null | cut -f1)"
|
||||
|
||||
echo ""
|
||||
echo "=== GPU memory ==="
|
||||
cat /sys/class/drm/card1/device/mem_info_vram_used 2>/dev/null || echo "No vram info"
|
||||
cat /sys/class/drm/card1/device/mem_info_gtt_used 2>/dev/null || echo "No gtt info"
|
||||
|
||||
echo ""
|
||||
echo "=== Process growth ==="
|
||||
PID=$(pgrep -f 'python main.py' | head -1)
|
||||
if [ -n "$PID" ]; then
|
||||
ps -p $PID -o pid,pcpu,rss,vsz --no-header
|
||||
echo "VmRSS: $(grep VmRSS /proc/$PID/status)"
|
||||
fi
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Trace actual runtime imports from server.py to find ALL missing deps."""
|
||||
import subprocess, sys
|
||||
|
||||
# Check which pip packages are installed vs which are imported
|
||||
cmd = """
|
||||
cd /opt/qwen3-tts
|
||||
python3 -c "
|
||||
import importlib, sys
|
||||
|
||||
# All third-party modules found in the grep scan
|
||||
third_party = [
|
||||
'gguf', 'torch', 'transformers', 'yaml', 'tqdm',
|
||||
'sounddevice', 'PySide6', 'onnxruntime', 'numpy',
|
||||
'scipy', 'soundfile', 'tokenizers', 'flask', 'requests'
|
||||
]
|
||||
|
||||
for mod in third_party:
|
||||
try:
|
||||
importlib.import_module(mod)
|
||||
print(f'OK {mod}')
|
||||
except ImportError:
|
||||
print(f'MISS {mod}')
|
||||
"
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "--entrypoint", "bash", "sudx/qwen3-tts:latest", "-c", cmd],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
print(result.stdout)
|
||||
if result.stderr:
|
||||
print("STDERR:", result.stderr[-500:], file=sys.stderr)
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
echo "=== GGMLTensor class ==="
|
||||
sed -n '1,85p' /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py
|
||||
echo ""
|
||||
echo "=== load sig ==="
|
||||
grep -n "def load_models_gpu" /home/fabian/ComfyUI/comfy/model_management.py
|
||||
echo ""
|
||||
echo "=== what is m.model ==="
|
||||
grep -n "class Loaded" /home/fabian/ComfyUI/comfy/model_management.py | head -n 5
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# Read GGUF metadata from correct path + ZImage model config
|
||||
echo "=== GGUF metadata ==="
|
||||
/home/fabian/ComfyUI/venv/bin/python3 -c "
|
||||
import gguf
|
||||
reader = gguf.GGUFReader('/home/fabian/ComfyUI/models/unet/z_image_turbo-Q5_K_S.gguf')
|
||||
for key in sorted(reader.fields.keys()):
|
||||
field = reader.fields[key]
|
||||
try:
|
||||
parts = field.parts
|
||||
data_indices = field.data
|
||||
tp = str(field.types)
|
||||
if len(data_indices) > 0 and len(data_indices) < 10:
|
||||
raw = parts[data_indices[0]]
|
||||
if hasattr(raw, 'tobytes'):
|
||||
val = raw.tobytes().decode('utf-8', errors='replace')
|
||||
else:
|
||||
val = str(list(raw)[:5]) if hasattr(raw, '__len__') and len(raw) > 1 else str(raw)
|
||||
else:
|
||||
val = f'data_len={len(data_indices)}'
|
||||
print(f' {key} = {val} ({tp})')
|
||||
except Exception as e:
|
||||
print(f' {key} = ERROR: {e}')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== ZImage class in supported_models ==="
|
||||
sed -n '/^class ZImage/,/^class [A-Z]/p' /home/fabian/ComfyUI/comfy/supported_models.py | head -40
|
||||
|
||||
echo ""
|
||||
echo "=== Lumina2 class (parent) ==="
|
||||
sed -n '/^class Lumina2/,/^class [A-Z]/p' /home/fabian/ComfyUI/comfy/supported_models.py | head -40
|
||||
|
||||
echo ""
|
||||
echo "=== What text encoder files exist ==="
|
||||
ls /home/fabian/ComfyUI/models/text_encoders/
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
# Detailed GGUF analysis: tensor names and shapes
|
||||
/home/fabian/ComfyUI/venv/bin/python3 << 'PYEOF'
|
||||
import gguf
|
||||
import numpy as np
|
||||
|
||||
reader = gguf.GGUFReader("/home/fabian/ComfyUI/models/unet/z_image_turbo-Q5_K_S.gguf")
|
||||
|
||||
print("=== GGUF Tensors (first 30) ===")
|
||||
for i, tensor in enumerate(reader.tensors[:30]):
|
||||
print(f" {tensor.name}: shape={list(tensor.shape)} type={tensor.tensor_type}")
|
||||
|
||||
print(f"\nTotal tensors: {len(reader.tensors)}")
|
||||
|
||||
# Check max dim to determine model size
|
||||
dims = set()
|
||||
for t in reader.tensors:
|
||||
for s in t.shape:
|
||||
dims.add(int(s))
|
||||
|
||||
# Typical dim signatures:
|
||||
# Lumina2 base: hidden=2304 (24 layers)
|
||||
# ZImage: hidden=3840 (32 layers? depends on config)
|
||||
print(f"\nDistinct tensor dimensions: {sorted(dims)[:20]}")
|
||||
|
||||
# Check if dim 3840 appears (Z-Image specific)
|
||||
has_3840 = any(3840 in t.shape for t in reader.tensors)
|
||||
has_2304 = any(2304 in t.shape for t in reader.tensors)
|
||||
print(f"\nHas dim 3840 (Z-Image): {has_3840}")
|
||||
print(f"Has dim 2304 (Lumina2 base): {has_2304}")
|
||||
|
||||
# Count layer numbers to determine depth
|
||||
import re
|
||||
layer_nums = set()
|
||||
for t in reader.tensors:
|
||||
m = re.search(r'\.(\d+)\.', t.name)
|
||||
if m:
|
||||
layer_nums.add(int(m.group(1)))
|
||||
if layer_nums:
|
||||
print(f"Layer range: {min(layer_nums)} to {max(layer_nums)} ({len(layer_nums)} layers)")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== Disk space check ==="
|
||||
df -h /home/fabian/ | tail -1
|
||||
|
||||
echo ""
|
||||
echo "=== Available model size ==="
|
||||
ls -lh /home/fabian/ComfyUI/models/unet/z_image_turbo-Q5_K_S.gguf
|
||||
ls -lh /home/fabian/ComfyUI/models/text_encoders/gemma2_2b_lumina2.safetensors
|
||||
ls -lh /home/fabian/ComfyUI/models/vae/ae.safetensors 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== RAM available ==="
|
||||
free -h | head -2
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
# Find the GGUF file and check its metadata
|
||||
echo "=== Finding GGUF file ==="
|
||||
find /home/fabian/ComfyUI/models -name '*.gguf' 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== GGUF metadata ==="
|
||||
GGUF_FILE=$(find /home/fabian/ComfyUI/models -name 'z_image_turbo*' 2>/dev/null | head -1)
|
||||
echo "Found: $GGUF_FILE"
|
||||
|
||||
if [ -n "$GGUF_FILE" ]; then
|
||||
/home/fabian/ComfyUI/venv/bin/python3 -c "
|
||||
import gguf
|
||||
reader = gguf.GGUFReader('$GGUF_FILE')
|
||||
print('GGUF fields:')
|
||||
for key in sorted(reader.fields.keys()):
|
||||
field = reader.fields[key]
|
||||
# Show field type and value if small
|
||||
parts = field.parts
|
||||
tp = str(field.types)
|
||||
if hasattr(field, 'data') and len(field.data) < 100:
|
||||
try:
|
||||
val = list(parts[field.data[0]])[:5] if len(field.data) > 0 else 'empty'
|
||||
except:
|
||||
val = '?'
|
||||
else:
|
||||
val = f'data_len={len(field.data) if hasattr(field, \"data\") else \"?\"}'
|
||||
print(f' {key}: types={tp} val={val}')
|
||||
"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Check supported_models for z_image ==="
|
||||
grep -n 'z_image\|ZImage\|z-image\|Z_IMAGE' /home/fabian/ComfyUI/comfy/supported_models.py | head -20
|
||||
|
||||
echo ""
|
||||
echo "=== Check what model arch the GGUF uses ==="
|
||||
if [ -n "$GGUF_FILE" ]; then
|
||||
/home/fabian/ComfyUI/venv/bin/python3 -c "
|
||||
import gguf
|
||||
reader = gguf.GGUFReader('$GGUF_FILE')
|
||||
# Get architecture-related fields
|
||||
for key in reader.fields:
|
||||
if 'arch' in key or 'model' in key or 'type' in key or 'name' in key:
|
||||
field = reader.fields[key]
|
||||
try:
|
||||
parts = field.parts
|
||||
data_indices = field.data
|
||||
if len(data_indices) > 0:
|
||||
raw = parts[data_indices[0]]
|
||||
if hasattr(raw, 'tobytes'):
|
||||
val = raw.tobytes().decode('utf-8', errors='replace')
|
||||
else:
|
||||
val = str(raw)
|
||||
else:
|
||||
val = '<empty>'
|
||||
except:
|
||||
val = '<error>'
|
||||
print(f' {key} = {val}')
|
||||
"
|
||||
fi
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
echo "=== GGUF ops.py cast_bias_weight ==="
|
||||
grep -n "cast_bias_weight\|def forward_ggml\|def forward_comfy" /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py | head -n 20
|
||||
echo ""
|
||||
echo "=== cast_bias_weight full function ==="
|
||||
sed -n '/def cast_bias_weight/,/^[[:space:]]*def /p' /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py | head -n 30
|
||||
echo ""
|
||||
echo "=== forward_ggml_cast_weights call ==="
|
||||
sed -n '/def forward_ggml_cast_weights/,/^[[:space:]]*def /p' /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/ops.py | head -n 20
|
||||
echo ""
|
||||
echo "=== dequant.py get_scale_min ==="
|
||||
sed -n '125,145p' /home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF/dequant.py
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check which imports the inference/ subpackage actually needs."""
|
||||
import subprocess, sys
|
||||
|
||||
cmd = """
|
||||
cd /opt/qwen3-tts
|
||||
# Only scan inference/ subdir (the actual runtime path)
|
||||
grep -rh '^import\\|^from' qwen3_tts_gguf/inference/ 2>/dev/null | \
|
||||
grep -v __pycache__ | grep -v '^\\.\\|^from \\.' | sort -u
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "--entrypoint", "bash", "sudx/qwen3-tts:latest", "-c", cmd],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
print("=== inference/ external imports ===")
|
||||
print(result.stdout)
|
||||
|
||||
# Also check schema/ since inference imports it
|
||||
cmd2 = """
|
||||
cd /opt/qwen3-tts
|
||||
grep -rh '^import\\|^from' qwen3_tts_gguf/schema/ 2>/dev/null | \
|
||||
grep -v __pycache__ | grep -v '^from \\.' | sort -u
|
||||
"""
|
||||
result2 = subprocess.run(
|
||||
["docker", "run", "--rm", "--entrypoint", "bash", "sudx/qwen3-tts:latest", "-c", cmd2],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
print("=== schema/ external imports ===")
|
||||
print(result2.stdout)
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check inference __init__.py and engine.py imports."""
|
||||
import subprocess, sys
|
||||
|
||||
cmd = """
|
||||
echo "=== inference/__init__.py ==="
|
||||
cat /opt/qwen3-tts/qwen3_tts_gguf/inference/__init__.py
|
||||
|
||||
echo ""
|
||||
echo "=== engine.py imports ==="
|
||||
head -30 /opt/qwen3-tts/qwen3_tts_gguf/inference/engine.py
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "--entrypoint", "bash", "sudx/qwen3-tts:latest", "-c", cmd],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
print(result.stdout)
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
# Check what the /models/load endpoint expects
|
||||
curl -s http://localhost:8080/openapi.json | python3 -c '
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
load = d["paths"].get("/models/load", {}).get("post", {})
|
||||
print(json.dumps(load, indent=2))
|
||||
'
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
grep -n -E "Requested|loaded completely|Using split|VAE load|Prompt executed|VAE decode" /home/fabian/comfyui8.log
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
echo "=== ulimit ==="
|
||||
ulimit -l
|
||||
echo "=== Process Memory ==="
|
||||
PID=$(pgrep -f "python.*main.py" | head -1)
|
||||
if [ -n "$PID" ]; then
|
||||
grep -i -E 'VmSize|VmRSS|VmLck|VmSwap' /proc/$PID/status
|
||||
else
|
||||
echo "ComfyUI not running"
|
||||
fi
|
||||
echo "=== System Memory ==="
|
||||
free -m
|
||||
echo "=== Swap ==="
|
||||
swapon --show
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
# Check how model_management handles VAE
|
||||
grep -n -E "class.*ModelPatcher|def load_model|def lowvram|keep_loaded|current_loaded|KEEP" /home/fabian/ComfyUI/comfy/model_management.py | head -40
|
||||
echo "=== VAE class ==="
|
||||
grep -n -E "class VAE|def decode|def encode|first_stage|load_device|offload" /home/fabian/ComfyUI/comfy/sd.py | head -30
|
||||
echo "=== model patcher keep ==="
|
||||
grep -n -E "keep|pin|persist|resident|locked" /home/fabian/ComfyUI/comfy/model_management.py | head -20
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
sleep 1
|
||||
curl -s http://localhost:9090/api/models | python3 -c "
|
||||
import sys, json
|
||||
for m in json.load(sys.stdin):
|
||||
print(m['id'], m['cat'])
|
||||
"
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# Analyze ppfeaturemask for AMD GPU OC
|
||||
|
||||
MASK=$(cat /sys/module/amdgpu/parameters/ppfeaturemask 2>/dev/null)
|
||||
echo "Current ppfeaturemask: $MASK"
|
||||
|
||||
python3 -c "
|
||||
mask = $MASK
|
||||
print(f'Hex: {mask:#010x}')
|
||||
print(f'Binary: {mask:032b}')
|
||||
print()
|
||||
bits = {
|
||||
0: 'PP_FEATURE_DPM_PREFETCHER',
|
||||
1: 'PP_FEATURE_DPM_GFXCLK',
|
||||
2: 'PP_FEATURE_DPM_UCLK',
|
||||
3: 'PP_FEATURE_DPM_SOCCLK',
|
||||
4: 'PP_FEATURE_DPM_MP0CLK',
|
||||
5: 'PP_FEATURE_DPM_LINK',
|
||||
6: 'PP_FEATURE_DPM_DCEFCLK',
|
||||
8: 'PP_FEATURE_DS_GFXCLK',
|
||||
9: 'PP_FEATURE_DS_SOCCLK',
|
||||
10: 'PP_FEATURE_DS_LCLK',
|
||||
11: 'PP_FEATURE_DS_FCLK',
|
||||
12: 'PP_FEATURE_DS_MP1CLK',
|
||||
13: 'PP_FEATURE_FW_DSTATE',
|
||||
14: 'PP_OVERDRIVE_MASK',
|
||||
15: 'PP_GFXOFF_MASK',
|
||||
}
|
||||
for bit, name in sorted(bits.items()):
|
||||
val = bool((mask >> bit) & 1)
|
||||
flag = 'ON' if val else 'OFF'
|
||||
print(f' Bit {bit:2d}: {flag:3s} - {name}')
|
||||
|
||||
# Try all bits ON
|
||||
full = 0xffffffff
|
||||
print(f'\nFull enable: {full:#010x}')
|
||||
# OD specifically
|
||||
od = mask | (1 << 14)
|
||||
print(f'With OD: {od:#010x}')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Current boot params ==="
|
||||
grep -i amdgpu /boot/limine.conf 2>/dev/null || grep -i amdgpu /proc/cmdline 2>/dev/null
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
PID=$(pgrep -f 'main.py')
|
||||
echo "PID: $PID"
|
||||
echo "=== STATE ==="
|
||||
cat /proc/$PID/status | grep -E 'State|Threads|VmRSS|VmSize'
|
||||
echo "=== TOP THREADS ==="
|
||||
ps -p $PID -T -o spid,state,%cpu,%mem,time,comm | head -20
|
||||
echo "=== COMGR CACHE ==="
|
||||
ls -lt ~/.cache/comgr/ | head -5
|
||||
echo "=== GPU MEM ==="
|
||||
cat /sys/class/drm/card0/device/mem_info_vram_used 2>/dev/null || echo "N/A"
|
||||
cat /sys/class/drm/card0/device/mem_info_gtt_used 2>/dev/null || echo "N/A"
|
||||
echo "=== STRACE SAMPLE ==="
|
||||
timeout 2 strace -p $PID -e trace=write,read,ioctl -c 2>&1 || echo "strace failed (needs root?)"
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# Check available Z-Image text encoder repos
|
||||
/home/fabian/ComfyUI/venv/bin/python3 << 'PYEOF'
|
||||
from huggingface_hub import HfApi, list_repo_files
|
||||
|
||||
# Check both found repos
|
||||
repos = [
|
||||
"Norby/Z_Image_text_encoders",
|
||||
"worstplayer/Z-Image_Qwen_3_4b_text_encoder_GGUF",
|
||||
]
|
||||
|
||||
for repo in repos:
|
||||
try:
|
||||
files = list_repo_files(repo)
|
||||
print(f"\n=== {repo} ===")
|
||||
for f in files:
|
||||
print(f" {f}")
|
||||
except Exception as e:
|
||||
print(f"\n=== {repo} ===")
|
||||
print(f" ERROR: {str(e)[:120]}")
|
||||
|
||||
# Also try official Tongyi repo
|
||||
try:
|
||||
files = list_repo_files("Tongyi-MAI/Z-Image-Turbo")
|
||||
print(f"\n=== Tongyi-MAI/Z-Image-Turbo ===")
|
||||
for f in files:
|
||||
print(f" {f}")
|
||||
except Exception as e:
|
||||
print(f"\n=== Tongyi-MAI/Z-Image-Turbo ===")
|
||||
print(f" ERROR: {str(e)[:120]}")
|
||||
|
||||
# Try Qwen3-4B base
|
||||
try:
|
||||
files = list_repo_files("Qwen/Qwen3-4B-Base")
|
||||
print(f"\n=== Qwen/Qwen3-4B-Base ===")
|
||||
for f in files[:15]:
|
||||
print(f" {f}")
|
||||
except Exception as e:
|
||||
print(f"\n=== Qwen/Qwen3-4B-Base ===")
|
||||
print(f" ERROR: {str(e)[:120]}")
|
||||
PYEOF
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
curl -s http://localhost:8080/openapi.json | python3 -c '
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
schema = d["components"]["schemas"].get("LoadModelRequest", {})
|
||||
print(json.dumps(schema, indent=2))
|
||||
'
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
echo "=== DMESG GPU ==="
|
||||
sudo dmesg | grep -iE "amdgpu|gfx|error|reset|fault" | tail -15
|
||||
|
||||
echo "=== KFD THREADS ==="
|
||||
ls /proc/15070/task/ | while read tid; do
|
||||
wchan=$(cat /proc/15070/task/$tid/wchan 2>/dev/null)
|
||||
if [ "$wchan" = "kfd_wait_on_events" ] || [ "$wchan" = "poll_idle" ]; then
|
||||
echo "Thread $tid: $wchan"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "=== LOG ERRORS ==="
|
||||
grep -i "error\|traceback\|exception\|fail" /home/fabian/comfyui5.log 2>/dev/null | tail -10
|
||||
|
||||
echo "=== LOG LAST LINE (not clip) ==="
|
||||
grep -v "clip missing" /home/fabian/comfyui5.log | tail -5
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
# Check threading settings + model caching behavior
|
||||
echo "=== THREADS ==="
|
||||
grep -n -i -E "thread|num_worker|dataloader|OMP|MKL|parallel" /home/fabian/ComfyUI/comfy/model_management.py | head -20
|
||||
echo "=== torch threads ==="
|
||||
grep -rn -i "set_num_threads\|num_threads\|OMP_NUM\|MKL_NUM\|torch.get_num_threads\|interop" /home/fabian/ComfyUI/comfy/ --include="*.py" | head -15
|
||||
echo "=== free_memory function ==="
|
||||
sed -n '636,680p' /home/fabian/ComfyUI/comfy/model_management.py
|
||||
echo "=== load_models_gpu ==="
|
||||
sed -n '680,780p' /home/fabian/ComfyUI/comfy/model_management.py
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# Check for tokenizer files and safetensors metadata
|
||||
|
||||
echo "=== Tokenizer files ==="
|
||||
find /home/fabian/ComfyUI -name '*spiece*' -o -name '*sentencepiece*' -o -name '*.model' 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "=== text_encoders directory ==="
|
||||
ls /home/fabian/ComfyUI/comfy/text_encoders/
|
||||
|
||||
echo ""
|
||||
echo "=== safetensors metadata ==="
|
||||
cd /home/fabian/ComfyUI
|
||||
/home/fabian/ComfyUI/venv/bin/python3 -c "
|
||||
import safetensors
|
||||
f = safetensors.safe_open('models/text_encoders/gemma2_2b_lumina2.safetensors', framework='pt')
|
||||
md = f.metadata()
|
||||
if md:
|
||||
print('metadata keys:', list(md.keys())[:30])
|
||||
for k in md:
|
||||
v = md[k]
|
||||
if len(v) > 200:
|
||||
print(f' {k}: len={len(v)} (first 100 chars: {v[:100]}...) ')
|
||||
else:
|
||||
print(f' {k}: {v}')
|
||||
else:
|
||||
print('No metadata found')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== SPieceTokenizer source ==="
|
||||
head -40 /home/fabian/ComfyUI/comfy/text_encoders/spiece_tokenizer.py
|
||||
|
||||
echo ""
|
||||
echo "=== Gemma2 tokenizer lookup in SDTokenizer ==="
|
||||
grep -n 'tokenizer_path\|tokenizer_data\|spiece\|from_pretrained' /home/fabian/ComfyUI/comfy/sd1_clip.py | head -20
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# Check GGUF file for tokenizer data and check available tokenizer files
|
||||
|
||||
echo "=== GGUF metadata check ==="
|
||||
cd /home/fabian/ComfyUI
|
||||
/home/fabian/ComfyUI/venv/bin/python3 -c "
|
||||
import gguf
|
||||
reader = gguf.GGUFReader('models/diffusion_models/z_image_turbo-Q5_K_S.gguf')
|
||||
print('GGUF fields:')
|
||||
for key in list(reader.fields.keys())[:40]:
|
||||
field = reader.fields[key]
|
||||
tp = str(field.types)
|
||||
dl = len(field.data) if hasattr(field, 'data') else 0
|
||||
print(f' {key}: types={tp} data_len={dl}')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Check for tokenizer.model files ==="
|
||||
find /home/fabian/ComfyUI/comfy/text_encoders -name 'tokenizer*' -type f 2>/dev/null
|
||||
echo ""
|
||||
|
||||
echo "=== Check llama_tokenizer ==="
|
||||
ls -la /home/fabian/ComfyUI/comfy/text_encoders/llama_tokenizer/ 2>/dev/null || echo "No llama_tokenizer dir"
|
||||
|
||||
echo ""
|
||||
echo "=== Check what z_image.py expects ==="
|
||||
cat /home/fabian/ComfyUI/comfy/text_encoders/z_image.py 2>/dev/null || echo "No z_image.py"
|
||||
|
||||
echo ""
|
||||
echo "=== Grep spiece_model in load functions ==="
|
||||
grep -n 'spiece_model\|tokenizer_data\[' /home/fabian/ComfyUI/comfy/sd.py | head -20
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
grep -n -i -E "vae|unload|offload|cleanup|load_model|lowvram|FREE|Requested to load|loaded completely|memory" /home/fabian/comfyui8.log | tail -60
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BC-250: Test CLIP on CPU only (bypass GPU kernel compilation).
|
||||
"""
|
||||
import os, sys, time
|
||||
|
||||
os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.1.0"
|
||||
os.environ["HSA_ENABLE_SDMA"] = "0"
|
||||
os.environ["HIP_VISIBLE_DEVICES"] = "0"
|
||||
os.environ["BC250_SOFTMAX_THRESHOLD"] = "512"
|
||||
|
||||
sys.path.insert(0, "/home/fabian/ComfyUI")
|
||||
|
||||
print("[T] Importing...", flush=True)
|
||||
import bc250_softmax_patch
|
||||
import torch
|
||||
import safetensors.torch
|
||||
|
||||
# Check file size
|
||||
clip_path = "/home/fabian/ComfyUI/models/text_encoders/gemma2_2b_lumina2.safetensors"
|
||||
fsize = os.path.getsize(clip_path) / (1024*1024*1024)
|
||||
print(f"[T] CLIP file: {fsize:.2f} GB", flush=True)
|
||||
|
||||
# Load directly to see what's in it
|
||||
print(f"[T] Loading safetensors headers...", flush=True)
|
||||
t0 = time.time()
|
||||
with safetensors.torch.safe_open(clip_path, framework="pt", device="cpu") as f:
|
||||
keys = list(f.keys())
|
||||
print(f"[T] Keys: {len(keys)}", flush=True)
|
||||
print(f"[T] First 5 keys: {keys[:5]}", flush=True)
|
||||
|
||||
# Check dtype and shapes of first key
|
||||
first_tensor = f.get_tensor(keys[0])
|
||||
print(f"[T] First tensor: {keys[0]} shape={first_tensor.shape} dtype={first_tensor.dtype}", flush=True)
|
||||
|
||||
# Check total parameter count
|
||||
total_params = 0
|
||||
for k in keys:
|
||||
t = f.get_tensor(k)
|
||||
total_params += t.numel()
|
||||
print(f"[T] Total params: {total_params/1e9:.2f}B", flush=True)
|
||||
|
||||
dt = time.time() - t0
|
||||
print(f"[T] Loaded headers in {dt:.1f}s", flush=True)
|
||||
|
||||
# Now try loading CLIP with ComfyUI but force CPU
|
||||
print(f"\n[T] Loading CLIP through ComfyUI (on CPU)...", flush=True)
|
||||
import comfy.sd
|
||||
import comfy.model_management
|
||||
import folder_paths
|
||||
|
||||
# Monkey-patch to force CPU loading for CLIP
|
||||
_orig_get_torch_device = comfy.model_management.get_torch_device
|
||||
_orig_text_encoder_device = comfy.model_management.text_encoder_device
|
||||
_orig_text_encoder_offload = comfy.model_management.text_encoder_offload_device
|
||||
|
||||
# Force text encoder to CPU
|
||||
comfy.model_management.text_encoder_device = lambda: torch.device("cpu")
|
||||
comfy.model_management.text_encoder_offload_device = lambda: torch.device("cpu")
|
||||
|
||||
t1 = time.time()
|
||||
try:
|
||||
clip = comfy.sd.load_clip(
|
||||
ckpt_paths=[clip_path],
|
||||
embedding_directory=None,
|
||||
clip_type=comfy.sd.CLIPType.LUMINA2,
|
||||
)
|
||||
dt = time.time() - t1
|
||||
print(f"[T] CLIP loaded in {dt:.1f}s", flush=True)
|
||||
|
||||
# Test encoding
|
||||
print(f"[T] Testing text encoding on CPU...", flush=True)
|
||||
t2 = time.time()
|
||||
tokens = clip.tokenize({"g": "a photo of a cat sitting on a windowsill"})
|
||||
print(f"[T] Tokenized in {time.time()-t2:.3f}s", flush=True)
|
||||
|
||||
t3 = time.time()
|
||||
output = clip.encode_from_tokens_scheduled(tokens)
|
||||
cond = output[0]
|
||||
dt = time.time() - t3
|
||||
print(f"[T] CLIP encoded in {dt:.1f}s", flush=True)
|
||||
print(f"[T] Output shape: {cond.shape}, dtype: {cond.dtype}", flush=True)
|
||||
print(f"\n[T] === CLIP ON CPU WORKS! ===", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[T] ERROR: {e}", flush=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Restore
|
||||
comfy.model_management.text_encoder_device = _orig_text_encoder_device
|
||||
comfy.model_management.text_encoder_offload_device = _orig_text_encoder_offload
|
||||
|
||||
print(f"[T] Total: {time.time()-t0:.1f}s", flush=True)
|
||||
os._exit(0)
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BC-250: Test CLIP encoding on CPU — find out if Gemma-2 2B works.
|
||||
"""
|
||||
import os, sys, time, signal
|
||||
|
||||
os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.1.0"
|
||||
os.environ["HSA_ENABLE_SDMA"] = "0"
|
||||
os.environ["HIP_VISIBLE_DEVICES"] = "0"
|
||||
os.environ["BC250_SOFTMAX_THRESHOLD"] = "512"
|
||||
|
||||
sys.path.insert(0, "/home/fabian/ComfyUI")
|
||||
|
||||
def timeout_handler(sig, frame):
|
||||
print("\n[T] === TIMEOUT HIT ===", flush=True)
|
||||
os._exit(1)
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
print("[T] Importing...", flush=True)
|
||||
import bc250_softmax_patch
|
||||
import torch
|
||||
import comfy.sd
|
||||
import comfy.model_management
|
||||
|
||||
# Force text encoder to CPU
|
||||
comfy.model_management.text_encoder_device = lambda: torch.device("cpu")
|
||||
comfy.model_management.text_encoder_offload_device = lambda: torch.device("cpu")
|
||||
|
||||
clip_path = "/home/fabian/ComfyUI/models/text_encoders/gemma2_2b_lumina2.safetensors"
|
||||
fsize = os.path.getsize(clip_path) / (1024*1024*1024)
|
||||
print(f"[T] CLIP: {fsize:.2f} GB", flush=True)
|
||||
|
||||
# Load CLIP
|
||||
t0 = time.time()
|
||||
clip = comfy.sd.load_clip(
|
||||
ckpt_paths=[clip_path],
|
||||
embedding_directory=None,
|
||||
clip_type=comfy.sd.CLIPType.LUMINA2,
|
||||
)
|
||||
print(f"[T] CLIP loaded in {time.time()-t0:.1f}s", flush=True)
|
||||
|
||||
# Check the clip object
|
||||
print(f"[T] CLIP type: {type(clip)}", flush=True)
|
||||
print(f"[T] CLIP cond_stage_model type: {type(clip.cond_stage_model)}", flush=True)
|
||||
|
||||
# Tokenize with just a string
|
||||
text = "a photo of a cat"
|
||||
print(f"[T] Tokenizing: '{text}'", flush=True)
|
||||
t1 = time.time()
|
||||
tokens = clip.tokenize(text)
|
||||
dt = time.time() - t1
|
||||
print(f"[T] Tokenized in {dt:.3f}s", flush=True)
|
||||
print(f"[T] Token keys: {list(tokens.keys()) if isinstance(tokens, dict) else type(tokens)}", flush=True)
|
||||
|
||||
# Encode with 120s timeout
|
||||
print(f"[T] Encoding (120s timeout)...", flush=True)
|
||||
signal.alarm(120)
|
||||
t2 = time.time()
|
||||
try:
|
||||
output = clip.encode_from_tokens_scheduled(tokens)
|
||||
dt = time.time() - t2
|
||||
signal.alarm(0)
|
||||
print(f"[T] Encoded in {dt:.1f}s", flush=True)
|
||||
|
||||
if isinstance(output, dict):
|
||||
for k, v in output.items():
|
||||
if hasattr(v, 'shape'):
|
||||
print(f"[T] {k}: shape={v.shape} dtype={v.dtype}", flush=True)
|
||||
else:
|
||||
print(f"[T] {k}: {type(v)}", flush=True)
|
||||
elif isinstance(output, (list, tuple)):
|
||||
for i, v in enumerate(output):
|
||||
if hasattr(v, 'shape'):
|
||||
print(f"[T] [{i}]: shape={v.shape} dtype={v.dtype}", flush=True)
|
||||
else:
|
||||
print(f"[T] [{i}]: {type(v)}", flush=True)
|
||||
|
||||
print(f"\n[T] === CLIP ENCODE ON CPU: SUCCESS ===", flush=True)
|
||||
except Exception as e:
|
||||
signal.alarm(0)
|
||||
print(f"[T] ERROR: {e}", flush=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print(f"[T] Total: {time.time()-t0:.1f}s", flush=True)
|
||||
os._exit(0)
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BC-250: Test CLIP with Gemma2 key fix — verify correct model detection.
|
||||
"""
|
||||
import os, sys, time, signal
|
||||
|
||||
os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.1.0"
|
||||
os.environ["HSA_ENABLE_SDMA"] = "0"
|
||||
os.environ["HIP_VISIBLE_DEVICES"] = "0"
|
||||
os.environ["BC250_SOFTMAX_THRESHOLD"] = "512"
|
||||
|
||||
sys.path.insert(0, "/home/fabian/ComfyUI")
|
||||
|
||||
def timeout_handler(sig, frame):
|
||||
print("\n[T] === TIMEOUT HIT ===", flush=True)
|
||||
os._exit(1)
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
print("[T] Importing...", flush=True)
|
||||
import bc250_softmax_patch
|
||||
import torch
|
||||
import comfy.sd
|
||||
import comfy.model_management
|
||||
|
||||
# Force CLIP to CPU to avoid GPU kernel compilation delays
|
||||
comfy.model_management.text_encoder_device = lambda: torch.device("cpu")
|
||||
comfy.model_management.text_encoder_offload_device = lambda: torch.device("cpu")
|
||||
|
||||
clip_path = "/home/fabian/ComfyUI/models/text_encoders/gemma2_2b_lumina2.safetensors"
|
||||
|
||||
# Load CLIP
|
||||
print(f"[T] Loading CLIP...", flush=True)
|
||||
t0 = time.time()
|
||||
clip = comfy.sd.load_clip(
|
||||
ckpt_paths=[clip_path],
|
||||
embedding_directory=None,
|
||||
clip_type=comfy.sd.CLIPType.LUMINA2,
|
||||
)
|
||||
dt = time.time() - t0
|
||||
print(f"[T] CLIP loaded in {dt:.1f}s", flush=True)
|
||||
print(f"[T] CLIP type: {type(clip)}", flush=True)
|
||||
print(f"[T] cond_stage_model type: {type(clip.cond_stage_model)}", flush=True)
|
||||
|
||||
# Check if it's Gemma2 now
|
||||
csm = clip.cond_stage_model
|
||||
print(f"[T] Has gemma2_2b attr: {hasattr(csm, 'gemma2_2b')}", flush=True)
|
||||
|
||||
# List attributes
|
||||
attrs = [a for a in dir(csm) if not a.startswith('_') and not callable(getattr(csm, a, None))]
|
||||
print(f"[T] CSM attrs (non-callable): {attrs[:15]}", flush=True)
|
||||
|
||||
# Tokenize
|
||||
text = "a photo of a cat sitting on a windowsill"
|
||||
print(f"\n[T] Tokenizing: '{text}'", flush=True)
|
||||
t1 = time.time()
|
||||
tokens = clip.tokenize(text)
|
||||
dt = time.time() - t1
|
||||
print(f"[T] Tokenized in {dt:.3f}s", flush=True)
|
||||
print(f"[T] Token keys: {list(tokens.keys()) if isinstance(tokens, dict) else type(tokens)}", flush=True)
|
||||
for k, v in tokens.items():
|
||||
if isinstance(v, list):
|
||||
for j, item in enumerate(v[:2]):
|
||||
if isinstance(item, list):
|
||||
print(f"[T] {k}[{j}]: list len={len(item)}", flush=True)
|
||||
elif hasattr(item, 'shape'):
|
||||
print(f"[T] {k}[{j}]: shape={item.shape}", flush=True)
|
||||
else:
|
||||
print(f"[T] {k}[{j}]: {type(item)}", flush=True)
|
||||
elif hasattr(v, 'shape'):
|
||||
print(f"[T] {k}: shape={v.shape}", flush=True)
|
||||
else:
|
||||
print(f"[T] {k}: {type(v)}", flush=True)
|
||||
|
||||
# Encode with 180s timeout (Gemma-2 2B on CPU = slow!)
|
||||
print(f"\n[T] Encoding (180s timeout)...", flush=True)
|
||||
signal.alarm(180)
|
||||
t2 = time.time()
|
||||
try:
|
||||
output = clip.encode_from_tokens_scheduled(tokens)
|
||||
dt = time.time() - t2
|
||||
signal.alarm(0)
|
||||
print(f"[T] Encoded in {dt:.1f}s", flush=True)
|
||||
|
||||
if isinstance(output, (list, tuple)):
|
||||
for i, item in enumerate(output):
|
||||
if isinstance(item, (list, tuple)):
|
||||
print(f"[T] [{i}]: list/tuple len={len(item)}", flush=True)
|
||||
if len(item) > 0 and isinstance(item[0], dict):
|
||||
for k, v in item[0].items():
|
||||
if hasattr(v, 'shape'):
|
||||
print(f"[T] [{i}][0]['{k}']: shape={v.shape} dtype={v.dtype}", flush=True)
|
||||
else:
|
||||
print(f"[T] [{i}][0]['{k}']: {type(v)} = {v}", flush=True)
|
||||
elif len(item) > 0 and hasattr(item[0], 'shape'):
|
||||
print(f"[T] [{i}][0]: shape={item[0].shape} dtype={item[0].dtype}", flush=True)
|
||||
elif hasattr(item, 'shape'):
|
||||
print(f"[T] [{i}]: shape={item.shape} dtype={item.dtype}", flush=True)
|
||||
else:
|
||||
print(f"[T] [{i}]: {type(item)}", flush=True)
|
||||
elif isinstance(output, dict):
|
||||
for k, v in output.items():
|
||||
if hasattr(v, 'shape'):
|
||||
print(f"[T] {k}: shape={v.shape} dtype={v.dtype}", flush=True)
|
||||
else:
|
||||
print(f"[T] {k}: {type(v)}", flush=True)
|
||||
|
||||
print(f"\n[T] === CLIP ENCODE SUCCESS ===", flush=True)
|
||||
except Exception as e:
|
||||
signal.alarm(0)
|
||||
print(f"[T] ERROR: {e}", flush=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print(f"[T] Total: {time.time()-t0:.1f}s", flush=True)
|
||||
os._exit(0)
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BC-250 ComfyUI GGUF Integration Test
|
||||
Tests the actual ComfyUI loading pipeline step by step.
|
||||
"""
|
||||
import os, sys, time
|
||||
|
||||
os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.1.0"
|
||||
os.environ["HSA_ENABLE_SDMA"] = "0"
|
||||
os.environ["HIP_VISIBLE_DEVICES"] = "0"
|
||||
os.environ["BC250_SOFTMAX_THRESHOLD"] = "512"
|
||||
|
||||
sys.path.insert(0, "/home/fabian/ComfyUI")
|
||||
|
||||
print("[TEST] Importing bc250_softmax_patch...", flush=True)
|
||||
import bc250_softmax_patch
|
||||
|
||||
print("[TEST] Importing torch...", flush=True)
|
||||
t0 = time.time()
|
||||
import torch
|
||||
print(f"[TEST] torch ready in {time.time()-t0:.1f}s", flush=True)
|
||||
|
||||
print(f"[TEST] CUDA available: {torch.cuda.is_available()}", flush=True)
|
||||
print(f"[TEST] Device: {torch.cuda.get_device_name(0)}", flush=True)
|
||||
|
||||
# Step 1: Load GGUF using ComfyUI-GGUF loader
|
||||
print(f"\n[TEST] === Step 1: gguf_sd_loader ===", flush=True)
|
||||
|
||||
# Import ComfyUI-GGUF properly as a package
|
||||
import importlib
|
||||
custom_nodes_path = "/home/fabian/ComfyUI/custom_nodes"
|
||||
if custom_nodes_path not in sys.path:
|
||||
sys.path.insert(0, custom_nodes_path)
|
||||
|
||||
# Force import as package
|
||||
gguf_pkg = importlib.import_module("ComfyUI-GGUF")
|
||||
from importlib import import_module
|
||||
gguf_loader = import_module("ComfyUI-GGUF.loader")
|
||||
gguf_dequant = import_module("ComfyUI-GGUF.dequant")
|
||||
gguf_ops = import_module("ComfyUI-GGUF.ops")
|
||||
|
||||
gguf_sd_loader = gguf_loader.gguf_sd_loader
|
||||
|
||||
t1 = time.time()
|
||||
sd, extra = gguf_sd_loader("/home/fabian/ComfyUI/models/unet/z_image_turbo-Q5_K_S.gguf")
|
||||
dt = time.time() - t1
|
||||
print(f"[TEST] State dict loaded in {dt:.1f}s", flush=True)
|
||||
print(f"[TEST] Keys: {len(sd)}", flush=True)
|
||||
print(f"[TEST] Architecture: {extra.get('arch_str')}", flush=True)
|
||||
|
||||
# Check some tensor info
|
||||
is_quantized = gguf_dequant.is_quantized
|
||||
q_count = sum(1 for v in sd.values() if is_quantized(v))
|
||||
print(f"[TEST] Quantized tensors: {q_count}/{len(sd)}", flush=True)
|
||||
|
||||
# Step 2: Test a single dequantize on CPU
|
||||
print(f"\n[TEST] === Step 2: Single tensor dequant ===", flush=True)
|
||||
dequantize_tensor = gguf_dequant.dequantize_tensor
|
||||
for k, v in sd.items():
|
||||
if is_quantized(v):
|
||||
print(f"[TEST] Dequantizing: {k} shape={v.tensor_shape} type={v.tensor_type}", flush=True)
|
||||
t2 = time.time()
|
||||
result = dequantize_tensor(v, dtype=torch.float16)
|
||||
dt = time.time() - t2
|
||||
print(f"[TEST] Done in {dt:.3f}s -> {result.shape} {result.dtype}", flush=True)
|
||||
|
||||
# Move to GPU
|
||||
t3 = time.time()
|
||||
gpu = result.to("cuda:0")
|
||||
torch.cuda.synchronize()
|
||||
dt2 = time.time() - t3
|
||||
print(f"[TEST] GPU transfer in {dt2:.3f}s", flush=True)
|
||||
del gpu, result
|
||||
break
|
||||
|
||||
# Step 3: Test loading the model through ComfyUI model management
|
||||
print(f"\n[TEST] === Step 3: ComfyUI model loading ===", flush=True)
|
||||
try:
|
||||
import comfy.sd
|
||||
import comfy.model_management
|
||||
|
||||
print(f"[TEST] Loading model config...", flush=True)
|
||||
t4 = time.time()
|
||||
|
||||
# Use the GGMLOps
|
||||
GGMLOps = gguf_ops.GGMLOps
|
||||
|
||||
# Try to load via comfy's model loading
|
||||
import comfy.supported_models
|
||||
import comfy.model_patcher
|
||||
|
||||
# Detect model config from state dict
|
||||
print(f"[TEST] Detecting model type...", flush=True)
|
||||
model_config = comfy.model_detection.model_config_from_unet(sd, "")
|
||||
print(f"[TEST] Model config: {type(model_config).__name__}", flush=True)
|
||||
|
||||
# Load into model skeleton
|
||||
print(f"[TEST] Loading into model skeleton...", flush=True)
|
||||
t5 = time.time()
|
||||
model = model_config.get_model(sd, "", device=comfy.model_management.unet_offload_device())
|
||||
model.model_config = model_config
|
||||
print(f"[TEST] Model skeleton in {time.time()-t5:.1f}s", flush=True)
|
||||
|
||||
# Set operations
|
||||
print(f"[TEST] Setting model operations...", flush=True)
|
||||
ops = GGMLOps()
|
||||
model.model.diffusion_model = comfy.ops.load_model_gpu(model.model.diffusion_model, ops.__class__)
|
||||
|
||||
# Load state dict
|
||||
print(f"[TEST] Loading state dict into model...", flush=True)
|
||||
t6 = time.time()
|
||||
model.model.diffusion_model.load_state_dict(sd, strict=False)
|
||||
dt = time.time() - t6
|
||||
print(f"[TEST] State dict loaded in {dt:.1f}s", flush=True)
|
||||
|
||||
print(f"[TEST] Total model load: {time.time()-t4:.1f}s", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[TEST] Error in Step 3: {type(e).__name__}: {e}", flush=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print(f"\n[TEST] COMPLETE in {time.time()-t0:.1f}s total", flush=True)
|
||||
os._exit(0)
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BC-250: Test each ComfyUI component in isolation to find which one hangs.
|
||||
Run from /home/fabian/ComfyUI with venv active.
|
||||
"""
|
||||
import os, sys, time, signal
|
||||
|
||||
os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.1.0"
|
||||
os.environ["HSA_ENABLE_SDMA"] = "0"
|
||||
os.environ["HIP_VISIBLE_DEVICES"] = "0"
|
||||
os.environ["BC250_SOFTMAX_THRESHOLD"] = "512"
|
||||
|
||||
sys.path.insert(0, "/home/fabian/ComfyUI")
|
||||
|
||||
# Timeout handler
|
||||
def timeout_handler(signum, frame):
|
||||
print(f"\n[TIMEOUT] Operation exceeded time limit!", flush=True)
|
||||
os._exit(1)
|
||||
|
||||
print("[T] Importing patch...", flush=True)
|
||||
import bc250_softmax_patch
|
||||
|
||||
print("[T] Importing comfy...", flush=True)
|
||||
t0 = time.time()
|
||||
import torch
|
||||
import comfy.sd
|
||||
import comfy.model_management
|
||||
import comfy.utils
|
||||
import comfy.clip_model
|
||||
import folder_paths
|
||||
print(f"[T] Imports done in {time.time()-t0:.1f}s", flush=True)
|
||||
print(f"[T] CUDA: {torch.cuda.is_available()}, Device: {torch.cuda.get_device_name(0)}", flush=True)
|
||||
|
||||
# === TEST 1: Load CLIP model ===
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
print(f"[T] TEST 1: Load CLIP model (gemma2_2b_lumina2)", flush=True)
|
||||
signal.alarm(60) # 60s timeout
|
||||
t1 = time.time()
|
||||
try:
|
||||
clip_path = os.path.join(folder_paths.get_folder_paths("clip")[0], "gemma2_2b_lumina2.safetensors")
|
||||
if not os.path.exists(clip_path):
|
||||
# Try text_encoders folder
|
||||
for p in folder_paths.get_folder_paths("text_encoders"):
|
||||
cp = os.path.join(p, "gemma2_2b_lumina2.safetensors")
|
||||
if os.path.exists(cp):
|
||||
clip_path = cp
|
||||
break
|
||||
|
||||
print(f"[T] CLIP path: {clip_path}", flush=True)
|
||||
print(f"[T] Loading CLIP...", flush=True)
|
||||
|
||||
clip = comfy.sd.load_clip(
|
||||
ckpt_paths=[clip_path],
|
||||
embedding_directory=None,
|
||||
clip_type=comfy.sd.CLIPType.LUMINA2,
|
||||
)
|
||||
dt = time.time() - t1
|
||||
print(f"[T] CLIP loaded in {dt:.1f}s", flush=True)
|
||||
print(f"[T] CLIP type: {type(clip).__name__}", flush=True)
|
||||
|
||||
# Check GPU memory after CLIP load
|
||||
print(f"[T] GPU VRAM after CLIP load:", flush=True)
|
||||
print(f"[T] allocated: {torch.cuda.memory_allocated()/1e6:.1f} MB", flush=True)
|
||||
print(f"[T] reserved: {torch.cuda.memory_reserved()/1e6:.1f} MB", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[T] TEST 1 ERROR: {e}", flush=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
clip = None
|
||||
|
||||
signal.alarm(0)
|
||||
|
||||
# === TEST 2: Run CLIP text encoding ===
|
||||
if clip is not None:
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
print(f"[T] TEST 2: CLIP text encoding", flush=True)
|
||||
signal.alarm(120) # 120s timeout
|
||||
t2 = time.time()
|
||||
try:
|
||||
print(f"[T] Encoding: 'a cat'...", flush=True)
|
||||
tokens = clip.tokenize({"g": "a cat"})
|
||||
print(f"[T] Tokenized in {time.time()-t2:.3f}s", flush=True)
|
||||
|
||||
t2b = time.time()
|
||||
print(f"[T] Running CLIP encode (this is the suspected hang point)...", flush=True)
|
||||
output = clip.encode_from_tokens_scheduled(tokens)
|
||||
cond, pooled = output[:2]
|
||||
dt = time.time() - t2b
|
||||
print(f"[T] CLIP encoded in {dt:.1f}s", flush=True)
|
||||
print(f"[T] Cond shape: {cond.shape}, dtype: {cond.dtype}", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[T] TEST 2 ERROR: {e}", flush=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
signal.alarm(0)
|
||||
|
||||
# === TEST 3: Load GGUF UNet ===
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
print(f"[T] TEST 3: Load GGUF UNet", flush=True)
|
||||
signal.alarm(60)
|
||||
t3 = time.time()
|
||||
try:
|
||||
# Ensure GGUF patch is applied
|
||||
bc250_softmax_patch._try_patch_gguf()
|
||||
|
||||
# Import GGUF nodes
|
||||
gguf_path = "/home/fabian/ComfyUI/custom_nodes/ComfyUI-GGUF"
|
||||
sys.path.insert(0, gguf_path)
|
||||
|
||||
# Use the loader directly
|
||||
from loader import gguf_sd_loader
|
||||
from ops import GGMLOps
|
||||
|
||||
unet_path = "/home/fabian/ComfyUI/models/unet/z_image_turbo-Q5_K_S.gguf"
|
||||
print(f"[T] Loading GGUF state dict...", flush=True)
|
||||
sd, extra = gguf_sd_loader(unet_path)
|
||||
print(f"[T] State dict: {len(sd)} keys, arch={extra.get('arch_str')}", flush=True)
|
||||
|
||||
# Now load through comfy
|
||||
print(f"[T] Creating diffusion model...", flush=True)
|
||||
ops = GGMLOps()
|
||||
model = comfy.sd.load_diffusion_model_state_dict(
|
||||
sd, model_options={"custom_operations": ops},
|
||||
metadata=extra.get("metadata", {}),
|
||||
)
|
||||
dt = time.time() - t3
|
||||
print(f"[T] UNet loaded in {dt:.1f}s", flush=True)
|
||||
if model is not None:
|
||||
print(f"[T] Model type: {type(model).__name__}", flush=True)
|
||||
else:
|
||||
print(f"[T] WARNING: model is None!", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[T] TEST 3 ERROR: {e}", flush=True)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
signal.alarm(0)
|
||||
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
print(f"[T] ALL TESTS COMPLETE in {time.time()-t0:.1f}s", flush=True)
|
||||
os._exit(0)
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
echo "=== CPU INFO ==="
|
||||
lscpu | grep -E 'Model name|CPU.s.|MHz|Thread|Core|Socket|Boost'
|
||||
echo "=== CPUFREQ GOVERNOR ==="
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "no cpufreq governor"
|
||||
echo "=== CPUFREQ AVAILABLE GOVERNORS ==="
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_governors 2>/dev/null || echo "none"
|
||||
echo "=== CPUFREQ MIN/MAX ==="
|
||||
echo "min: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq 2>/dev/null || echo N/A)"
|
||||
echo "max: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq 2>/dev/null || echo N/A)"
|
||||
echo "cur: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq 2>/dev/null || echo N/A)"
|
||||
echo "cpuinfo_max: $(cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq 2>/dev/null || echo N/A)"
|
||||
echo "=== BOOST ==="
|
||||
cat /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || echo "no boost sysfs"
|
||||
echo "=== MSR BOOST ==="
|
||||
sudo rdmsr 0xC0010015 2>/dev/null || echo "rdmsr not available"
|
||||
echo "=== ACTUAL FREQ PER CORE ==="
|
||||
for i in 0 1 2 3 4 5; do
|
||||
freq=$(cat /sys/devices/system/cpu/cpu${i}/cpufreq/scaling_cur_freq 2>/dev/null || echo "N/A")
|
||||
echo "CPU${i}: ${freq} kHz"
|
||||
done
|
||||
echo "=== AMD PSTATE ==="
|
||||
cat /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || echo "no amd_pstate"
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_driver 2>/dev/null || echo "no scaling_driver"
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
cd /home/fabian/Terminator
|
||||
set +e
|
||||
bash -x ./TERMINATOR.sh train-tts > /tmp/term_debug.log 2>&1
|
||||
echo "EXIT=$?"
|
||||
echo "--- LAST 50 LINES OF TRACE ---"
|
||||
tail -50 /tmp/term_debug.log
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
# Deep process diagnosis
|
||||
PID=$(pgrep -f 'python main.py' | head -1)
|
||||
if [ -z "$PID" ]; then
|
||||
echo "NO PROCESS FOUND"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Process $PID ==="
|
||||
ps -p $PID -o pid,pcpu,pmem,vsz,rss,state --no-header
|
||||
|
||||
echo ""
|
||||
echo "=== Process state ==="
|
||||
cat /proc/$PID/status | grep -E 'State|Threads|VmRSS|VmSwap|voluntary|nonvoluntary'
|
||||
|
||||
echo ""
|
||||
echo "=== Waiting on ==="
|
||||
cat /proc/$PID/wchan 2>/dev/null
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo "=== Thread CPU usage ==="
|
||||
ps -p $PID -L -o tid,pcpu,state --no-header | sort -k2 -rn | head -10
|
||||
|
||||
echo ""
|
||||
echo "=== Strace (2 sec) ==="
|
||||
timeout 2 strace -p $PID -c 2>&1 | head -30
|
||||
|
||||
echo ""
|
||||
echo "=== Log file check ==="
|
||||
wc -c /home/fabian/comfyui7.log
|
||||
ls -la /home/fabian/comfyui7.log
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
# Deep diagnostic: strace + py-spy + /proc analysis
|
||||
pid=$(pgrep -f "python.*main.py" | head -1)
|
||||
if [ -z "$pid" ]; then
|
||||
echo "NO PROCESS"
|
||||
exit 1
|
||||
fi
|
||||
echo "PID: $pid"
|
||||
|
||||
# What files does it have open? (compiler artifacts?)
|
||||
echo "=== Open files (interesting ones) ==="
|
||||
ls -la /proc/$pid/fd 2>/dev/null | wc -l
|
||||
readlink /proc/$pid/fd/* 2>/dev/null | grep -iE "comgr|miopen|\.co|\.hsaco|\.hip|tmp|cache|rocm" | head -20
|
||||
|
||||
echo ""
|
||||
echo "=== Check comgr cache ==="
|
||||
find /home/fabian/.cache/ -name "*.co" -o -name "*.hsaco" 2>/dev/null | head -20
|
||||
du -sh /home/fabian/.cache/comgr/ 2>/dev/null
|
||||
du -sh /home/fabian/.cache/miopen/ 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== strace snapshot (2 seconds, top syscalls) ==="
|
||||
timeout 3 strace -p $pid -c 2>&1 | tail -25
|
||||
|
||||
echo ""
|
||||
echo "=== py-spy dump ==="
|
||||
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope > /dev/null 2>&1
|
||||
py-spy dump --pid $pid 2>&1 | head -50
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
echo "=== FULL LOG (without clip missing) ==="
|
||||
grep -v "clip missing" /home/fabian/comfyui5.log
|
||||
echo "=== OUTPUT DIR ==="
|
||||
ls -la /home/fabian/ComfyUI/output/ 2>/dev/null
|
||||
echo "=== THREAD STATES ==="
|
||||
for tid in $(ls /proc/15070/task/ 2>/dev/null); do
|
||||
wchan=$(cat /proc/15070/task/$tid/wchan 2>/dev/null)
|
||||
if [ -n "$wchan" ] && [ "$wchan" != "0" ]; then
|
||||
echo "$tid: $wchan"
|
||||
fi
|
||||
done
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
echo "=== CPU TIME CHECK ==="
|
||||
cat /proc/15070/stat 2>/dev/null | awk '{print "utime="$14, "stime="$15, "threads="$20, "vsize_mb="int($23/1024/1024)}'
|
||||
sleep 3
|
||||
cat /proc/15070/stat 2>/dev/null | awk '{print "utime="$14, "stime="$15, "threads="$20, "vsize_mb="int($23/1024/1024)}'
|
||||
|
||||
echo "=== LOG SIZE CHECK ==="
|
||||
wc -c < /home/fabian/comfyui5.log
|
||||
sleep 3
|
||||
wc -c < /home/fabian/comfyui5.log
|
||||
|
||||
echo "=== COMGR CACHE ==="
|
||||
du -sh /home/fabian/.cache/comgr/ 2>/dev/null
|
||||
ls /home/fabian/.cache/comgr/ 2>/dev/null | wc -l
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
echo "=== T=0 ==="
|
||||
cat /proc/15070/stat 2>/dev/null | awk '{print "utime="$14, "stime="$15}'
|
||||
wc -c < /home/fabian/comfyui5.log
|
||||
du -sh /home/fabian/.cache/comgr/ 2>/dev/null
|
||||
ls /home/fabian/.cache/comgr/ 2>/dev/null | wc -l
|
||||
|
||||
sleep 30
|
||||
|
||||
echo "=== T=30 ==="
|
||||
cat /proc/15070/stat 2>/dev/null | awk '{print "utime="$14, "stime="$15}'
|
||||
wc -c < /home/fabian/comfyui5.log
|
||||
du -sh /home/fabian/.cache/comgr/ 2>/dev/null
|
||||
ls /home/fabian/.cache/comgr/ 2>/dev/null | wc -l
|
||||
|
||||
echo "=== NEW LOG LINES ==="
|
||||
grep -v "clip missing" /home/fabian/comfyui5.log | tail -10
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
echo "=== FD CHECK (open files) ==="
|
||||
ls -la /proc/15070/fd/ 2>/dev/null | grep -E "gguf|model|unet" | head -10
|
||||
|
||||
echo "=== IO COUNTERS ==="
|
||||
cat /proc/15070/io 2>/dev/null
|
||||
sleep 5
|
||||
echo "=== IO COUNTERS AFTER 5s ==="
|
||||
cat /proc/15070/io 2>/dev/null
|
||||
|
||||
echo "=== MEMORY ==="
|
||||
cat /proc/15070/status 2>/dev/null | grep -E "VmRSS|VmSize|Threads"
|
||||
|
||||
echo "=== GPU MEMORY (sysfs) ==="
|
||||
cat /sys/class/drm/card0/device/mem_info_vram_used 2>/dev/null
|
||||
cat /sys/class/drm/card0/device/mem_info_gtt_used 2>/dev/null
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
# Diagnose → Kill → Fix
|
||||
set -e
|
||||
|
||||
echo "=== DIAG ==="
|
||||
PID=$(pgrep -f "python main.py" || true)
|
||||
if [ -n "$PID" ]; then
|
||||
echo "PID: $PID"
|
||||
cat /proc/$PID/status 2>/dev/null | grep -E 'State|Threads|VmRSS' || true
|
||||
cat /proc/$PID/wchan 2>/dev/null; echo
|
||||
# Try py-spy but don't block
|
||||
timeout 5 /home/fabian/ComfyUI/venv/bin/py-spy dump --pid $PID 2>/dev/null | grep -E 'active|rope|sdpa|matmul|softmax|sample' | head -10 || echo "py-spy failed/timeout"
|
||||
echo "=== KILLING ==="
|
||||
kill -9 $PID 2>/dev/null || true
|
||||
sleep 2
|
||||
else
|
||||
echo "No python process found"
|
||||
fi
|
||||
|
||||
echo "=== LOG TAIL ==="
|
||||
tail -10 /home/fabian/comfyui.log
|
||||
|
||||
echo "=== MEMORY ==="
|
||||
free -h | head -2
|
||||
echo 3 > /proc/sys/vm/drop_caches
|
||||
free -h | head -2
|
||||
|
||||
echo "=== DONE ==="
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
PID=$(pgrep -f 'main.py')
|
||||
echo "PID: $PID"
|
||||
|
||||
echo "=== SUDO STRACE 3s ==="
|
||||
sudo timeout 3 strace -p $PID -e trace=write,read,ioctl,futex -c 2>&1
|
||||
|
||||
echo "=== THREAD STACKS (py-spy) ==="
|
||||
sudo py-spy dump --pid $PID 2>/dev/null || echo "py-spy not available"
|
||||
|
||||
echo "=== LOG TAIL ==="
|
||||
tail -5 /home/fabian/comfyui.log
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
PID=$(pgrep -f main.py | head -n 1)
|
||||
echo "PID=$PID"
|
||||
timeout 10 sudo /home/fabian/ComfyUI/venv/bin/py-spy dump --pid $PID 2>&1 | head -n 80
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
PID=147178
|
||||
echo "=== Thread wait channels ==="
|
||||
for tid_dir in /proc/$PID/task/*/; do
|
||||
tid=$(basename "$tid_dir")
|
||||
wc=$(cat "$tid_dir/wchan" 2>/dev/null)
|
||||
echo "$wc"
|
||||
done | sort | uniq -c | sort -rn | head -20
|
||||
|
||||
echo ""
|
||||
echo "=== Log tail ==="
|
||||
tail -5 /home/fabian/comfyui3.log
|
||||
|
||||
echo ""
|
||||
echo "=== GPU power ==="
|
||||
cat /sys/class/drm/card0/device/hwmon/hwmon*/power1_average 2>/dev/null || echo "no power info"
|
||||
|
||||
echo ""
|
||||
echo "=== Memory ==="
|
||||
free -h | head -3
|
||||
|
||||
echo ""
|
||||
echo "=== comgr cache ==="
|
||||
ls /home/fabian/.cache/comgr/ 2>/dev/null | wc -l
|
||||
echo "DONE"
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Find the correct Qwen3-4B text encoder for Z-Image-Turbo
|
||||
|
||||
cd /home/fabian/ComfyUI
|
||||
|
||||
# Check HuggingFace cache for previous downloads
|
||||
echo "=== HF download logs ==="
|
||||
ls -la models/unet/.cache/huggingface/ 2>/dev/null
|
||||
cat models/unet/.cache/huggingface/download/*.json 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "=== Check sd-models text_encoders ==="
|
||||
ls -la /home/fabian/sd-models/text_encoders/ 2>/dev/null
|
||||
find /home/fabian/sd-models -name '*.safetensors' 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== Try to find the model via huggingface_hub ==="
|
||||
/home/fabian/ComfyUI/venv/bin/python3 << 'PYEOF'
|
||||
from huggingface_hub import HfApi, list_repo_files
|
||||
api = HfApi()
|
||||
|
||||
# Check common repos for z-image text encoders
|
||||
repos_to_check = [
|
||||
"Comfy-Org/z_image_text_encoders",
|
||||
"city96/z-image-turbo-GGUF",
|
||||
"THUDM/z-image-turbo",
|
||||
"Comfy-Org/lumina2_text_encoders",
|
||||
]
|
||||
|
||||
for repo in repos_to_check:
|
||||
try:
|
||||
files = list_repo_files(repo)
|
||||
print(f"\n{repo}:")
|
||||
for f in files:
|
||||
print(f" {f}")
|
||||
except Exception as e:
|
||||
print(f"\n{repo}: {str(e)[:80]}")
|
||||
|
||||
# Also search for z_image text encoder repos
|
||||
try:
|
||||
results = api.list_models(search="z_image text_encoder", limit=5)
|
||||
print("\n=== Search: z_image text_encoder ===")
|
||||
for m in results:
|
||||
print(f" {m.modelId}: {m.tags[:3] if m.tags else 'no tags'}")
|
||||
except Exception as e:
|
||||
print(f"Search failed: {e}")
|
||||
PYEOF
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
pid=$(pgrep -f "python.*main.py" | head -1)
|
||||
echo "PID: $pid"
|
||||
|
||||
# Find py-spy
|
||||
echo "=== Finding py-spy ==="
|
||||
find /home/fabian/ComfyUI/venv -name "py-spy" 2>/dev/null
|
||||
which py-spy 2>/dev/null
|
||||
find /home/fabian -name "py-spy" -type f 2>/dev/null | head -3
|
||||
|
||||
# Find strace
|
||||
echo "=== Finding strace ==="
|
||||
which strace 2>/dev/null
|
||||
pacman -Ql strace 2>/dev/null | grep bin | head -3
|
||||
|
||||
# Open files of the process
|
||||
echo "=== Open files (all readlink) ==="
|
||||
readlink /proc/$pid/fd/* 2>/dev/null | head -30
|
||||
|
||||
echo ""
|
||||
echo "=== /proc/pid/wchan (what syscall is it in?) ==="
|
||||
cat /proc/$pid/wchan 2>/dev/null
|
||||
echo ""
|
||||
|
||||
echo "=== /proc/pid/stack (kernel stack) ==="
|
||||
sudo cat /proc/$pid/stack 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "=== Thread status ==="
|
||||
ls /proc/$pid/task/ 2>/dev/null | head -5
|
||||
echo "..."
|
||||
ls /proc/$pid/task/ 2>/dev/null | wc -l
|
||||
echo "total threads"
|
||||
|
||||
# Check specific thread that's stuck (prompt_worker)
|
||||
echo ""
|
||||
echo "=== Thread wchan (first 10) ==="
|
||||
for tid in $(ls /proc/$pid/task/ | head -10); do
|
||||
wchan=$(cat /proc/$pid/task/$tid/wchan 2>/dev/null)
|
||||
stat=$(cat /proc/$pid/task/$tid/stat 2>/dev/null | awk '{print $3}')
|
||||
echo " TID $tid: $wchan (state: $stat)"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== comgr cache modify times ==="
|
||||
ls -lt /home/fabian/.cache/comgr/ 2>/dev/null | head -10
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Find which inference/ files import torch and sounddevice."""
|
||||
import subprocess, sys
|
||||
|
||||
cmd = """
|
||||
grep -rn 'import torch\\|import sounddevice' /opt/qwen3-tts/qwen3_tts_gguf/inference/ 2>/dev/null | grep -v __pycache__
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "--entrypoint", "bash", "sudx/qwen3-tts:latest", "-c", cmd],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
print(result.stdout)
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
grep -r "webui" /home/fabian/sd-restapi/src/ --include="*.cpp" --include="*.h" 2>/dev/null | grep -iE "dir|path|register|WEBUI" | head -30
|
||||
echo "---"
|
||||
grep -r "SDCPP_WEBUI" /home/fabian/sd-restapi/CMakeLists.txt /home/fabian/sd-restapi/src/ 2>/dev/null | head -10
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
echo "=== workflow/default files ==="
|
||||
find /home/fabian/ComfyUI -maxdepth 3 \( -name "*workflow*" -o -name "*default*" \) 2>/dev/null | grep -v __pycache__ | grep -v node_modules | grep -v ".pyc" | head -30
|
||||
|
||||
echo "=== user dir ==="
|
||||
ls -la /home/fabian/ComfyUI/user/ 2>/dev/null
|
||||
|
||||
echo "=== user files ==="
|
||||
find /home/fabian/ComfyUI/user -type f 2>/dev/null | head -30
|
||||
|
||||
echo "=== web json files ==="
|
||||
find /home/fabian/ComfyUI/web -maxdepth 3 -type f -name "*.json" 2>/dev/null | head -10
|
||||
|
||||
echo "=== comfy settings ==="
|
||||
cat /home/fabian/ComfyUI/user/default/comfy.settings.json 2>/dev/null | head -30
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Find the busy thread (the one eating CPU)
|
||||
echo "=== BUSY THREADS ==="
|
||||
for tid in $(ls /proc/15070/task/); do
|
||||
utime1=$(cat /proc/15070/task/$tid/stat 2>/dev/null | awk '{print $14}')
|
||||
sleep 1
|
||||
utime2=$(cat /proc/15070/task/$tid/stat 2>/dev/null | awk '{print $14}')
|
||||
if [ -n "$utime1" ] && [ -n "$utime2" ]; then
|
||||
diff=$((utime2 - utime1))
|
||||
if [ "$diff" -gt 50 ]; then
|
||||
echo "Thread $tid: delta_utime=$diff (BUSY)"
|
||||
wchan=$(cat /proc/15070/task/$tid/wchan 2>/dev/null)
|
||||
echo " wchan: $wchan"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "=== STRACE SAMPLE (2s) ==="
|
||||
sudo timeout 2 strace -p 15070 -e trace=write,read,openat -c 2>&1 | head -30
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
# Set memlock limit for fabian user (needed for mlockall in BC-250 patch)
|
||||
echo "fabian soft memlock unlimited" >> /etc/security/limits.conf
|
||||
echo "fabian hard memlock unlimited" >> /etc/security/limits.conf
|
||||
echo "Done. Log out and back in, or reboot for limits to take effect."
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Fix SDPA patch for BC-250 gfx1010: always use manual SDPA on CUDA"""
|
||||
import sys
|
||||
|
||||
path = '/home/fabian/ComfyUI/bc250_softmax_patch.py'
|
||||
with open(path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Fix 1: patched_sdpa - ALWAYS use manual SDPA on CUDA (no threshold)
|
||||
old_sdpa = '''def patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
S = key.size(-2)
|
||||
if query.is_cuda and S > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_sdpa, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual SDPA: Q={list(query.shape)}, S={S}")
|
||||
patched_sdpa._logged = True
|
||||
return _safe_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
return _original_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)'''
|
||||
|
||||
new_sdpa = '''def patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
# gfx1010: ALWAYS use manual SDPA on CUDA — built-in math backend kernel hangs
|
||||
if query.is_cuda:
|
||||
if not getattr(patched_sdpa, '_logged', False):
|
||||
S = key.size(-2)
|
||||
logger.warning(f"[BC-250] Manual SDPA (ALWAYS): Q={list(query.shape)}, S={S}")
|
||||
patched_sdpa._logged = True
|
||||
return _safe_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)
|
||||
return _original_sdpa(query, key, value, attn_mask=attn_mask,
|
||||
dropout_p=dropout_p, is_causal=is_causal, scale=scale)'''
|
||||
|
||||
if old_sdpa not in content:
|
||||
print("ERROR: patched_sdpa not found!")
|
||||
sys.exit(1)
|
||||
content = content.replace(old_sdpa, new_sdpa)
|
||||
print("OK: patched_sdpa → always manual on CUDA")
|
||||
|
||||
# Fix 2: _safe_sdpa - add sync after GPU matmul+softmax operations
|
||||
old_safe = '''def _safe_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
L, S = query.size(-2), key.size(-2)
|
||||
if scale is None:
|
||||
scale = query.size(-1) ** -0.5
|
||||
attn_weight = torch.matmul(query, key.transpose(-2, -1)) * scale
|
||||
if is_causal:
|
||||
causal_mask = torch.triu(torch.ones(L, S, dtype=torch.bool, device=query.device), diagonal=1)
|
||||
attn_weight = attn_weight.masked_fill(causal_mask, float('-inf'))
|
||||
if attn_mask is not None:
|
||||
if attn_mask.dtype == torch.bool:
|
||||
attn_weight = attn_weight.masked_fill(~attn_mask, float('-inf'))
|
||||
else:
|
||||
attn_weight = attn_weight + attn_mask
|
||||
attn_weight = _safe_softmax_impl(attn_weight, dim=-1)
|
||||
if dropout_p > 0.0:
|
||||
attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p)
|
||||
return torch.matmul(attn_weight, value)'''
|
||||
|
||||
new_safe = '''def _safe_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None):
|
||||
L, S = query.size(-2), key.size(-2)
|
||||
if scale is None:
|
||||
scale = query.size(-1) ** -0.5
|
||||
# gfx1010: sync between GPU ops to prevent kernel queue buildup + hang
|
||||
attn_weight = torch.matmul(query, key.transpose(-2, -1)) * scale
|
||||
if query.is_cuda:
|
||||
torch.cuda.synchronize()
|
||||
if is_causal:
|
||||
causal_mask = torch.triu(torch.ones(L, S, dtype=torch.bool, device=query.device), diagonal=1)
|
||||
attn_weight = attn_weight.masked_fill(causal_mask, float('-inf'))
|
||||
if attn_mask is not None:
|
||||
if attn_mask.dtype == torch.bool:
|
||||
attn_weight = attn_weight.masked_fill(~attn_mask, float('-inf'))
|
||||
else:
|
||||
attn_weight = attn_weight + attn_mask
|
||||
attn_weight = _safe_softmax_impl(attn_weight, dim=-1)
|
||||
if query.is_cuda:
|
||||
torch.cuda.synchronize()
|
||||
if dropout_p > 0.0:
|
||||
attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p)
|
||||
output = torch.matmul(attn_weight, value)
|
||||
if query.is_cuda:
|
||||
torch.cuda.synchronize()
|
||||
return output'''
|
||||
|
||||
if old_safe not in content:
|
||||
print("ERROR: _safe_sdpa not found!")
|
||||
sys.exit(1)
|
||||
content = content.replace(old_safe, new_safe)
|
||||
print("OK: _safe_sdpa → sync after GPU ops")
|
||||
|
||||
# Fix 3: Also patch softmax to ALWAYS use manual on CUDA (same reason)
|
||||
old_softmax = '''def patched_softmax(input, dim=None, _stacklevel=3, dtype=None):
|
||||
if dim is None:
|
||||
dim = -1
|
||||
if dtype is not None:
|
||||
input = input.to(dtype)
|
||||
if input.is_cuda and input.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual F.softmax: shape={list(input.shape)}, dim={dim}")
|
||||
patched_softmax._logged = True
|
||||
return _safe_softmax_impl(input, dim)
|
||||
return _original_softmax(input, dim=dim)'''
|
||||
|
||||
new_softmax = '''def patched_softmax(input, dim=None, _stacklevel=3, dtype=None):
|
||||
if dim is None:
|
||||
dim = -1
|
||||
if dtype is not None:
|
||||
input = input.to(dtype)
|
||||
# gfx1010: ALWAYS use manual softmax on CUDA — native kernel unreliable
|
||||
if input.is_cuda:
|
||||
if not getattr(patched_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual F.softmax (ALWAYS): shape={list(input.shape)}, dim={dim}")
|
||||
patched_softmax._logged = True
|
||||
return _safe_softmax_impl(input, dim)
|
||||
return _original_softmax(input, dim=dim)'''
|
||||
|
||||
if old_softmax not in content:
|
||||
print("WARNING: patched_softmax not found (may already be fixed)")
|
||||
else:
|
||||
content = content.replace(old_softmax, new_softmax)
|
||||
print("OK: patched_softmax → always manual on CUDA")
|
||||
|
||||
# Fix 4: Same for tensor.softmax
|
||||
old_tsm = '''def patched_tensor_softmax(self, dim=-1, dtype=None):
|
||||
if dtype is not None:
|
||||
self = self.to(dtype)
|
||||
if self.is_cuda and self.shape[dim] > SAFE_SOFTMAX_THRESHOLD:
|
||||
if not getattr(patched_tensor_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual softmax: shape={list(self.shape)}, dim={dim}")
|
||||
patched_tensor_softmax._logged = True
|
||||
return _safe_softmax_impl(self, dim)
|
||||
return _original_tensor_softmax(self, dim=dim)'''
|
||||
|
||||
new_tsm = '''def patched_tensor_softmax(self, dim=-1, dtype=None):
|
||||
if dtype is not None:
|
||||
self = self.to(dtype)
|
||||
# gfx1010: ALWAYS use manual softmax on CUDA
|
||||
if self.is_cuda:
|
||||
if not getattr(patched_tensor_softmax, '_logged', False):
|
||||
logger.warning(f"[BC-250] Manual softmax (ALWAYS): shape={list(self.shape)}, dim={dim}")
|
||||
patched_tensor_softmax._logged = True
|
||||
return _safe_softmax_impl(self, dim)
|
||||
return _original_tensor_softmax(self, dim=dim)'''
|
||||
|
||||
if old_tsm not in content:
|
||||
print("WARNING: patched_tensor_softmax not found (may already be fixed)")
|
||||
else:
|
||||
content = content.replace(old_tsm, new_tsm)
|
||||
print("OK: patched_tensor_softmax → always manual on CUDA")
|
||||
|
||||
# Fix 5: Update version string
|
||||
content = content.replace('v17 ready', 'v19 ready — ALL CUDA ops manual (no native kernels)')
|
||||
content = content.replace('Comprehensive Monkey-Patch v17', 'Comprehensive Monkey-Patch v19')
|
||||
|
||||
with open(path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("\nALL PATCHES APPLIED — v19")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user