Fabian
37 lines
1.1 KiB
Bash
37 lines
1.1 KiB
Bash
#!/bin/bash
|
|
# Check for tokenizer files and safetensors metadata
|
|
|
|
echo "=== Tokenizer files ==="
|
|
find /home/fabian/ComfyUI -name '*spiece*' -o -name '*sentencepiece*' -o -name '*.model' 2>/dev/null | head -20
|
|
|
|
echo ""
|
|
echo "=== text_encoders directory ==="
|
|
ls /home/fabian/ComfyUI/comfy/text_encoders/
|
|
|
|
echo ""
|
|
echo "=== safetensors metadata ==="
|
|
cd /home/fabian/ComfyUI
|
|
/home/fabian/ComfyUI/venv/bin/python3 -c "
|
|
import safetensors
|
|
f = safetensors.safe_open('models/text_encoders/gemma2_2b_lumina2.safetensors', framework='pt')
|
|
md = f.metadata()
|
|
if md:
|
|
print('metadata keys:', list(md.keys())[:30])
|
|
for k in md:
|
|
v = md[k]
|
|
if len(v) > 200:
|
|
print(f' {k}: len={len(v)} (first 100 chars: {v[:100]}...) ')
|
|
else:
|
|
print(f' {k}: {v}')
|
|
else:
|
|
print('No metadata found')
|
|
"
|
|
|
|
echo ""
|
|
echo "=== SPieceTokenizer source ==="
|
|
head -40 /home/fabian/ComfyUI/comfy/text_encoders/spiece_tokenizer.py
|
|
|
|
echo ""
|
|
echo "=== Gemma2 tokenizer lookup in SDTokenizer ==="
|
|
grep -n 'tokenizer_path\|tokenizer_data\|spiece\|from_pretrained' /home/fabian/ComfyUI/comfy/sd1_clip.py | head -20
|