#!/bin/bash
# Check for tokenizer files and safetensors metadata

echo "=== Tokenizer files ==="
find /home/fabian/ComfyUI -name '*spiece*' -o -name '*sentencepiece*' -o -name '*.model' 2>/dev/null | head -20

echo ""
echo "=== text_encoders directory ==="
ls /home/fabian/ComfyUI/comfy/text_encoders/

echo ""
echo "=== safetensors metadata ==="
cd /home/fabian/ComfyUI
/home/fabian/ComfyUI/venv/bin/python3 -c "
import safetensors
f = safetensors.safe_open('models/text_encoders/gemma2_2b_lumina2.safetensors', framework='pt')
md = f.metadata()
if md:
    print('metadata keys:', list(md.keys())[:30])
    for k in md:
        v = md[k]
        if len(v) > 200:
            print(f'  {k}: len={len(v)} (first 100 chars: {v[:100]}...) ')
        else:
            print(f'  {k}: {v}')
else:
    print('No metadata found')
"

echo ""
echo "=== SPieceTokenizer source ==="
head -40 /home/fabian/ComfyUI/comfy/text_encoders/spiece_tokenizer.py

echo ""
echo "=== Gemma2 tokenizer lookup in SDTokenizer ==="
grep -n 'tokenizer_path\|tokenizer_data\|spiece\|from_pretrained' /home/fabian/ComfyUI/comfy/sd1_clip.py | head -20