Uploaded sanitized BC250/ROCm Repository.

Scripts and Tests/check_tokenizer.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Check for tokenizer files and safetensors metadata
+
+echo "=== Tokenizer files ==="
+find /home/fabian/ComfyUI -name '*spiece*' -o -name '*sentencepiece*' -o -name '*.model' 2>/dev/null | head -20
+
+echo ""
+echo "=== text_encoders directory ==="
+ls /home/fabian/ComfyUI/comfy/text_encoders/
+
+echo ""
+echo "=== safetensors metadata ==="
+cd /home/fabian/ComfyUI
+/home/fabian/ComfyUI/venv/bin/python3 -c "
+import safetensors
+f = safetensors.safe_open('models/text_encoders/gemma2_2b_lumina2.safetensors', framework='pt')
+md = f.metadata()
+if md:
+    print('metadata keys:', list(md.keys())[:30])
+    for k in md:
+        v = md[k]
+        if len(v) > 200:
+            print(f'  {k}: len={len(v)} (first 100 chars: {v[:100]}...) ')
+        else:
+            print(f'  {k}: {v}')
+else:
+    print('No metadata found')
+"
+
+echo ""
+echo "=== SPieceTokenizer source ==="
+head -40 /home/fabian/ComfyUI/comfy/text_encoders/spiece_tokenizer.py
+
+echo ""
+echo "=== Gemma2 tokenizer lookup in SDTokenizer ==="
+grep -n 'tokenizer_path\|tokenizer_data\|spiece\|from_pretrained' /home/fabian/ComfyUI/comfy/sd1_clip.py | head -20