Upgrade RAG stack; fix chunk_size tokenizer mismatch

build_store.py

@@ -9,6 +9,14 @@
 # E. M. Furst
 # Used Sonnet 4.5 to suggest changes; Opus 4.6 for incremental update
 
+# Environment vars must be set before importing huggingface/transformers
+# libraries, because huggingface_hub.constants evaluates HF_HUB_OFFLINE
+# at import time. Matches the query scripts.
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+os.environ["HF_HUB_CACHE"] = "./models"
+os.environ["HF_HUB_OFFLINE"] = "1"
+
 from llama_index.core import (
     Document,
     SimpleDirectoryReader,
@@ -23,7 +31,6 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.core.node_parser import SentenceSplitter
 import argparse
 import datetime
-import os
 import re
 import time
 
@@ -101,11 +108,36 @@ def load_documents(input_files=None) -> list[Document]:
     return apply_metadata_policy(reader.load_data())
 
 
+_EMBED_TOKENIZER = None
+
+
+def get_embed_tokenizer():
+    """The embedding model's own tokenizer, loaded once.
+
+    SentenceSplitter otherwise counts with tiktoken (cl100k_base) while the
+    embedder counts in BERT WordPiece. They agree for ordinary prose (median
+    ratio 1.01x) but diverge badly on unusual text -- up to 2.96x here. That
+    let 94 chunks exceed CHUNK_SIZE and pushed 4 past bge-large's 512-token
+    max_seq_length, where their tails were silently truncated at embed time and
+    never made it into the vector. Counting in the embedder's own units makes
+    chunk_size mean what it says.
+    """
+    global _EMBED_TOKENIZER
+    if _EMBED_TOKENIZER is None:
+        from transformers import AutoTokenizer
+        _EMBED_TOKENIZER = AutoTokenizer.from_pretrained(
+            EMBED_MODEL_NAME, cache_dir="./models", local_files_only=True
+        )
+    return _EMBED_TOKENIZER
+
+
 def get_text_splitter():
+    tok = get_embed_tokenizer()
     return SentenceSplitter(
         chunk_size=CHUNK_SIZE,
         chunk_overlap=CHUNK_OVERLAP,
         paragraph_separator="\n\n",
+        tokenizer=lambda text: tok.encode(text, add_special_tokens=False),
     )