Upgrade RAG stack; fix chunk_size tokenizer mismatch

clippings_search/build_clippings.py

@@ -60,11 +60,32 @@ SUPPORTED_EXTS = READER_EXTS | CUSTOM_EXTS
 MIN_TEXT_LENGTH = 100
 
 
+_EMBED_TOKENIZER = None
+
+
+def get_embed_tokenizer():
+    """The embedding model's own tokenizer. See build_store.py for why.
+
+    Without this, SentenceSplitter counts in tiktoken while the embedder counts
+    in BERT WordPiece; the two diverge on unusual text and chunks can exceed
+    bge-large's 512-token limit, where their tails are silently truncated.
+    """
+    global _EMBED_TOKENIZER
+    if _EMBED_TOKENIZER is None:
+        from transformers import AutoTokenizer
+        _EMBED_TOKENIZER = AutoTokenizer.from_pretrained(
+            EMBED_MODEL_NAME, cache_dir="./models", local_files_only=True
+        )
+    return _EMBED_TOKENIZER
+
+
 def get_text_splitter():
+    tok = get_embed_tokenizer()
     return SentenceSplitter(
         chunk_size=CHUNK_SIZE,
         chunk_overlap=CHUNK_OVERLAP,
         paragraph_separator="\n\n",
+        tokenizer=lambda text: tok.encode(text, add_special_tokens=False),
     )