Upgrade RAG stack; fix chunk_size tokenizer mismatch

pyproject.toml

@@ -9,21 +9,25 @@ authors = [{ name = "E. M. Furst" }]
 
 dependencies = [
     # --- RAG framework (version-sensitive: pinned exactly) ---
-    "llama-index-core==0.14.14",
-    "llama-index-embeddings-huggingface==0.6.1",
-    "llama-index-readers-file==0.5.6",
-    "llama-index-llms-ollama==0.9.1",
-    "llama-index-llms-openai==0.6.18",
-    "llama-index-retrievers-bm25==0.6.5",
-    "llama-index-vector-stores-chroma==0.5.5",
+    "llama-index-core==0.14.24",
+    "llama-index-embeddings-huggingface==0.8.0",
+    "llama-index-readers-file==0.7.0",
+    "llama-index-llms-ollama==0.11.0",
+    "llama-index-llms-openai==0.8.1",
+    "llama-index-retrievers-bm25==0.8.0",
+    "llama-index-vector-stores-chroma==0.6.0",
     # --- Vector store (clippings) ---
-    "chromadb==1.5.1",
+    "chromadb==1.5.9",
     # --- Embeddings / cross-encoder re-ranking (pulls torch) ---
-    "sentence-transformers>=5.1.0",
-    # transformers + huggingface-hub pinned: a transformers update once broke
-    # offline model loading (AutoTokenizer phoning home). See devlog 2026-02-20.
-    "transformers==4.55.0",
-    "huggingface-hub==0.34.4",
+    # sentence-transformers 6.x requires transformers>=5 and huggingface-hub>=1.3.
+    # These were previously pinned at transformers==4.55.0 / hub==0.34.4 after an
+    # update broke offline model loading (AutoTokenizer phoning home; devlog
+    # 2026-02-20). That pin had started holding sentence-transformers at 5.5.1,
+    # so it was lifted 2026-09-20 and the offline path re-verified.
+    # Upper bounds only -- majors are the ones that bite.
+    "sentence-transformers>=6.1.0,<7.0.0",
+    "transformers>=5.17.0,<6.0.0",
+    "huggingface-hub>=1.32.0,<2.0.0",
     # --- Keyword search / document loaders ---
     "nltk>=3.9.1",
     # --- eval harness (eval/, see EVAL_PLAN.md) ---