Add bge-reranker-v2-m3; fix metadata and chunking bugs

build_store.py

@@ -10,6 +10,7 @@
 # Used Sonnet 4.5 to suggest changes; Opus 4.6 for incremental update
 
 from llama_index.core import (
+    Document,
     SimpleDirectoryReader,
     StorageContext,
     VectorStoreIndex,
@@ -17,11 +18,13 @@ from llama_index.core import (
     Settings,
 )
 from pathlib import Path
+from llama_index.core.readers.file.base import default_file_metadata_func
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.core.node_parser import SentenceSplitter
 import argparse
 import datetime
 import os
+import re
 import time
 
 # Shared constants
@@ -31,6 +34,72 @@ EMBED_MODEL_NAME = "BAAI/bge-large-en-v1.5"
 CHUNK_SIZE = 256
 CHUNK_OVERLAP = 25
 
+# Journal files are named YYYY-MM-DD.txt; the entry date is the filename.
+# Filesystem timestamps are NOT the entry date (the 2000-07-29 entry carries
+# creation_date 2001-05-25), so the date has to be parsed from the name.
+ENTRY_DATE_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})\.txt$")
+
+# --- Metadata policy -------------------------------------------------------
+#
+# Every key below stays in the node's metadata dict and comes back on every
+# retrieval result, whatever these lists say. The lists only control which
+# keys get concatenated into the *text* that the embedder and the LLM see.
+#
+# Previously nothing was excluded from the embed text, so all 5210 chunks were
+# embedded with a 36-token absolute path glued to the front -- 14% of a
+# 256-token chunk, and the same constant string in every vector, which pulled
+# the mean pairwise cosine up from 0.561 to 0.627 and made similarity cutoffs
+# useless.
+#
+# Note BM25Retriever also indexes MetadataMode.EMBED, so the embed text is the
+# only place it can see dates. That is why entry_date stays in: it keeps BM25
+# able to match "2022-08-14" or "2022" at ~8 tokens instead of 36.
+EXCLUDED_EMBED_KEYS = [
+    "file_path", "file_name", "file_type", "file_size",
+    "creation_date", "last_modified_date", "last_accessed_date",
+    "year", "month",
+]
+# The LLM gets file_name instead -- it is the same date, and the synthesis
+# prompt asks the model to cite file names.
+EXCLUDED_LLM_KEYS = [
+    "file_path", "file_type", "file_size",
+    "creation_date", "last_modified_date", "last_accessed_date",
+    "entry_date", "year", "month",
+]
+
+
+def journal_file_metadata(file_path: str) -> dict:
+    """Default file metadata, plus the entry date parsed from the filename."""
+    meta = default_file_metadata_func(file_path)
+    m = ENTRY_DATE_RE.match(Path(file_path).name)
+    if m:
+        year, month, day = m.groups()
+        meta["entry_date"] = f"{year}-{month}-{day}"
+        meta["year"] = int(year)
+        meta["month"] = int(month)
+    return meta
+
+
+def apply_metadata_policy(documents: list[Document]) -> list[Document]:
+    """Set embed/LLM metadata exclusions. Nodes inherit these from the parent."""
+    for doc in documents:
+        doc.excluded_embed_metadata_keys = list(EXCLUDED_EMBED_KEYS)
+        doc.excluded_llm_metadata_keys = list(EXCLUDED_LLM_KEYS)
+    return documents
+
+
+def load_documents(input_files=None) -> list[Document]:
+    """Load journal documents with the shared metadata policy applied."""
+    if input_files is not None:
+        reader = SimpleDirectoryReader(
+            input_files=input_files, file_metadata=journal_file_metadata
+        )
+    else:
+        reader = SimpleDirectoryReader(
+            str(DATA_DIR), file_metadata=journal_file_metadata
+        )
+    return apply_metadata_policy(reader.load_data())
+
 
 def get_text_splitter():
     return SentenceSplitter(
@@ -46,7 +115,7 @@ def rebuild():
         raise FileNotFoundError(f"Data directory not found: {DATA_DIR.absolute()}")
 
     print(f"Loading documents from {DATA_DIR.absolute()}...")
-    documents = SimpleDirectoryReader(str(DATA_DIR)).load_data()
+    documents = load_documents()
 
     if not documents:
         raise ValueError("No documents found in data directory")
@@ -69,13 +138,23 @@ def update():
     if not DATA_DIR.exists():
         raise FileNotFoundError(f"Data directory not found: {DATA_DIR.absolute()}")
 
+    # Set transformations BEFORE constructing the index.
+    #
+    # index.insert() chunks with self._transformations, which BaseIndex captures
+    # once at construction time (transformations or Settings.transformations).
+    # Setting Settings.transformations after load_index_from_storage() is too
+    # late and is silently ignored -- which is what happened here until
+    # 2026-09-20: every incrementally added file was chunked at the 1024-token
+    # default instead of CHUNK_SIZE. Pass it explicitly as well, so this does
+    # not depend on global state at all.
+    Settings.transformations = [get_text_splitter()]
+
     # Load existing index
     print(f"Loading existing index from {PERSIST_DIR}...")
     storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
-    index = load_index_from_storage(storage_context)
-
-    # Set transformations so index.insert() chunks correctly
-    Settings.transformations = [get_text_splitter()]
+    index = load_index_from_storage(
+        storage_context, transformations=[get_text_splitter()]
+    )
 
     # Build lookup of indexed files: file_name -> (ref_doc_id, metadata)
     all_ref_docs = index.docstore.get_all_ref_doc_info()
@@ -87,8 +166,17 @@ def update():
 
     print(f"Index contains {len(indexed)} documents")
 
-    # Scan current files on disk
-    disk_files = {f.name: f for f in sorted(DATA_DIR.glob("*.txt"))}
+    # Scan current files on disk.
+    #
+    # Skip hidden files. pathlib's glob("*.txt") matches dotfiles (unlike shell
+    # globbing), but rebuild() goes through SimpleDirectoryReader, which skips
+    # them. Without this filter the two modes disagree: a stray .2026-02-21.txt
+    # was classified "new" after every rebuild and re-inserted via input_files=,
+    # which bypasses the hidden-file filter.
+    disk_files = {
+        f.name: f for f in sorted(DATA_DIR.glob("*.txt"))
+        if not f.name.startswith(".")
+    }
     print(f"Data directory contains {len(disk_files)} files")
 
     # Classify files
@@ -146,9 +234,7 @@ def update():
     if files_to_add:
         print(f"\nIndexing {len(files_to_add)} file(s)...")
         # Use "./" prefix to match paths from full build (pathlib strips it)
-        docs = SimpleDirectoryReader(
-            input_files=[f"./{f}" for f in files_to_add]
-        ).load_data()
+        docs = load_documents(input_files=[f"./{f}" for f in files_to_add])
         for doc in docs:
             index.insert(doc)
 
@@ -169,7 +255,13 @@ def main():
     args = parser.parse_args()
 
     # Configure embedding model
-    embed_model = HuggingFaceEmbedding(model_name=EMBED_MODEL_NAME)
+    # cache_folder + local_files_only match the query scripts, so the build
+    # and the queries are guaranteed to use identical weights.
+    embed_model = HuggingFaceEmbedding(
+        model_name=EMBED_MODEL_NAME,
+        cache_folder="./models",
+        local_files_only=True,
+    )
     Settings.embed_model = embed_model
 
     start = time.time()