Add bge-reranker-v2-m3; fix metadata and chunking bugs

.gitignore

@@ -34,3 +34,10 @@ query.log
 
 # Duplicate of CLAUDE.md
 claude.md
+
+# Eval data generated FROM the journal -- contains verbatim personal text.
+# The harness code in eval/*.py is safe to share; these are not.
+eval/passages.yaml
+eval/llm_compare/
+eval/snapshots/
+eval/__pycache__/

build_store.py

@@ -10,6 +10,7 @@
 # Used Sonnet 4.5 to suggest changes; Opus 4.6 for incremental update
 
 from llama_index.core import (
+    Document,
     SimpleDirectoryReader,
     StorageContext,
     VectorStoreIndex,
@@ -17,11 +18,13 @@ from llama_index.core import (
     Settings,
 )
 from pathlib import Path
+from llama_index.core.readers.file.base import default_file_metadata_func
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.core.node_parser import SentenceSplitter
 import argparse
 import datetime
 import os
+import re
 import time
 
 # Shared constants
@@ -31,6 +34,72 @@ EMBED_MODEL_NAME = "BAAI/bge-large-en-v1.5"
 CHUNK_SIZE = 256
 CHUNK_OVERLAP = 25
 
+# Journal files are named YYYY-MM-DD.txt; the entry date is the filename.
+# Filesystem timestamps are NOT the entry date (the 2000-07-29 entry carries
+# creation_date 2001-05-25), so the date has to be parsed from the name.
+ENTRY_DATE_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})\.txt$")
+
+# --- Metadata policy -------------------------------------------------------
+#
+# Every key below stays in the node's metadata dict and comes back on every
+# retrieval result, whatever these lists say. The lists only control which
+# keys get concatenated into the *text* that the embedder and the LLM see.
+#
+# Previously nothing was excluded from the embed text, so all 5210 chunks were
+# embedded with a 36-token absolute path glued to the front -- 14% of a
+# 256-token chunk, and the same constant string in every vector, which pulled
+# the mean pairwise cosine up from 0.561 to 0.627 and made similarity cutoffs
+# useless.
+#
+# Note BM25Retriever also indexes MetadataMode.EMBED, so the embed text is the
+# only place it can see dates. That is why entry_date stays in: it keeps BM25
+# able to match "2022-08-14" or "2022" at ~8 tokens instead of 36.
+EXCLUDED_EMBED_KEYS = [
+    "file_path", "file_name", "file_type", "file_size",
+    "creation_date", "last_modified_date", "last_accessed_date",
+    "year", "month",
+]
+# The LLM gets file_name instead -- it is the same date, and the synthesis
+# prompt asks the model to cite file names.
+EXCLUDED_LLM_KEYS = [
+    "file_path", "file_type", "file_size",
+    "creation_date", "last_modified_date", "last_accessed_date",
+    "entry_date", "year", "month",
+]
+
+
+def journal_file_metadata(file_path: str) -> dict:
+    """Default file metadata, plus the entry date parsed from the filename."""
+    meta = default_file_metadata_func(file_path)
+    m = ENTRY_DATE_RE.match(Path(file_path).name)
+    if m:
+        year, month, day = m.groups()
+        meta["entry_date"] = f"{year}-{month}-{day}"
+        meta["year"] = int(year)
+        meta["month"] = int(month)
+    return meta
+
+
+def apply_metadata_policy(documents: list[Document]) -> list[Document]:
+    """Set embed/LLM metadata exclusions. Nodes inherit these from the parent."""
+    for doc in documents:
+        doc.excluded_embed_metadata_keys = list(EXCLUDED_EMBED_KEYS)
+        doc.excluded_llm_metadata_keys = list(EXCLUDED_LLM_KEYS)
+    return documents
+
+
+def load_documents(input_files=None) -> list[Document]:
+    """Load journal documents with the shared metadata policy applied."""
+    if input_files is not None:
+        reader = SimpleDirectoryReader(
+            input_files=input_files, file_metadata=journal_file_metadata
+        )
+    else:
+        reader = SimpleDirectoryReader(
+            str(DATA_DIR), file_metadata=journal_file_metadata
+        )
+    return apply_metadata_policy(reader.load_data())
+
 
 def get_text_splitter():
     return SentenceSplitter(
@@ -46,7 +115,7 @@ def rebuild():
         raise FileNotFoundError(f"Data directory not found: {DATA_DIR.absolute()}")
 
     print(f"Loading documents from {DATA_DIR.absolute()}...")
-    documents = SimpleDirectoryReader(str(DATA_DIR)).load_data()
+    documents = load_documents()
 
     if not documents:
         raise ValueError("No documents found in data directory")
@@ -69,13 +138,23 @@ def update():
     if not DATA_DIR.exists():
         raise FileNotFoundError(f"Data directory not found: {DATA_DIR.absolute()}")
 
+    # Set transformations BEFORE constructing the index.
+    #
+    # index.insert() chunks with self._transformations, which BaseIndex captures
+    # once at construction time (transformations or Settings.transformations).
+    # Setting Settings.transformations after load_index_from_storage() is too
+    # late and is silently ignored -- which is what happened here until
+    # 2026-09-20: every incrementally added file was chunked at the 1024-token
+    # default instead of CHUNK_SIZE. Pass it explicitly as well, so this does
+    # not depend on global state at all.
+    Settings.transformations = [get_text_splitter()]
+
     # Load existing index
     print(f"Loading existing index from {PERSIST_DIR}...")
     storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
-    index = load_index_from_storage(storage_context)
-
-    # Set transformations so index.insert() chunks correctly
-    Settings.transformations = [get_text_splitter()]
+    index = load_index_from_storage(
+        storage_context, transformations=[get_text_splitter()]
+    )
 
     # Build lookup of indexed files: file_name -> (ref_doc_id, metadata)
     all_ref_docs = index.docstore.get_all_ref_doc_info()
@@ -87,8 +166,17 @@ def update():
 
     print(f"Index contains {len(indexed)} documents")
 
-    # Scan current files on disk
-    disk_files = {f.name: f for f in sorted(DATA_DIR.glob("*.txt"))}
+    # Scan current files on disk.
+    #
+    # Skip hidden files. pathlib's glob("*.txt") matches dotfiles (unlike shell
+    # globbing), but rebuild() goes through SimpleDirectoryReader, which skips
+    # them. Without this filter the two modes disagree: a stray .2026-02-21.txt
+    # was classified "new" after every rebuild and re-inserted via input_files=,
+    # which bypasses the hidden-file filter.
+    disk_files = {
+        f.name: f for f in sorted(DATA_DIR.glob("*.txt"))
+        if not f.name.startswith(".")
+    }
     print(f"Data directory contains {len(disk_files)} files")
 
     # Classify files
@@ -146,9 +234,7 @@ def update():
     if files_to_add:
         print(f"\nIndexing {len(files_to_add)} file(s)...")
         # Use "./" prefix to match paths from full build (pathlib strips it)
-        docs = SimpleDirectoryReader(
-            input_files=[f"./{f}" for f in files_to_add]
-        ).load_data()
+        docs = load_documents(input_files=[f"./{f}" for f in files_to_add])
         for doc in docs:
             index.insert(doc)
 
@@ -169,7 +255,13 @@ def main():
     args = parser.parse_args()
 
     # Configure embedding model
-    embed_model = HuggingFaceEmbedding(model_name=EMBED_MODEL_NAME)
+    # cache_folder + local_files_only match the query scripts, so the build
+    # and the queries are guaranteed to use identical weights.
+    embed_model = HuggingFaceEmbedding(
+        model_name=EMBED_MODEL_NAME,
+        cache_folder="./models",
+        local_files_only=True,
+    )
     Settings.embed_model = embed_model
 
     start = time.time()

clippings_search/build_clippings.py

@@ -23,7 +23,7 @@ from pathlib import Path
 os.chdir(Path(__file__).resolve().parent.parent)
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
-os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models"
+os.environ["HF_HUB_CACHE"] = "./models"  # HF hub cache for offline model loads
 os.environ["HF_HUB_OFFLINE"] = "1"
 
 import chromadb

clippings_search/retrieve_clippings.py

@@ -19,7 +19,7 @@ from pathlib import Path
 os.chdir(Path(__file__).resolve().parent.parent)
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
-os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models"
+os.environ["HF_HUB_CACHE"] = "./models"  # HF hub cache for offline model loads
 os.environ["HF_HUB_OFFLINE"] = "1"
 
 import chromadb
@@ -45,7 +45,9 @@ EMBED_MODEL = HuggingFaceEmbedding(
 )
 
 # Cross-encoder model for re-ranking (cached in ./models/)
-RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+# Must match query_hybrid.py -- see the note there on why this replaced
+# cross-encoder/ms-marco-MiniLM-L-12-v2.
+RERANK_MODEL = "BAAI/bge-reranker-v2-m3"
 RERANK_TOP_N = 15
 RETRIEVE_TOP_K = 30
 

pyproject.toml

@@ -26,6 +26,8 @@ dependencies = [
     "huggingface-hub==0.34.4",
     # --- Keyword search / document loaders ---
     "nltk>=3.9.1",
+    # --- eval harness (eval/, see EVAL_PLAN.md) ---
+    "pyyaml>=6.0",
     "beautifulsoup4>=4.13.4",
     "striprtf>=0.0.26",
     "pypdf>=6.7.1",

query_hybrid.py

@@ -20,7 +20,7 @@
 # at import time.
 import os
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
-os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models"
+os.environ["HF_HUB_CACHE"] = "./models"  # HF hub cache for offline model loads
 os.environ["HF_HUB_OFFLINE"] = "1"
 
 from llama_index.core import (
@@ -33,6 +33,7 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.llms.ollama import Ollama
 from llama_index.core.prompts import PromptTemplate
 from llama_index.core.postprocessor import SentenceTransformerRerank
+from llama_index.core.schema import MetadataMode
 from llama_index.retrievers.bm25 import BM25Retriever
 import sys
 
@@ -43,13 +44,27 @@ import sys
 # Embedding model (must match build_store.py)
 EMBED_MODEL = HuggingFaceEmbedding(cache_folder="./models", model_name="BAAI/bge-large-en-v1.5", local_files_only=True)
 
-# LLM model for generation. Use temp 0.3.
+# LLM model for generation.
 #LLM_MODEL = "command-r7b"
-# testing gemma4:e4b. Recommendations are to use temp 1.0
 LLM_MODEL = "gemma4:e4b"
 
+# Temperature. Gemma's own guidance is 1.0, but that is for open-ended chat.
+# This is grounded extraction -- the prompt says to answer only from context --
+# so a low temperature is the safer default. 1.0 previously; raise it back if
+# answers come out flat.
+LLM_TEMPERATURE = 0.3
+
+# Ollama num_ctx. The prompt itself is only ~2500 tokens with 15 chunks, but
+# thinking=True spends reasoning tokens out of this same budget, so leave room.
+LLM_CONTEXT_WINDOW = 16384
+
 # Cross-encoder model for re-ranking (cached in ./models/)
-RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+# Was cross-encoder/ms-marco-MiniLM-L-12-v2 (2021, trained on MS MARCO web
+# queries). It scored journal-style queries far out of its calibrated range --
+# top hits at -0.77 and -1.90 logits -- because "what would the author say
+# about..." is nothing like a search-engine query. bge-reranker-v2-m3 returns
+# calibrated 0-1 scores and handles the phrasing. Costs ~5s more per query.
+RERANK_MODEL = "BAAI/bge-reranker-v2-m3"
 RERANK_TOP_N = 15
 
 # Retrieval parameters
@@ -91,10 +106,10 @@ def main():
     # Note: Ollama temperature defaults to 0.8
     Settings.llm = Ollama(
         model=LLM_MODEL,
-        temperature=1.0,
+        temperature=LLM_TEMPERATURE,
         thinking=True,          # enable
         request_timeout=360.0,
-        context_window=8000,
+        context_window=LLM_CONTEXT_WINDOW,
     )
     
     # Use OpenAI API:
@@ -157,7 +172,11 @@ def main():
     n_both = len(vector_nodes) + len(bm25_nodes) - len(merged)
 
     # Estimate context length (prompt + node text)
-    context_text = "\n\n".join(n.get_content() for n in reranked)
+    # MetadataMode.LLM to match what the synthesizer actually sends
+    # (response_synthesizers/base.py uses MetadataMode.LLM). The default mode
+    # strips the "file_name: ..." prefix and understates the prompt.
+    context_text = "\n\n".join(
+        n.node.get_content(metadata_mode=MetadataMode.LLM) for n in reranked)
     prompt_text = PROMPT.format(context_str=context_text, query_str=q)
     n_context_tokens = len(prompt_text.split())  # rough word count; ~1.3 tokens/word
 

retrieve.py

@@ -15,7 +15,7 @@
 # at import time.
 import os
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
-os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models"
+os.environ["HF_HUB_CACHE"] = "./models"  # HF hub cache for offline model loads
 os.environ["HF_HUB_OFFLINE"] = "1"
 
 from llama_index.core import (
@@ -37,7 +37,9 @@ import textwrap
 EMBED_MODEL = HuggingFaceEmbedding(cache_folder="./models", model_name="BAAI/bge-large-en-v1.5", local_files_only=True)
 
 # Cross-encoder model for re-ranking (cached in ./models/)
-RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+# Must match query_hybrid.py -- see the note there on why this replaced
+# cross-encoder/ms-marco-MiniLM-L-12-v2.
+RERANK_MODEL = "BAAI/bge-reranker-v2-m3"
 RERANK_TOP_N = 15
 
 # Retrieval parameters

uv.lock

@@ -4277,6 +4277,7 @@ dependencies = [
     { name = "llama-index-vector-stores-chroma" },
     { name = "nltk" },
     { name = "pypdf" },
+    { name = "pyyaml" },
     { name = "sentence-transformers" },
     { name = "striprtf" },
     { name = "transformers" },
@@ -4308,6 +4309,7 @@ requires-dist = [
     { name = "llama-index-vector-stores-chroma", specifier = "==0.5.5" },
     { name = "nltk", specifier = ">=3.9.1" },
     { name = "pypdf", specifier = ">=6.7.1" },
+    { name = "pyyaml", specifier = ">=6.0" },
     { name = "sentence-transformers", specifier = ">=5.1.0" },
     { name = "striprtf", specifier = ">=0.0.26" },
     { name = "transformers", specifier = "==4.55.0" },