diff --git a/.gitignore b/.gitignore index af82761..991e82e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,10 @@ query.log # Duplicate of CLAUDE.md claude.md + +# Eval data generated FROM the journal -- contains verbatim personal text. +# The harness code in eval/*.py is safe to share; these are not. +eval/passages.yaml +eval/llm_compare/ +eval/snapshots/ +eval/__pycache__/ diff --git a/build_store.py b/build_store.py index add3db3..74983af 100644 --- a/build_store.py +++ b/build_store.py @@ -10,6 +10,7 @@ # Used Sonnet 4.5 to suggest changes; Opus 4.6 for incremental update from llama_index.core import ( + Document, SimpleDirectoryReader, StorageContext, VectorStoreIndex, @@ -17,11 +18,13 @@ from llama_index.core import ( Settings, ) from pathlib import Path +from llama_index.core.readers.file.base import default_file_metadata_func from llama_index.embeddings.huggingface import HuggingFaceEmbedding from llama_index.core.node_parser import SentenceSplitter import argparse import datetime import os +import re import time # Shared constants @@ -31,6 +34,72 @@ EMBED_MODEL_NAME = "BAAI/bge-large-en-v1.5" CHUNK_SIZE = 256 CHUNK_OVERLAP = 25 +# Journal files are named YYYY-MM-DD.txt; the entry date is the filename. +# Filesystem timestamps are NOT the entry date (the 2000-07-29 entry carries +# creation_date 2001-05-25), so the date has to be parsed from the name. +ENTRY_DATE_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})\.txt$") + +# --- Metadata policy ------------------------------------------------------- +# +# Every key below stays in the node's metadata dict and comes back on every +# retrieval result, whatever these lists say. The lists only control which +# keys get concatenated into the *text* that the embedder and the LLM see. +# +# Previously nothing was excluded from the embed text, so all 5210 chunks were +# embedded with a 36-token absolute path glued to the front -- 14% of a +# 256-token chunk, and the same constant string in every vector, which pulled +# the mean pairwise cosine up from 0.561 to 0.627 and made similarity cutoffs +# useless. +# +# Note BM25Retriever also indexes MetadataMode.EMBED, so the embed text is the +# only place it can see dates. That is why entry_date stays in: it keeps BM25 +# able to match "2022-08-14" or "2022" at ~8 tokens instead of 36. +EXCLUDED_EMBED_KEYS = [ + "file_path", "file_name", "file_type", "file_size", + "creation_date", "last_modified_date", "last_accessed_date", + "year", "month", +] +# The LLM gets file_name instead -- it is the same date, and the synthesis +# prompt asks the model to cite file names. +EXCLUDED_LLM_KEYS = [ + "file_path", "file_type", "file_size", + "creation_date", "last_modified_date", "last_accessed_date", + "entry_date", "year", "month", +] + + +def journal_file_metadata(file_path: str) -> dict: + """Default file metadata, plus the entry date parsed from the filename.""" + meta = default_file_metadata_func(file_path) + m = ENTRY_DATE_RE.match(Path(file_path).name) + if m: + year, month, day = m.groups() + meta["entry_date"] = f"{year}-{month}-{day}" + meta["year"] = int(year) + meta["month"] = int(month) + return meta + + +def apply_metadata_policy(documents: list[Document]) -> list[Document]: + """Set embed/LLM metadata exclusions. Nodes inherit these from the parent.""" + for doc in documents: + doc.excluded_embed_metadata_keys = list(EXCLUDED_EMBED_KEYS) + doc.excluded_llm_metadata_keys = list(EXCLUDED_LLM_KEYS) + return documents + + +def load_documents(input_files=None) -> list[Document]: + """Load journal documents with the shared metadata policy applied.""" + if input_files is not None: + reader = SimpleDirectoryReader( + input_files=input_files, file_metadata=journal_file_metadata + ) + else: + reader = SimpleDirectoryReader( + str(DATA_DIR), file_metadata=journal_file_metadata + ) + return apply_metadata_policy(reader.load_data()) + def get_text_splitter(): return SentenceSplitter( @@ -46,7 +115,7 @@ def rebuild(): raise FileNotFoundError(f"Data directory not found: {DATA_DIR.absolute()}") print(f"Loading documents from {DATA_DIR.absolute()}...") - documents = SimpleDirectoryReader(str(DATA_DIR)).load_data() + documents = load_documents() if not documents: raise ValueError("No documents found in data directory") @@ -69,13 +138,23 @@ def update(): if not DATA_DIR.exists(): raise FileNotFoundError(f"Data directory not found: {DATA_DIR.absolute()}") + # Set transformations BEFORE constructing the index. + # + # index.insert() chunks with self._transformations, which BaseIndex captures + # once at construction time (transformations or Settings.transformations). + # Setting Settings.transformations after load_index_from_storage() is too + # late and is silently ignored -- which is what happened here until + # 2026-09-20: every incrementally added file was chunked at the 1024-token + # default instead of CHUNK_SIZE. Pass it explicitly as well, so this does + # not depend on global state at all. + Settings.transformations = [get_text_splitter()] + # Load existing index print(f"Loading existing index from {PERSIST_DIR}...") storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR) - index = load_index_from_storage(storage_context) - - # Set transformations so index.insert() chunks correctly - Settings.transformations = [get_text_splitter()] + index = load_index_from_storage( + storage_context, transformations=[get_text_splitter()] + ) # Build lookup of indexed files: file_name -> (ref_doc_id, metadata) all_ref_docs = index.docstore.get_all_ref_doc_info() @@ -87,8 +166,17 @@ def update(): print(f"Index contains {len(indexed)} documents") - # Scan current files on disk - disk_files = {f.name: f for f in sorted(DATA_DIR.glob("*.txt"))} + # Scan current files on disk. + # + # Skip hidden files. pathlib's glob("*.txt") matches dotfiles (unlike shell + # globbing), but rebuild() goes through SimpleDirectoryReader, which skips + # them. Without this filter the two modes disagree: a stray .2026-02-21.txt + # was classified "new" after every rebuild and re-inserted via input_files=, + # which bypasses the hidden-file filter. + disk_files = { + f.name: f for f in sorted(DATA_DIR.glob("*.txt")) + if not f.name.startswith(".") + } print(f"Data directory contains {len(disk_files)} files") # Classify files @@ -146,9 +234,7 @@ def update(): if files_to_add: print(f"\nIndexing {len(files_to_add)} file(s)...") # Use "./" prefix to match paths from full build (pathlib strips it) - docs = SimpleDirectoryReader( - input_files=[f"./{f}" for f in files_to_add] - ).load_data() + docs = load_documents(input_files=[f"./{f}" for f in files_to_add]) for doc in docs: index.insert(doc) @@ -169,7 +255,13 @@ def main(): args = parser.parse_args() # Configure embedding model - embed_model = HuggingFaceEmbedding(model_name=EMBED_MODEL_NAME) + # cache_folder + local_files_only match the query scripts, so the build + # and the queries are guaranteed to use identical weights. + embed_model = HuggingFaceEmbedding( + model_name=EMBED_MODEL_NAME, + cache_folder="./models", + local_files_only=True, + ) Settings.embed_model = embed_model start = time.time() diff --git a/clippings_search/build_clippings.py b/clippings_search/build_clippings.py index ff58088..72aee1a 100644 --- a/clippings_search/build_clippings.py +++ b/clippings_search/build_clippings.py @@ -23,7 +23,7 @@ from pathlib import Path os.chdir(Path(__file__).resolve().parent.parent) os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models" +os.environ["HF_HUB_CACHE"] = "./models" # HF hub cache for offline model loads os.environ["HF_HUB_OFFLINE"] = "1" import chromadb diff --git a/clippings_search/retrieve_clippings.py b/clippings_search/retrieve_clippings.py index 91c612c..7bbc905 100644 --- a/clippings_search/retrieve_clippings.py +++ b/clippings_search/retrieve_clippings.py @@ -19,7 +19,7 @@ from pathlib import Path os.chdir(Path(__file__).resolve().parent.parent) os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models" +os.environ["HF_HUB_CACHE"] = "./models" # HF hub cache for offline model loads os.environ["HF_HUB_OFFLINE"] = "1" import chromadb @@ -45,7 +45,9 @@ EMBED_MODEL = HuggingFaceEmbedding( ) # Cross-encoder model for re-ranking (cached in ./models/) -RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2" +# Must match query_hybrid.py -- see the note there on why this replaced +# cross-encoder/ms-marco-MiniLM-L-12-v2. +RERANK_MODEL = "BAAI/bge-reranker-v2-m3" RERANK_TOP_N = 15 RETRIEVE_TOP_K = 30 diff --git a/pyproject.toml b/pyproject.toml index fa7277c..61bf4fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ dependencies = [ "huggingface-hub==0.34.4", # --- Keyword search / document loaders --- "nltk>=3.9.1", + # --- eval harness (eval/, see EVAL_PLAN.md) --- + "pyyaml>=6.0", "beautifulsoup4>=4.13.4", "striprtf>=0.0.26", "pypdf>=6.7.1", diff --git a/query_hybrid.py b/query_hybrid.py index 62175fa..18d7243 100644 --- a/query_hybrid.py +++ b/query_hybrid.py @@ -20,7 +20,7 @@ # at import time. import os os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models" +os.environ["HF_HUB_CACHE"] = "./models" # HF hub cache for offline model loads os.environ["HF_HUB_OFFLINE"] = "1" from llama_index.core import ( @@ -33,6 +33,7 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding from llama_index.llms.ollama import Ollama from llama_index.core.prompts import PromptTemplate from llama_index.core.postprocessor import SentenceTransformerRerank +from llama_index.core.schema import MetadataMode from llama_index.retrievers.bm25 import BM25Retriever import sys @@ -43,13 +44,27 @@ import sys # Embedding model (must match build_store.py) EMBED_MODEL = HuggingFaceEmbedding(cache_folder="./models", model_name="BAAI/bge-large-en-v1.5", local_files_only=True) -# LLM model for generation. Use temp 0.3. +# LLM model for generation. #LLM_MODEL = "command-r7b" -# testing gemma4:e4b. Recommendations are to use temp 1.0 LLM_MODEL = "gemma4:e4b" +# Temperature. Gemma's own guidance is 1.0, but that is for open-ended chat. +# This is grounded extraction -- the prompt says to answer only from context -- +# so a low temperature is the safer default. 1.0 previously; raise it back if +# answers come out flat. +LLM_TEMPERATURE = 0.3 + +# Ollama num_ctx. The prompt itself is only ~2500 tokens with 15 chunks, but +# thinking=True spends reasoning tokens out of this same budget, so leave room. +LLM_CONTEXT_WINDOW = 16384 + # Cross-encoder model for re-ranking (cached in ./models/) -RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2" +# Was cross-encoder/ms-marco-MiniLM-L-12-v2 (2021, trained on MS MARCO web +# queries). It scored journal-style queries far out of its calibrated range -- +# top hits at -0.77 and -1.90 logits -- because "what would the author say +# about..." is nothing like a search-engine query. bge-reranker-v2-m3 returns +# calibrated 0-1 scores and handles the phrasing. Costs ~5s more per query. +RERANK_MODEL = "BAAI/bge-reranker-v2-m3" RERANK_TOP_N = 15 # Retrieval parameters @@ -91,10 +106,10 @@ def main(): # Note: Ollama temperature defaults to 0.8 Settings.llm = Ollama( model=LLM_MODEL, - temperature=1.0, + temperature=LLM_TEMPERATURE, thinking=True, # enable request_timeout=360.0, - context_window=8000, + context_window=LLM_CONTEXT_WINDOW, ) # Use OpenAI API: @@ -157,7 +172,11 @@ def main(): n_both = len(vector_nodes) + len(bm25_nodes) - len(merged) # Estimate context length (prompt + node text) - context_text = "\n\n".join(n.get_content() for n in reranked) + # MetadataMode.LLM to match what the synthesizer actually sends + # (response_synthesizers/base.py uses MetadataMode.LLM). The default mode + # strips the "file_name: ..." prefix and understates the prompt. + context_text = "\n\n".join( + n.node.get_content(metadata_mode=MetadataMode.LLM) for n in reranked) prompt_text = PROMPT.format(context_str=context_text, query_str=q) n_context_tokens = len(prompt_text.split()) # rough word count; ~1.3 tokens/word diff --git a/retrieve.py b/retrieve.py index 28f92e1..c922aa8 100644 --- a/retrieve.py +++ b/retrieve.py @@ -15,7 +15,7 @@ # at import time. import os os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models" +os.environ["HF_HUB_CACHE"] = "./models" # HF hub cache for offline model loads os.environ["HF_HUB_OFFLINE"] = "1" from llama_index.core import ( @@ -37,7 +37,9 @@ import textwrap EMBED_MODEL = HuggingFaceEmbedding(cache_folder="./models", model_name="BAAI/bge-large-en-v1.5", local_files_only=True) # Cross-encoder model for re-ranking (cached in ./models/) -RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2" +# Must match query_hybrid.py -- see the note there on why this replaced +# cross-encoder/ms-marco-MiniLM-L-12-v2. +RERANK_MODEL = "BAAI/bge-reranker-v2-m3" RERANK_TOP_N = 15 # Retrieval parameters diff --git a/uv.lock b/uv.lock index 5836589..b77ccad 100644 --- a/uv.lock +++ b/uv.lock @@ -4277,6 +4277,7 @@ dependencies = [ { name = "llama-index-vector-stores-chroma" }, { name = "nltk" }, { name = "pypdf" }, + { name = "pyyaml" }, { name = "sentence-transformers" }, { name = "striprtf" }, { name = "transformers" }, @@ -4308,6 +4309,7 @@ requires-dist = [ { name = "llama-index-vector-stores-chroma", specifier = "==0.5.5" }, { name = "nltk", specifier = ">=3.9.1" }, { name = "pypdf", specifier = ">=6.7.1" }, + { name = "pyyaml", specifier = ">=6.0" }, { name = "sentence-transformers", specifier = ">=5.1.0" }, { name = "striprtf", specifier = ">=0.0.26" }, { name = "transformers", specifier = "==4.55.0" },