Add bge-reranker-v2-m3; fix metadata and chunking bugs

query_hybrid.py

@@ -20,7 +20,7 @@
 # at import time.
 import os
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
-os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models"
+os.environ["HF_HUB_CACHE"] = "./models"  # HF hub cache for offline model loads
 os.environ["HF_HUB_OFFLINE"] = "1"
 
 from llama_index.core import (
@@ -33,6 +33,7 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.llms.ollama import Ollama
 from llama_index.core.prompts import PromptTemplate
 from llama_index.core.postprocessor import SentenceTransformerRerank
+from llama_index.core.schema import MetadataMode
 from llama_index.retrievers.bm25 import BM25Retriever
 import sys
 
@@ -43,13 +44,27 @@ import sys
 # Embedding model (must match build_store.py)
 EMBED_MODEL = HuggingFaceEmbedding(cache_folder="./models", model_name="BAAI/bge-large-en-v1.5", local_files_only=True)
 
-# LLM model for generation. Use temp 0.3.
+# LLM model for generation.
 #LLM_MODEL = "command-r7b"
-# testing gemma4:e4b. Recommendations are to use temp 1.0
 LLM_MODEL = "gemma4:e4b"
 
+# Temperature. Gemma's own guidance is 1.0, but that is for open-ended chat.
+# This is grounded extraction -- the prompt says to answer only from context --
+# so a low temperature is the safer default. 1.0 previously; raise it back if
+# answers come out flat.
+LLM_TEMPERATURE = 0.3
+
+# Ollama num_ctx. The prompt itself is only ~2500 tokens with 15 chunks, but
+# thinking=True spends reasoning tokens out of this same budget, so leave room.
+LLM_CONTEXT_WINDOW = 16384
+
 # Cross-encoder model for re-ranking (cached in ./models/)
-RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+# Was cross-encoder/ms-marco-MiniLM-L-12-v2 (2021, trained on MS MARCO web
+# queries). It scored journal-style queries far out of its calibrated range --
+# top hits at -0.77 and -1.90 logits -- because "what would the author say
+# about..." is nothing like a search-engine query. bge-reranker-v2-m3 returns
+# calibrated 0-1 scores and handles the phrasing. Costs ~5s more per query.
+RERANK_MODEL = "BAAI/bge-reranker-v2-m3"
 RERANK_TOP_N = 15
 
 # Retrieval parameters
@@ -91,10 +106,10 @@ def main():
     # Note: Ollama temperature defaults to 0.8
     Settings.llm = Ollama(
         model=LLM_MODEL,
-        temperature=1.0,
+        temperature=LLM_TEMPERATURE,
         thinking=True,          # enable
         request_timeout=360.0,
-        context_window=8000,
+        context_window=LLM_CONTEXT_WINDOW,
     )
     
     # Use OpenAI API:
@@ -157,7 +172,11 @@ def main():
     n_both = len(vector_nodes) + len(bm25_nodes) - len(merged)
 
     # Estimate context length (prompt + node text)
-    context_text = "\n\n".join(n.get_content() for n in reranked)
+    # MetadataMode.LLM to match what the synthesizer actually sends
+    # (response_synthesizers/base.py uses MetadataMode.LLM). The default mode
+    # strips the "file_name: ..." prefix and understates the prompt.
+    context_text = "\n\n".join(
+        n.node.get_content(metadata_mode=MetadataMode.LLM) for n in reranked)
     prompt_text = PROMPT.format(context_str=context_text, query_str=q)
     n_context_tokens = len(prompt_text.split())  # rough word count; ~1.3 tokens/word