Reorganize project: rename scripts, archive superseded, add clippings_search/
- Rename build_exp_claude.py → build_store.py - Rename query_hybrid_bm25_v4.py → query_hybrid.py - Rename retrieve_hybrid_raw.py → retrieve.py - Archive query_topk_prompt_engine_v3.py (superseded by hybrid) - Archive retrieve_raw.py (superseded by hybrid) - Move build_clippings.py, retrieve_clippings.py → clippings_search/ - Update run_query.sh, README.md, CLAUDE.md for new names
This commit is contained in:
parent
b4bf89ce4b
commit
5a3294f74c
9 changed files with 80 additions and 87 deletions
136
archived/query_topk_prompt_engine_v3.py
Normal file
136
archived/query_topk_prompt_engine_v3.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
# query_topk_prompt_engine_v3.py
|
||||
# Run a query on a vector store with cross-encoder re-ranking
|
||||
#
|
||||
# Based on v2. Adds a cross-encoder re-ranking step:
|
||||
# 1. Retrieve top-30 chunks via vector similarity (bi-encoder, fast)
|
||||
# 2. Re-rank to top-15 using a cross-encoder (slower but more accurate)
|
||||
# 3. Pass re-ranked chunks to LLM for synthesis
|
||||
#
|
||||
# The cross-encoder scores each (query, chunk) pair jointly, which captures
|
||||
# nuance that bi-encoder dot-product similarity misses.
|
||||
#
|
||||
# E.M.F. February 2026
|
||||
|
||||
# Environment vars must be set before importing huggingface/transformers
|
||||
# libraries, because huggingface_hub.constants evaluates HF_HUB_OFFLINE
|
||||
# at import time.
|
||||
import os
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models"
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
|
||||
from llama_index.core import (
|
||||
StorageContext,
|
||||
load_index_from_storage,
|
||||
Settings,
|
||||
)
|
||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||
from llama_index.llms.ollama import Ollama
|
||||
from llama_index.core.prompts import PromptTemplate
|
||||
from llama_index.core.postprocessor import SentenceTransformerRerank
|
||||
import sys
|
||||
|
||||
#
|
||||
# Globals
|
||||
#
|
||||
|
||||
# Embedding model used in vector store (must match build_exp_claude.py)
|
||||
EMBED_MODEL = HuggingFaceEmbedding(cache_folder="./models", model_name="BAAI/bge-large-en-v1.5", local_files_only=True)
|
||||
|
||||
# LLM model for generation
|
||||
llm = "command-r7b"
|
||||
|
||||
# Cross-encoder model for re-ranking (cached in ./models/)
|
||||
#RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
|
||||
RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
|
||||
#RERANK_MODEL = "cross-encoder/stsb-roberta-base"
|
||||
#RERANK_MODEL = "BAAI/bge-reranker-v2-m3"
|
||||
RERANK_TOP_N = 15 # keep top 15 after re-ranking
|
||||
RETRIEVE_TOP_K = 30 # retrieve wider pool for re-ranker to work with
|
||||
|
||||
#
|
||||
# Custom prompt for the query engine - Version 3
|
||||
#
|
||||
# Adapted for re-ranked context: every excerpt below has been scored for
|
||||
# relevance by a cross-encoder, so even lower-ranked ones are worth examining.
|
||||
# The prompt encourages the LLM to draw from all provided excerpts and to
|
||||
# note what each distinct file contributes rather than collapsing onto one.
|
||||
#
|
||||
PROMPT = PromptTemplate(
|
||||
"""You are a precise research assistant analyzing excerpts from a personal journal collection.
|
||||
Every excerpt below has been selected and ranked for relevance to the query.
|
||||
|
||||
CONTEXT (ranked by relevance):
|
||||
{context_str}
|
||||
|
||||
QUERY:
|
||||
{query_str}
|
||||
|
||||
Instructions:
|
||||
- Answer ONLY using information explicitly present in the CONTEXT above
|
||||
- Examine ALL provided excerpts, not just the top few -- each one was selected for relevance
|
||||
- Be specific: quote or closely paraphrase key passages and cite their file names
|
||||
- When multiple files touch on the query, note what each one contributes
|
||||
- If the context doesn't contain enough information to answer fully, say so
|
||||
|
||||
Your response should:
|
||||
1. Directly answer the query, drawing on as many relevant excerpts as possible
|
||||
2. Reference specific files and their content (e.g., "In <filename>, ...")
|
||||
3. End with a list of all files that contributed to your answer, with a brief note on each
|
||||
|
||||
If the context is insufficient, explain what's missing."""
|
||||
)
|
||||
|
||||
#
|
||||
# Main program routine
|
||||
#
|
||||
|
||||
def main():
|
||||
# Use a local model to generate -- in this case using Ollama
|
||||
Settings.llm = Ollama(
|
||||
model=llm,
|
||||
request_timeout=360.0,
|
||||
context_window=8000
|
||||
)
|
||||
|
||||
# Load embedding model (same as used for vector store)
|
||||
Settings.embed_model = EMBED_MODEL
|
||||
|
||||
# Load persisted vector store + metadata
|
||||
storage_context = StorageContext.from_defaults(persist_dir="./storage_exp")
|
||||
index = load_index_from_storage(storage_context)
|
||||
|
||||
# Cross-encoder re-ranker
|
||||
reranker = SentenceTransformerRerank(
|
||||
model=RERANK_MODEL,
|
||||
top_n=RERANK_TOP_N,
|
||||
)
|
||||
|
||||
# Build query engine: retrieve wide (top-30), re-rank to top-15, then synthesize
|
||||
query_engine = index.as_query_engine(
|
||||
similarity_top_k=RETRIEVE_TOP_K,
|
||||
text_qa_template=PROMPT,
|
||||
node_postprocessors=[reranker],
|
||||
)
|
||||
|
||||
# Query
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python query_topk_prompt_engine_v3.py QUERY_TEXT")
|
||||
sys.exit(1)
|
||||
q = " ".join(sys.argv[1:])
|
||||
|
||||
# Generate the response by querying the engine
|
||||
response = query_engine.query(q)
|
||||
|
||||
# Return the query response and source documents
|
||||
print("\nResponse:\n")
|
||||
print(response.response)
|
||||
|
||||
print("\nSource documents:")
|
||||
for node in response.source_nodes:
|
||||
meta = getattr(node, "metadata", None) or node.node.metadata
|
||||
print(f"{meta.get('file_name')} {meta.get('file_path')} {getattr(node, 'score', None):.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
97
archived/retrieve_raw.py
Normal file
97
archived/retrieve_raw.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
# retrieve_raw.py
|
||||
# Verbatim chunk retrieval: vector search + cross-encoder re-ranking, no LLM.
|
||||
#
|
||||
# Returns the top re-ranked chunks with their full text, file metadata, and
|
||||
# scores. Useful for browsing source material directly and verifying what
|
||||
# the RAG pipeline retrieves before LLM synthesis.
|
||||
#
|
||||
# Uses the same vector store, embedding model, and re-ranker as
|
||||
# query_topk_prompt_engine_v3.py, but skips the LLM step entirely.
|
||||
#
|
||||
# E.M.F. February 2026
|
||||
|
||||
# Environment vars must be set before importing huggingface/transformers
|
||||
# libraries, because huggingface_hub.constants evaluates HF_HUB_OFFLINE
|
||||
# at import time.
|
||||
import os
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
os.environ["SENTENCE_TRANSFORMERS_HOME"] = "./models"
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
|
||||
from llama_index.core import (
|
||||
StorageContext,
|
||||
load_index_from_storage,
|
||||
Settings,
|
||||
)
|
||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||
from llama_index.core.postprocessor import SentenceTransformerRerank
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
#
|
||||
# Globals
|
||||
#
|
||||
|
||||
# Embedding model (must match build_exp_claude.py)
|
||||
EMBED_MODEL = HuggingFaceEmbedding(cache_folder="./models", model_name="BAAI/bge-large-en-v1.5", local_files_only=True)
|
||||
|
||||
# Cross-encoder model for re-ranking (cached in ./models/)
|
||||
RERANK_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
|
||||
RERANK_TOP_N = 15
|
||||
RETRIEVE_TOP_K = 30
|
||||
|
||||
# Output formatting
|
||||
WRAP_WIDTH = 80
|
||||
|
||||
|
||||
def main():
|
||||
# No LLM needed -- set embed model only
|
||||
Settings.embed_model = EMBED_MODEL
|
||||
|
||||
# Load persisted vector store
|
||||
storage_context = StorageContext.from_defaults(persist_dir="./storage_exp")
|
||||
index = load_index_from_storage(storage_context)
|
||||
|
||||
# Build retriever (vector search only, no query engine / LLM)
|
||||
retriever = index.as_retriever(similarity_top_k=RETRIEVE_TOP_K)
|
||||
|
||||
# Cross-encoder re-ranker
|
||||
reranker = SentenceTransformerRerank(
|
||||
model=RERANK_MODEL,
|
||||
top_n=RERANK_TOP_N,
|
||||
)
|
||||
|
||||
# Query
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python retrieve_raw.py QUERY_TEXT")
|
||||
sys.exit(1)
|
||||
q = " ".join(sys.argv[1:])
|
||||
|
||||
# Retrieve and re-rank
|
||||
nodes = retriever.retrieve(q)
|
||||
reranked = reranker.postprocess_nodes(nodes, query_str=q)
|
||||
|
||||
# Output
|
||||
print(f"\nQuery: {q}")
|
||||
print(f"Retrieved {len(nodes)} chunks, re-ranked to top {len(reranked)}\n")
|
||||
|
||||
for i, node in enumerate(reranked, 1):
|
||||
meta = getattr(node, "metadata", None) or node.node.metadata
|
||||
score = getattr(node, "score", None)
|
||||
file_name = meta.get("file_name", "unknown")
|
||||
text = node.get_content()
|
||||
|
||||
print("="*WRAP_WIDTH)
|
||||
print(f"=== [{i}] {file_name} (score: {score:.3f}) ")
|
||||
print("="*WRAP_WIDTH)
|
||||
# Wrap text for readability
|
||||
for line in text.splitlines():
|
||||
if line.strip():
|
||||
print(textwrap.fill(line, width=WRAP_WIDTH))
|
||||
else:
|
||||
print()
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue