ssearch/archived/claude_diagnostic.py

# Better HyDE debugging with targeted tests

from llama_index.core.indices.query.query_transform import HyDEQueryTransform
from llama_index.core import PromptTemplate
from llama_index.core import Settings
from llama_index.core.base.base_query_engine import BaseQueryEngine
from llama_index.llms.ollama import Ollama

llm="llama3.1:8B"

# Use a local model to generate
Settings.llm = Ollama(
    model=llm,    # First model tested
    request_timeout=360.0,
    context_window=8000,
    temperature=0.7,
    )


# Test queries that should produce very different hypothetical documents
test_queries = [
    "What is the capital of France?",
    "How do you make chocolate chip cookies?", 
    "Explain quantum physics",
    "Write a love letter",
    "Describe symptoms of the common cold"
]

print("=== DEBUGGING HYDE STEP BY STEP ===\n")

# 1. Test the LLM with HyDE-style prompts directly
print("1. Testing LLM directly with HyDE-style prompts:")
print("-" * 50)

for query in test_queries[:2]:  # Just test 2 to keep output manageable
    direct_prompt = f"""Generate a hypothetical document that would contain the answer to this query.

Query: {query}

Hypothetical document:"""
    
    response = Settings.llm.complete(direct_prompt)
    print(f"Query: {query}")
    print(f"Direct LLM Response: {response.text[:100]}...")
    print()

# 2. Check HyDE internals - let's see what's actually happening
print("\n2. Examining HyDE internal behavior:")
print("-" * 50)

# Create a custom HyDE that shows us everything
class VerboseHyDETransform(HyDEQueryTransform):
    def _get_prompts(self):
        """Show what prompts are being used"""
        prompts = super()._get_prompts()
        print(f"HyDE prompts: {prompts}")
        return prompts
    
    def _run_component(self, **kwargs):
        """Show what's being passed to the LLM"""
        print(f"HyDE _run_component kwargs: {kwargs}")
        result = super()._run_component(**kwargs)
        print(f"HyDE _run_component result: {result}")
        return result

# Test with verbose HyDE
verbose_hyde = VerboseHyDETransform(llm=Settings.llm)
test_result = verbose_hyde.run("What is machine learning?")
print(f"Final verbose result: {test_result}")

# 3. Try the most basic possible test
print("\n3. Most basic HyDE test:")
print("-" * 50)

basic_hyde = HyDEQueryTransform(llm=Settings.llm)
basic_result = basic_hyde.run("Paris")
print(f"Input: 'Paris'")
print(f"Output: '{basic_result}'")
print(f"Same as input? {basic_result.strip() == 'Paris'}")

# 4. Check if it's a version issue - try alternative approach
print("\n4. Alternative HyDE approach:")
print("-" * 50)

try:
    # Some versions might need different initialization
    from llama_index.core.query_engine import TransformQueryEngine
    from llama_index.core.indices.query.query_transform import HyDEQueryTransform
    
    # Try with explicit prompt template
    hyde_prompt_template = PromptTemplate(
        "Please write a passage to answer the question\n"
        "Try to include as many key details as possible\n"
        "\n"
        "\n"
        "Passage:{query_str}\n"
        "\n"
        "\n"
        "Passage:"
    )
    
    alt_hyde = HyDEQueryTransform(
        llm=Settings.llm,
        hyde_prompt=hyde_prompt_template
    )
    
    alt_result = alt_hyde.run("What causes rain?")
    print(f"Alternative approach result: {alt_result}")
    
except Exception as e:
    print(f"Alternative approach failed: {e}")

# 5. Check what happens with different query formats
print("\n5. Testing different input formats:")
print("-" * 50)

from llama_index.core.schema import QueryBundle

# Test with QueryBundle vs string
hyde_test = HyDEQueryTransform(llm=Settings.llm)

string_result = hyde_test.run("test query")
print(f"String input result: '{string_result}'")

query_bundle = QueryBundle(query_str="test query")
bundle_result = hyde_test.run(query_bundle)
print(f"QueryBundle input result: '{bundle_result}'")

# 6. Version and import check
print("\n6. Environment check:")
print("-" * 50)
import llama_index
print(f"LlamaIndex version: {llama_index.__version__}")

# Check what LLM you're actually using
print(f"LLM type: {type(Settings.llm)}")
print(f"LLM model name: {getattr(Settings.llm, 'model', 'Unknown')}")

# 7. Try the nuclear option - completely manual implementation
print("\n7. Manual HyDE implementation:")
print("-" * 50)

def manual_hyde(query: str, llm):
    """Completely manual HyDE to see if the concept works"""
    prompt = f"""You are an expert writer. Generate a realistic document excerpt that would contain the answer to this question.

Question: {query}

Document excerpt:"""
    
    response = llm.complete(prompt)
    return response.text

manual_result = manual_hyde("What is photosynthesis?", Settings.llm)
print(f"Manual HyDE result: {manual_result[:150]}...")

# 8. Final diagnostic
print("\n8. Final diagnostic questions:")
print("-" * 50)
print("If all the above show the LLM generating proper responses but HyDE still returns original:")
print("- What LLM are you using? (OpenAI, Anthropic, local model, etc.)")
print("- What's your LlamaIndex version?")
print("- Are there any error messages in the logs?")
print("- Does the LLM have any special configuration or wrappers?")