49 lines
1.3 KiB
Python
49 lines
1.3 KiB
Python
# build.py
|
|
#
|
|
# Import documents from data, generate embedded vector store
|
|
# and save to disk in directory ./storage
|
|
#
|
|
# August 2025
|
|
# E. M. Furst
|
|
|
|
from llama_index.core import (
|
|
SimpleDirectoryReader,
|
|
VectorStoreIndex,
|
|
Settings,
|
|
)
|
|
|
|
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
|
from llama_index.core.node_parser import SentenceSplitter
|
|
|
|
def main():
|
|
# Choose your embedding model
|
|
embed_model = HuggingFaceEmbedding(cache_folder="./models",
|
|
model_name="BAAI/bge-large-en-v1.5")
|
|
|
|
# Configure global settings for LlamaIndex
|
|
Settings.embed_model = embed_model
|
|
|
|
# Load documents
|
|
documents = SimpleDirectoryReader("./data").load_data()
|
|
|
|
# Create the custom textsplitter
|
|
# Set chunk size and overlap (e.g., 256 tokens, 25 tokens overlap)
|
|
text_splitter = SentenceSplitter(
|
|
chunk_size=500,
|
|
chunk_overlap=50,
|
|
)
|
|
Settings.text_splitter = text_splitter
|
|
|
|
# Build the index
|
|
index = VectorStoreIndex.from_documents(
|
|
documents, transformations=[text_splitter],
|
|
show_progress=True,
|
|
)
|
|
|
|
# Persist both vector store and index metadata
|
|
index.storage_context.persist(persist_dir="./storage")
|
|
|
|
print("Index built and saved to ./storage")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|