ingestion.py

from llama_index.core import SimpleDirectoryReader, get_response_synthesizer
from llama_index.core import DocumentSummaryIndex
from llama_index.llms.ollama import Ollama
from llama_index.core.node_parser import SentenceSplitter
from langchain_community.embeddings import OllamaEmbeddings
from llama_index.core import Settings
import nest_asyncio

nest_asyncio.apply()

# Load data
react_doc = SimpleDirectoryReader(input_dir="../data/pdf", file_metadata={"category": "AI applications"}).load_data()

llm = Ollama(model="llama3", request_timeout=120, base_url="http://localhost:11434")
embed_model = OllamaEmbeddings(model="llama3")
Settings.llm = llm
Settings.embed_model = embed_model

# Text Splitter
sentence_splitter = SentenceSplitter(chunk_size=500, chunk_overlap=0)

# response_mode: 检索返回node的模式
response_synthesizer = get_response_synthesizer(
    response_mode="tree_summarize",
    use_async=True,
)

# Buid index： 建立索引，transformations可以指定对文档进行预处理，response_synthesizer指定检索返回node的模式
index = DocumentSummaryIndex.from_documents(
    react_doc,
    llm=llm,
    transformations=[sentence_splitter],
    response_synthesizer=response_synthesizer,
    show_progress=True,
)

# 存储index
index.storage_context.persist("../kb/index")