add rag

Modules/rag.py

@@ -19,20 +19,7 @@ from langchain_mistralai import ChatMistralAI
 from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
 from langchain_community.tools import DuckDuckGoSearchRun
 
-# urls = [
-#     "https://www.toutelanutrition.com/wikifit/guide-nutrition/nutrition-sportive/apports-proteines",
 
-# ]
-
-# docs = [WebBaseLoader(url).load() for url in urls]
-# docs_list = [item for sublist in docs for item in sublist]
-
-# text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
-#     chunk_size=250, chunk_overlap=0
-# )
-# doc_splits = text_splitter.split_documents(docs_list)
-
-####### PDF
 def load_chunk_persist_pdf() -> Chroma:
     pdf_folder_path = "data/pdf_folder/"
     documents = []
@@ -52,13 +39,6 @@ def load_chunk_persist_pdf() -> Chroma:
     vectorstore.persist()
     return vectorstore
 
-# from langchain_community.document_loaders import PyPDFLoader
-# loader = PyPDFLoader('data/fitness_programs/ZeroToHero.pdf')
-# pages = loader.load_and_split()
-
-# text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-# splits = text_splitter.split_documents(pages)
-# vectorstore = Chroma.from_documents(documents=splits, embedding=MistralAIEmbeddings())
 vectorstore = load_chunk_persist_pdf()
 retriever = vectorstore.as_retriever()
 prompt = hub.pull("rlm/rag-prompt")