rag 코드 비활성화

rag.py

@@ -1,64 +1,40 @@
-################
-# PDF 파일을 로드하고 문서를 쪼개서 문서벡터화 한 후 질의하기
-################
-import tiktoken
-tokenizer = tiktoken.get_encoding('cl100k_base')
-def tiktoken_len(text):
-    tokens = tokenizer.encode(text)
-    return len(tokens)
+# ################
+# # PDF 파일을 로드하고 문서를 쪼개서 문서벡터화 한 후 질의하기
+# ################
+# import tiktoken
+# tokenizer = tiktoken.get_encoding('cl100k_base')
+# def tiktoken_len(text):
+#     tokens = tokenizer.encode(text)
+#     return len(tokens)
     
-from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import Chroma
-from langchain.document_loaders import PyPDFLoader
-from langchain.embeddings import HuggingFaceEmbeddings
-
-## pdf 파일로드 하고 쪼개기
-loader = PyPDFLoader('https://wdr.ubion.co.kr/wowpass/img/event/gsat_170823/gsat_170823.pdf')
-pages = loader.load_and_split()
-
-## chunk로 쪼개기
-text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=80,length_function=tiktoken_len)
-sourceDocs = text_splitter.split_documents(pages)
-
-################
-# HuggingFace 모델로 문서벡터화 후 유사도 탐색
-################
-from langchain.embeddings import HuggingFaceEmbeddings
-
-model_huggingface = HuggingFaceEmbeddings(model_name = 'jhgan/ko-sroberta-multitask', 
-                                          model_kwargs = {'device':'cpu'}, 
-                                          encode_kwargs = {'normalize_embeddings' : True})
+# from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
+# from langchain.text_splitter import RecursiveCharacterTextSplitter
+# from langchain.vectorstores import Chroma
+# from langchain.document_loaders import PyPDFLoader
+# from langchain.embeddings import HuggingFaceEmbeddings
 
-## Chroma 기반 pdf(docs 벡터화)
-db = Chroma.from_documents(sourceDocs, model_huggingface)
+# ## pdf 파일로드 하고 쪼개기
+# loader = PyPDFLoader('gsat_170823.pdf')
+# pages = loader.load_and_split()
 
-## 질의하기
-def SearchDocs(question, k=1):
-    results = db.similarity_search_with_relevance_scores(question, k = k)
-    merged = ' '.join([sourceDocs[result[0]][0] for result in results])
-    return merged
+# ## chunk로 쪼개기
+# text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=80,length_function=tiktoken_len)
+# sourceDocs = text_splitter.split_documents(pages)
 
 # ################
-# # 찾은 문서를 프롬프트에 전달하여 LLM으로 답변 생성
+# # HuggingFace 모델로 문서벡터화 후 유사도 탐색
 # ################
-# from langchain_community.chat_models import ChatOllama
-# llm = ChatOllama(
-#     base_url='http://localhost:11434',
-#     # model="phi3:medium", # 너무 느려서 mini로 변경
-#     model="phi3:mini",
-# )
+# from langchain.embeddings import HuggingFaceEmbeddings
 
-# from langchain_core.prompts import ChatPromptTemplate
+# model_huggingface = HuggingFaceEmbeddings(model_name = 'jhgan/ko-sroberta-multitask', 
+#                                           model_kwargs = {'device':'cpu'}, 
+#                                           encode_kwargs = {'normalize_embeddings' : True})
 
-# prompt = ChatPromptTemplate.from_messages([
-#     ("system", "Please answer the following question from the document: {document}"),
-#     ("user", "{question}"),
-# ])
+# ## Chroma 기반 pdf(docs 벡터화)
+# db = Chroma.from_documents(sourceDocs, model_huggingface)
 
-# # print('-'*50)
-# chain = prompt | llm
-# def Response(question):
-#     searchedDocs = SearchDocs(question)
-#     mergedDoc = ' '.join(searchedDocs[0][0])
-#     return chain.invoke({"question": question, "document": mergedDoc})
+# ## 질의하기
+# def SearchDocs(question, k=1):
+#     results = db.similarity_search_with_relevance_scores(question, k = k)
+#     merged = ' '.join([sourceDocs[result[0]][0] for result in results])
+#     return merged