import os import pickle import langchain import faiss from langchain import HuggingFaceHub from langchain.chains import ConversationalRetrievalChain from langchain.chat_models import ChatOpenAI from langchain.document_loaders import DirectoryLoader, TextLoader, UnstructuredHTMLLoader from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings from langchain.memory import ConversationBufferWindowMemory from langchain.prompts.chat import ( ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate, ) from langchain.text_splitter import CharacterTextSplitter from langchain.vectorstores.faiss import FAISS from langchain.cache import InMemoryCache # Create a dict of filenames and their corresponding url links to be used as source in line #203 file_url_mapping = { 'docs/01_course-orientation/01_about-the-course/01_introduction-to-the-3d-printing-specialization.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/mLvWU/introduction-to-the-3d-printing-specialization', 'docs/01_course-orientation/01_about-the-course/02_welcome-to-the-3d-printing-revolution.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/zQ7lG/welcome-to-the-3d-printing-revolution', 'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/02_how-will-3d-printing-change-business.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/D8VUj/how-will-3d-printing-change-business', 'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/01_whats-special-about-3d-printing.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/WmJQL/whats-special-about-3d-printing', 'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/04_remixing-products-exercise-overview.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/J3Tq3/remixing-products-exercise-overview', 'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/03_the-future-of-3d-printing.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/tjSlh/the-future-of-3d-printing', 'docs/03_module-2-why-is-it-revolutionary/02_an-early-look-at-the-coming-revolution/01_tour-of-the-illinois-makerlab-vishal-sachdev.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/rcVnN/tour-of-the-illinois-makerlab-vishal-sachdev', 'docs/03_module-2-why-is-it-revolutionary/02_an-early-look-at-the-coming-revolution/02_meet-the-makers-arielle-rausin-cameron-alberg-scott-zelman.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/J2nD2/meet-the-makers-arielle-rausin-cameron-alberg-scott-zelman', 'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/03_shapeways-lauren-slowik.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/mi1zr/shapeways-lauren-slowik', 'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/04_3d-fashion-francis-bitonti.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/KmYxf/3d-fashion-francis-bitonti', 'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/06_whats-next-hod-lipson.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/3zKRN/whats-next-hod-lipson', 'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/05_3d-printed-battery-paul-braun.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/mrEAK/3d-printed-battery-paul-braun', 'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/02_normal-nikki-kaufman.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/I8QXq/normal-nikki-kaufman', 'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/01_voodoo-manufacturing-max-friefeld.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/atrTP/voodoo-manufacturing-max-friefeld', 'docs/04_course-conclusion/01_course-wrap-up-whats-next/01_the-3d-printing-revolution-wrap-up.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/f6XvX/the-3d-printing-revolution-wrap-up', 'docs/04_course-conclusion/01_course-wrap-up-whats-next/03_gies-online-programs.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/JdvZk/gies-online-programs', 'docs/02_module-1-what-is-3d-printing/05_optional-content/01_like-this-course-learn-more-with-the-imba-optional.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/MIATt/like-this-course-learn-more-with-the-imba-optional', 'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/04_where-designs-come-from.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/0fU1x/where-designs-come-from', 'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/05_where-to-find-3d-printers.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/3XTPj/where-to-find-3d-printers', 'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/01_history-of-3d-printing.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/HMvuh/history-of-3d-printing', 'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/02_how-3d-printers-work.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/dNpvw/how-3d-printers-work', 'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/03_materials-costs.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/j63D2/materials-costs', 'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/06_3d-printing-applications.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/EJdi9/3d-printing-applications', 'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/03_selective-laser-melting-rodrigo-gutierrez.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/kkpwz/selective-laser-melting-rodrigo-gutierrez', 'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/04_3d-printing-ecosystem-aaron-roy.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/2zjzy/3d-printing-ecosystem-aaron-roy', 'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/01_3d-printing-demonstration-danny-lohan.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/5b8IG/3d-printing-demonstration-danny-lohan', 'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/02_3d-printing-vs-additive-manufacturing-mark-cotteleer.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/qBvTb/3d-printing-vs-additive-manufacturing-mark-cotteleer', 'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/02_my-3d-printing-story-aric-rindfleisch.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/hhWc8/my-3d-printing-story-aric-rindfleisch', 'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/04_3d-printing-the-maker-movement-hackerspaces-chris-meyer.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/8HO88/3d-printing-the-maker-movement-hackerspaces-chris-meyer', 'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/06_what-would-you-make-exercise-overview.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/kfDp2/what-would-you-make-exercise-overview', 'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/05_the-birth-of-desktop-3d-printing-matt-griffin.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/RlvEF/the-birth-of-desktop-3d-printing-matt-griffin', 'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/03_tour-of-sector67-chris-meyer.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/HXJrm/tour-of-sector67-chris-meyer', 'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/01_views-on-3d-printing-champaign-new-york.en.txt': 'https://www.coursera.org/learn/3d-printing-revolution/lecture/jvULv/views-on-3d-printing-champaign-new-york', 'docs/01_course-orientation/01_about-the-course/05_glossary_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/515pX/glossary', 'docs/01_course-orientation/01_about-the-course/03_syllabus_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/l2J8d/syllabus', 'docs/01_course-orientation/01_about-the-course/06_learner-stories_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/goE8a/learner-stories', 'docs/01_course-orientation/01_about-the-course/04_about-the-discussion-forums_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/HA7H7/about-the-discussion-forums', 'docs/01_course-orientation/02_about-your-classmates/02_social-media_illinois.edu.html': 'https://www.facebook.com/illinois.edu', 'docs/01_course-orientation/02_about-your-classmates/01_updating-your-profile_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/f2iFO/updating-your-profile', 'docs/01_course-orientation/02_about-your-classmates/02_social-media_termsDknlfPe-5xKJJaHWl4j6kveY2GjKiZxK_vd-JDIN3-6piB5w.2PhI_l8TrILDNPPI-v-VPg.kolgdmfhbhzcbgm3-lv_.html': 'https://www.coursera.org/about/termsDknlfPe-5xKJJaHWl4j6kveY2GjKiZxK_vd-JDIN3-6piB5w.2PhI_l8TrILDNPPI-v-VPg.kOLGdmfhbhzCbGM3-lv_j477mwb9Ffuxi5xhEac6Biqu1NvspELKPGjNRjoAzuv8LasLtq22lxzrgdd9C8Y4JQ4gHm7FuRqTL4rlby3Pb_N4mpVXMkT83a3Ob_0QWVHv7LiZghGDTwCWYxU4lMfZpfsqsm7PoQ7HfKUFvHwgUIbOGox3ZZgJtBE2t-TDkbegcktpcn6k2VqZZ0WYvQTad7oijs5WHJLfL7EYiUGb01udFqMaOLIPP1msztyo496GDNUgBSvsJcPfHE20dluqe5_KzaSoXzKxXdiW12DjKJk_XDNc14mf41U17h5HgMXg', 'docs/01_course-orientation/02_about-your-classmates/02_social-media_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/1vga3/social-media', 'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/05_remixing-products-exercise_peer_assignment_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/peer/FAZ0P/remixing-products-exercise', 'docs/03_module-2-why-is-it-revolutionary/01_module-2-information/01_module-2-overview_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/UwuNp/module-2-overview', 'docs/04_course-conclusion/01_course-wrap-up-whats-next/02_congratulations_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/vISgA/congratulations', 'docs/05_Resources/03_3d-printing-services-and-products/01__resources.html': 'https://www.coursera.org/learn/3d-printing-revolution/resources/aPz3d', 'docs/05_Resources/04_3d-printing-community/01__resources.html': 'https://www.coursera.org/learn/3d-printing-revolution/resources/lfbM9', 'docs/05_Resources/02_3d-printing-softwares/01__resources.html': 'https://www.coursera.org/learn/3d-printing-revolution/resources/ltlHt', 'docs/05_Resources/01_books-articles/01__resources.html': 'https://www.coursera.org/learn/3d-printing-revolution/resources/FH3x3', 'docs/05_Resources/05_explore-the-imba/01__resources.html': 'https://www.coursera.org/learn/3d-printing-revolution/resources/0AejF', 'docs/02_module-1-what-is-3d-printing/01_module-1-overview/01_module-1-overview_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/supplement/HZXB5/module-1-overview', 'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/07_what-would-you-make-exercise_peer_assignment_instructions.html': 'https://www.coursera.org/learn/3d-printing-revolution/peer/t8bqq/what-would-you-make-exercise'} langchain.llm_cache = InMemoryCache() global model_name models = ["GPT-3.5", "Flan UL2", "GPT-4", "Flan T5"] pickle_file = "_vs.pkl" index_file = "_vs.index" models_folder = "models/" llm = ChatOpenAI(model_name="gpt-4", temperature=0.1) embeddings = OpenAIEmbeddings(model='text-embedding-ada-002') chat_history = [] memory = ConversationBufferWindowMemory(memory_key="chat_history", k=10) vectorstore_index = None system_template = """You are Coursera QA Bot. Have a conversation with a human, answering the following questions as best you can. You are a teaching assistant for a Coursera Course: The 3D Printing Revolution and can answer any question about that using vectorstore or context. Use the following pieces of context to answer the users question. ---------------- {context}""" messages = [ SystemMessagePromptTemplate.from_template(system_template), HumanMessagePromptTemplate.from_template("{question}"), ] CHAT_PROMPT = ChatPromptTemplate.from_messages(messages) def set_model_and_embeddings(model): global chat_history set_model(model) # set_embeddings(model) chat_history = [] def set_model(model): global llm print("Setting model to " + str(model)) if model == "GPT-3.5": print("Loading GPT-3.5") llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.1) elif model == "GPT-4": print("Loading GPT-4") llm = ChatOpenAI(model_name="gpt-4", temperature=0.1) elif model == "Flan UL2": print("Loading Flan-UL2") llm = HuggingFaceHub(repo_id="google/flan-ul2", model_kwargs={"temperature": 0.1, "max_new_tokens":500}) elif model == "Flan T5": print("Loading Flan T5") llm = HuggingFaceHub(repo_id="google/flan-t5-base", model_kwargs={"temperature": 0.1}) else: print("Loading GPT-3.5 from else") llm = ChatOpenAI(model_name="text-davinci-002", temperature=0.1) def set_embeddings(model): global embeddings if model == "GPT-3.5" or model == "GPT-4": print("Loading OpenAI embeddings") embeddings = OpenAIEmbeddings(model='text-embedding-ada-002') elif model == "Flan UL2" or model == "Flan T5": print("Loading Hugging Face embeddings") embeddings = HuggingFaceHubEmbeddings(repo_id="sentence-transformers/all-MiniLM-L6-v2") def get_search_index(model): global vectorstore_index if os.path.isfile(get_file_path(model, pickle_file)) and os.path.isfile( get_file_path(model, index_file)) and os.path.getsize(get_file_path(model, pickle_file)) > 0: # Load index from pickle file with open(get_file_path(model, pickle_file), "rb") as f: search_index = pickle.load(f) print("Loaded index") else: search_index = create_index(model) print("Created index") vectorstore_index = search_index return search_index def create_index(model): source_chunks = create_chunk_documents() search_index = search_index_from_docs(source_chunks) faiss.write_index(search_index.index, get_file_path(model, index_file)) # Save index to pickle file with open(get_file_path(model, pickle_file), "wb") as f: pickle.dump(search_index, f) return search_index def get_file_path(model, file): # If model is GPT3.5 or GPT4 return models_folder + openai + file else return models_folder + hf + file if model == "GPT-3.5" or model == "GPT-4": return models_folder + "openai" + file else: return models_folder + "hf" + file def search_index_from_docs(source_chunks): # print("source chunks: " + str(len(source_chunks))) # print("embeddings: " + str(embeddings)) search_index = FAISS.from_documents(source_chunks, embeddings) return search_index def get_html_files(): loader = DirectoryLoader('docs', glob="**/*.html", loader_cls=UnstructuredHTMLLoader, recursive=True) document_list = loader.load() return document_list def fetch_data_for_embeddings(): document_list = get_text_files() document_list.extend(get_html_files()) # use file_url_mapping to set metadata of document to url which has been set as the source for document in document_list: document.metadata["url"] = file_url_mapping.get(document.metadata["source"]) print("document list: " + str(len(document_list))) return document_list def get_text_files(): loader = DirectoryLoader('docs', glob="**/*.txt", loader_cls=TextLoader, recursive=True) document_list = loader.load() return document_list def create_chunk_documents(): sources = fetch_data_for_embeddings() splitter = CharacterTextSplitter(separator=" ", chunk_size=800, chunk_overlap=0) source_chunks = splitter.split_documents(sources) print("chunks: " + str(len(source_chunks))) return source_chunks def get_qa_chain(vectorstore_index): global llm, model_name print(llm) # embeddings_filter = EmbeddingsFilter(embeddings=embeddings, similarity_threshold=0.76) # compression_retriever = ContextualCompressionRetriever(base_compressor=embeddings_filter, base_retriever=gpt_3_5_index.as_retriever()) retriever = vectorstore_index.as_retriever(search_type="similarity_score_threshold", search_kwargs={"score_threshold": .7}) chain = ConversationalRetrievalChain.from_llm(llm, retriever, return_source_documents=True, verbose=True, get_chat_history=get_chat_history, combine_docs_chain_kwargs={"prompt": CHAT_PROMPT}) return chain def get_chat_history(inputs) -> str: res = [] for human, ai in inputs: res.append(f"Human:{human}\nAI:{ai}") return "\n".join(res) def generate_answer(question) -> str: global chat_history, vectorstore_index chain = get_qa_chain(vectorstore_index) result = chain( {"question": question, "chat_history": chat_history, "vectordbkwargs": {"search_distance": 0.6}}) chat_history = [(question, result["answer"])] sources = [] print(result) for document in result['source_documents']: sources.append("\n" + document.metadata['url']) # sources.append(source.split('/')[-1].split('.')[0]) print(sources) source = ',\n'.join(set(sources)) return result['answer'] + '\nSOURCES: ' + source