Spaces:
Sleeping
Sleeping
File size: 3,072 Bytes
f1268bf 8e29230 83d98e9 5ffaf21 f1268bf 8e29230 83d98e9 8e29230 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 |
from langchain_core.documents import Document
from chains import generate_document_summary_prompt
from config import SEVEN_B_LLM_MODEL
# embeddings functions
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.embeddings.sentence_transformer import (
SentenceTransformerEmbeddings,
)
import time
from langchain_core.documents import Document
from config import EMBEDDING_MODEL
from langchain.retrievers import VectorStoreRetriever
from langchain_core.vectorstores import VectorStoreRetriever
def create_embeddings(
docs: list[Document],
chunk_size:int = 500,
chunk_overlap:int = 50,
):
"""given a sequence of `Document` objects this fucntion will
generate embeddings for it.
## argument
:params docs (list[Document]) -> list of `list[Document]`
:params chunk_size (int) -> chunk size in which documents are chunks, defaults to 500
:params chunk_overlap (int) -> the amount of token that will be overlapped between chunks, defaults to 50
:params embedding_model (str) -> the huggingspace model that will embed the documents
## Return
Tuple of embedding and chunks
"""
text_splitter = RecursiveCharacterTextSplitter(
separators=["\n\n", "\n", "(?<=\. )", " ", ""],
chunk_size = chunk_size,
chunk_overlap = chunk_overlap,
length_function = len,
)
# Stage one: read all the docs, split them into chunks.
st = time.time()
print('Loading documents and creating chunks ...')
# Split each document into chunks using the configured text splitter
chunks = text_splitter.create_documents([doc.page_content for doc in docs], metadatas=[doc.metadata for doc in docs])
et = time.time() - st
print(f'Time taken to chunk {len(docs)} documents: {et} seconds.')
#Stage two: embed the docs.
embeddings = SentenceTransformerEmbeddings(model_name=EMBEDDING_MODEL)
print(f"created a total of {len(chunks)} chunks")
return embeddings,chunks
def generate_document_summaries(
docs: list[Document]
) -> list[Document]:
"""
Generates summaries for a list of Document objects and updates their metadata with the summaries.
Args:
docs (List[Document]): A list of Document objects to generate summaries for.
Returns:
List[Document]: A new list of Document objects with updated metadata containing the summaries.
Example:
docs = [Document(metadata={"title": "Doc1"}), Document(metadata={"title": "Doc2"})]
updated_docs = generate_document_summaries(docs)
for doc in updated_docs:
print(doc.metadata["summary"])
"""
new_docs = docs.copy()
for doc in new_docs:
genrate_summary_chain = generate_document_summary_prompt | SEVEN_B_LLM_MODEL
summary = genrate_summary_chain.invoke(
{"document":str(doc.metadata)}
)
doc.metadata.update(
{"summary":summary}
)
return new_docs |