Spaces:
Sleeping
Sleeping
Update appStore/prep_utils.py
Browse files- appStore/prep_utils.py +12 -2
appStore/prep_utils.py
CHANGED
@@ -1,8 +1,10 @@
|
|
1 |
from langchain_text_splitters import TokenTextSplitter
|
2 |
from langchain.docstore.document import Document
|
3 |
import pandas as pd
|
4 |
-
|
|
|
5 |
path_to_data = "./docStore/"
|
|
|
6 |
|
7 |
def create_chunks(text, tokens_count =400, overlap_count=0):
|
8 |
"""TAKES A TEXT AND CERATES CREATES CHUNKS"""
|
@@ -20,4 +22,12 @@ def create_documents(df, text_column):
|
|
20 |
metadata = {col: str(df.loc[i, col]) for col in list(set(df.columns) - {text_column})}
|
21 |
placeholder.append(Document(page_content= df.loc[i,text_column],
|
22 |
metadata=metadata))
|
23 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
from langchain_text_splitters import TokenTextSplitter
|
2 |
from langchain.docstore.document import Document
|
3 |
import pandas as pd
|
4 |
+
from qdrant_client import QdrantClient
|
5 |
+
path_to_qdrant = "/data/local_qdrant"
|
6 |
path_to_data = "./docStore/"
|
7 |
+
import streamlit as st
|
8 |
|
9 |
def create_chunks(text, tokens_count =400, overlap_count=0):
|
10 |
"""TAKES A TEXT AND CERATES CREATES CHUNKS"""
|
|
|
22 |
metadata = {col: str(df.loc[i, col]) for col in list(set(df.columns) - {text_column})}
|
23 |
placeholder.append(Document(page_content= df.loc[i,text_column],
|
24 |
metadata=metadata))
|
25 |
+
return
|
26 |
+
|
27 |
+
@st.cache_resource
|
28 |
+
def get_client():
|
29 |
+
if 'qdrant' in st.session_state:
|
30 |
+
return st.session_state.qdrant
|
31 |
+
else:
|
32 |
+
client = QdrantClient(path=path_to_qdrant)
|
33 |
+
return client
|