ppsingh commited on
Commit
1170eaf
·
verified ·
1 Parent(s): 01014d6

Update appStore/prep_utils.py

Browse files
Files changed (1) hide show
  1. appStore/prep_utils.py +12 -2
appStore/prep_utils.py CHANGED
@@ -1,8 +1,10 @@
1
  from langchain_text_splitters import TokenTextSplitter
2
  from langchain.docstore.document import Document
3
  import pandas as pd
4
-
 
5
  path_to_data = "./docStore/"
 
6
 
7
  def create_chunks(text, tokens_count =400, overlap_count=0):
8
  """TAKES A TEXT AND CERATES CREATES CHUNKS"""
@@ -20,4 +22,12 @@ def create_documents(df, text_column):
20
  metadata = {col: str(df.loc[i, col]) for col in list(set(df.columns) - {text_column})}
21
  placeholder.append(Document(page_content= df.loc[i,text_column],
22
  metadata=metadata))
23
- return placeholder
 
 
 
 
 
 
 
 
 
1
  from langchain_text_splitters import TokenTextSplitter
2
  from langchain.docstore.document import Document
3
  import pandas as pd
4
+ from qdrant_client import QdrantClient
5
+ path_to_qdrant = "/data/local_qdrant"
6
  path_to_data = "./docStore/"
7
+ import streamlit as st
8
 
9
  def create_chunks(text, tokens_count =400, overlap_count=0):
10
  """TAKES A TEXT AND CERATES CREATES CHUNKS"""
 
22
  metadata = {col: str(df.loc[i, col]) for col in list(set(df.columns) - {text_column})}
23
  placeholder.append(Document(page_content= df.loc[i,text_column],
24
  metadata=metadata))
25
+ return
26
+
27
+ @st.cache_resource
28
+ def get_client():
29
+ if 'qdrant' in st.session_state:
30
+ return st.session_state.qdrant
31
+ else:
32
+ client = QdrantClient(path=path_to_qdrant)
33
+ return client