Update functions.py
Browse files- functions.py +1 -1
functions.py
CHANGED
|
@@ -106,7 +106,7 @@ def preprocess_plain_text(text,window_size=3):
|
|
| 106 |
return passages
|
| 107 |
|
| 108 |
@st.experimental_memo(suppress_st_warning=True)
|
| 109 |
-
def
|
| 110 |
|
| 111 |
"""Chunk text longer than 500 tokens"""
|
| 112 |
|
|
|
|
| 106 |
return passages
|
| 107 |
|
| 108 |
@st.experimental_memo(suppress_st_warning=True)
|
| 109 |
+
def chunk_and_preprocess_text(text):
|
| 110 |
|
| 111 |
"""Chunk text longer than 500 tokens"""
|
| 112 |
|