Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -17,7 +17,7 @@ def length_tokens(txt):
|
|
| 17 |
|
| 18 |
|
| 19 |
def extract_separators_from_string(separators_str):
|
| 20 |
-
print('Received:', type(separators_str), 'with value', separators_str)
|
| 21 |
try:
|
| 22 |
separators = separators_str[1:-1].split(", ")
|
| 23 |
return [separator.replace('"', "").replace("'", "") for separator in separators]
|
|
@@ -46,7 +46,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
|
|
| 46 |
separator=" ",
|
| 47 |
)
|
| 48 |
elif splitter_selection == LABEL_RECURSIVE:
|
| 49 |
-
print('Splitting with separators:', ',,'.join(separators), f'and chunk length {length} and chunk overlap {chunk_overlap}')
|
| 50 |
text_splitter = RecursiveCharacterTextSplitter(
|
| 51 |
chunk_size=length,
|
| 52 |
chunk_overlap=int(chunk_overlap),
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
def extract_separators_from_string(separators_str):
|
| 20 |
+
print('Received:', type(separators_str), 'with value', repr(separators_str))
|
| 21 |
try:
|
| 22 |
separators = separators_str[1:-1].split(", ")
|
| 23 |
return [separator.replace('"', "").replace("'", "") for separator in separators]
|
|
|
|
| 46 |
separator=" ",
|
| 47 |
)
|
| 48 |
elif splitter_selection == LABEL_RECURSIVE:
|
| 49 |
+
print('Splitting with separators:', ',,'.join([repr(el) for el in separators]), f'and chunk length {length} and chunk overlap {chunk_overlap}')
|
| 50 |
text_splitter = RecursiveCharacterTextSplitter(
|
| 51 |
chunk_size=length,
|
| 52 |
chunk_overlap=int(chunk_overlap),
|