Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -17,7 +17,7 @@ def length_tokens(txt):
|
|
17 |
|
18 |
|
19 |
def extract_separators_from_string(separators_str):
|
20 |
-
print('Received:', type(separators_str), 'with value', separators_str)
|
21 |
try:
|
22 |
separators = separators_str[1:-1].split(", ")
|
23 |
return [separator.replace('"', "").replace("'", "") for separator in separators]
|
@@ -46,7 +46,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
|
|
46 |
separator=" ",
|
47 |
)
|
48 |
elif splitter_selection == LABEL_RECURSIVE:
|
49 |
-
print('Splitting with separators:', ',,'.join(separators), f'and chunk length {length} and chunk overlap {chunk_overlap}')
|
50 |
text_splitter = RecursiveCharacterTextSplitter(
|
51 |
chunk_size=length,
|
52 |
chunk_overlap=int(chunk_overlap),
|
|
|
17 |
|
18 |
|
19 |
def extract_separators_from_string(separators_str):
|
20 |
+
print('Received:', type(separators_str), 'with value', repr(separators_str))
|
21 |
try:
|
22 |
separators = separators_str[1:-1].split(", ")
|
23 |
return [separator.replace('"', "").replace("'", "") for separator in separators]
|
|
|
46 |
separator=" ",
|
47 |
)
|
48 |
elif splitter_selection == LABEL_RECURSIVE:
|
49 |
+
print('Splitting with separators:', ',,'.join([repr(el) for el in separators]), f'and chunk length {length} and chunk overlap {chunk_overlap}')
|
50 |
text_splitter = RecursiveCharacterTextSplitter(
|
51 |
chunk_size=length,
|
52 |
chunk_overlap=int(chunk_overlap),
|