m-ric HF staff commited on
Commit
0e7a241
·
verified ·
1 Parent(s): 262dfde

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -35,7 +35,6 @@ def change_split_selection(split_selection):
35
 
36
  def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
37
  separators = extract_separators_from_string(separators_str)
38
- print('Extracted separators:', ',,'.join(separators))
39
  length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
40
  if splitter_selection == LABEL_TEXTSPLITTER:
41
  text_splitter = CharacterTextSplitter(
@@ -47,6 +46,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
47
  separator=" ",
48
  )
49
  elif splitter_selection == LABEL_RECURSIVE:
 
50
  text_splitter = RecursiveCharacterTextSplitter(
51
  chunk_size=length,
52
  chunk_overlap=int(chunk_overlap),
 
35
 
36
  def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
37
  separators = extract_separators_from_string(separators_str)
 
38
  length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
39
  if splitter_selection == LABEL_TEXTSPLITTER:
40
  text_splitter = CharacterTextSplitter(
 
46
  separator=" ",
47
  )
48
  elif splitter_selection == LABEL_RECURSIVE:
49
+ print('Splitting with separators:', ',,'.join(separators))
50
  text_splitter = RecursiveCharacterTextSplitter(
51
  chunk_size=length,
52
  chunk_overlap=int(chunk_overlap),