Lahiru Menikdiwela commited on
Commit
26c4dd0
·
1 Parent(s): c6d2349

fix input format issue2

Browse files
Files changed (1) hide show
  1. preprocess.py +1 -1
preprocess.py CHANGED
@@ -22,7 +22,7 @@ def get_document_splits_from_text(text:str) -> Document:
22
 
23
  def prepare_for_summarize(text:str,tokenizer):
24
  #!!!!!!!!!!!!Only for small inputs
25
- return text
26
  no_input_tokens = get_input_token_count(text,tokenizer)
27
  if no_input_tokens<12000:
28
  text_to_summarize = text
 
22
 
23
  def prepare_for_summarize(text:str,tokenizer):
24
  #!!!!!!!!!!!!Only for small inputs
25
+ return text, "short"
26
  no_input_tokens = get_input_token_count(text,tokenizer)
27
  if no_input_tokens<12000:
28
  text_to_summarize = text