Lahiru Menikdiwela
commited on
Commit
·
26c4dd0
1
Parent(s):
c6d2349
fix input format issue2
Browse files- preprocess.py +1 -1
preprocess.py
CHANGED
@@ -22,7 +22,7 @@ def get_document_splits_from_text(text:str) -> Document:
|
|
22 |
|
23 |
def prepare_for_summarize(text:str,tokenizer):
|
24 |
#!!!!!!!!!!!!Only for small inputs
|
25 |
-
return text
|
26 |
no_input_tokens = get_input_token_count(text,tokenizer)
|
27 |
if no_input_tokens<12000:
|
28 |
text_to_summarize = text
|
|
|
22 |
|
23 |
def prepare_for_summarize(text:str,tokenizer):
|
24 |
#!!!!!!!!!!!!Only for small inputs
|
25 |
+
return text, "short"
|
26 |
no_input_tokens = get_input_token_count(text,tokenizer)
|
27 |
if no_input_tokens<12000:
|
28 |
text_to_summarize = text
|