Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -153,7 +153,7 @@ def save_bibtex(bibtex_content):
|
|
153 |
class CombinedProcessor:
|
154 |
def process(self, user_message):
|
155 |
editorial_text = re.sub("\n", " ¶ ", user_message)
|
156 |
-
editorial_text = re.sub(r'\s*([
|
157 |
num_tokens = len(tokenizer.tokenize(editorial_text))
|
158 |
|
159 |
batch_prompts = split_text(editorial_text, max_tokens=500) if num_tokens > 500 else [editorial_text]
|
|
|
153 |
class CombinedProcessor:
|
154 |
def process(self, user_message):
|
155 |
editorial_text = re.sub("\n", " ¶ ", user_message)
|
156 |
+
editorial_text = re.sub(r'\s*([;:,\.])\s*', r' \1 ', editorial_text)
|
157 |
num_tokens = len(tokenizer.tokenize(editorial_text))
|
158 |
|
159 |
batch_prompts = split_text(editorial_text, max_tokens=500) if num_tokens > 500 else [editorial_text]
|