Pclanglais commited on
Commit
b1f9918
·
verified ·
1 Parent(s): ebd0c9c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -153,7 +153,7 @@ def save_bibtex(bibtex_content):
153
  class CombinedProcessor:
154
  def process(self, user_message):
155
  editorial_text = re.sub("\n", " ¶ ", user_message)
156
- editorial_text = re.sub(r'\s*([;:,])\s*', r' \1 ', editorial_text)
157
  num_tokens = len(tokenizer.tokenize(editorial_text))
158
 
159
  batch_prompts = split_text(editorial_text, max_tokens=500) if num_tokens > 500 else [editorial_text]
 
153
  class CombinedProcessor:
154
  def process(self, user_message):
155
  editorial_text = re.sub("\n", " ¶ ", user_message)
156
+ editorial_text = re.sub(r'\s*([;:,\.])\s*', r' \1 ', editorial_text)
157
  num_tokens = len(tokenizer.tokenize(editorial_text))
158
 
159
  batch_prompts = split_text(editorial_text, max_tokens=500) if num_tokens > 500 else [editorial_text]