Update app.py

app.py

@@ -16,12 +16,6 @@ bot_token = "<Assistant>"
 
 max_context_length = 750
 
-def is_english_word(tested_string):
-    pattern = re.compile(r"^[a-zA-Z]+$")
-    return pattern.match(tested_string) is not None
-
-
-
 def format(history):
     prompt = bos_token
 
@@ -34,6 +28,10 @@ def format(history):
     print(prompt)
     return prompt
 
+def remove_spaces_between_chinese(text):
+    rex = r"(?<![a-zA-Z]{2})(?<=[a-zA-Z]{1})[ ]+(?=[a-zA-Z] |.$)|(?<=\p{Han}) +"
+    return re.sub(rex, "", text, 0, re.MULTILINE | re.UNICODE)
+
 def gradio(model, tokenizer):
     def response(
         user_input,
@@ -72,13 +70,9 @@ def gradio(model, tokenizer):
         )
         output = beam_output[0][prompt_length:]
 
-        tokens = tokenizer.convert_ids_to_tokens(output)
-        for i, token in enumerate(tokens[:-1]):
-            if is_english_word(token) and is_english_word(tokens[i + 1]):
-                tokens[i] = token + " "
-        text = "".join(tokens).replace("##", "").replace("[UNK]", "").strip()
-
-        return text
+        generated = remove_spaces_between_chinese(tokenizer.decode(output, skip_special_tokens=True, clean_up_tokenization_spaces=True))
+        
+        return generated
 
     bot = gr.Chatbot(show_copy_button=True, show_share_button=True)
 
@@ -135,14 +129,13 @@ def gradio(model, tokenizer):
                 repetition_penalty,
                 no_repeat_ngram_size,
             ],
-            stop_btn = "🛑 Stop",
             retry_btn = "🔄 Regenerate",
             undo_btn = "↩️ Remove last turn",
             clear_btn = "➕ New conversation",
             examples=[
                 ["帮我生成一个句子，描述春天的美好。", 30, 0.9, 0.95, 1.2, 5],
                 ["给我讲一个笑话。", 50, 0.8, 0.9, 1.3, 6],
-                ["Give me some chinese names.", 100, 0.9, 1.0, 1, 5]
+                ["Give me five english names.", 100, 0.9, 1.0, 1, 5]
             ]
         )