Update tokenizer.py

tokenizer.py

@@ -4,18 +4,18 @@ subprocess.run(["pip", "install", "spacy"])
 
 import spacy
 
-#spacy.cli.download("en_core_web_sm")
+spacy.cli.download("en_core_web_sm")
 
-#from spacy.tokens import Doc
+from spacy.tokens import Doc
 
 # 加载英文模型
-#nlp = spacy.load('en_core_web_sm')
+nlp = spacy.load('en_core_web_sm')
 
 import nltk
 
-#nltk.download('punkt')
+nltk.download('punkt')
 
-#from nltk.tokenize import word_tokenize
+from nltk.tokenize import word_tokenize
 
 import jieba
 
@@ -38,26 +38,24 @@ with codecs.open('model2_data/bpecode.en', 'r', 'utf-8') as f:
 
 def spacy_tokenize(line):
     # 使用spaCy处理文本
-    #doc = nlp(line)
+    doc = nlp(line)
     # 获取单词列表
-    #words = [token.text for token in doc]
+    words = [token.text for token in doc]
     # 将单词连接成一个字符串，单词间用一个空格间隔
-    #return ' '.join(words)
-    return ""
+    return ' '.join(words)
 
 
 def nltk_tokenize(line):
     # 使用NLTK的word_tokenize进行分词
-    #tokens = word_tokenize(line)
-    #print(tokens)
-    #return tokens
-    return []
+    tokens = word_tokenize(line)
+
+    return tokens
 
 
 def jieba_tokenize(line):
     # 使用jieba进行分词
     tokens = list(jieba1.cut(line.strip()))  # strip用于去除可能的空白字符
-    #print(tokens)
+
     return tokens
 
 def tokenize(line, mode):