| import nltk | |
| import jieba | |
| import sudachipy | |
| import langid | |
| nltk.download('punkt') | |
| langid.set_languages(['en', 'zh', 'ja']) | |
| def split_text_into_sentences(text): | |
| if langid.classify(text)[0] == "en": | |
| sentences = nltk.tokenize.sent_tokenize(text) | |
| return sentences | |
| elif langid.classify(text)[0] == "zh": | |
| sentences = [] | |
| segs = jieba.cut(text, cut_all=False) | |
| segs = list(segs) | |
| start = 0 | |
| for i, seg in enumerate(segs): | |
| if seg in ["。", "！", "？", "……"]: | |
| sentences.append("".join(segs[start:i + 1])) | |
| start = i + 1 | |
| if start < len(segs): | |
| sentences.append("".join(segs[start:])) | |
| return sentences | |
| elif langid.classify(text)[0] == "ja": | |
| sentences = [] | |
| tokenizer = sudachipy.Dictionary().create() | |
| tokens = tokenizer.tokenize(text) | |
| current_sentence = "" | |
| for token in tokens: | |
| current_sentence += token.surface() | |
| if token.part_of_speech()[0] == "補助記号" and token.part_of_speech()[1] == "句点": | |
| sentences.append(current_sentence) | |
| current_sentence = "" | |
| if current_sentence: | |
| sentences.append(current_sentence) | |
| return sentences | |
| raise RuntimeError("It is impossible to reach here.") |