File size: 1,136 Bytes
dbaf7c6 cf27cda dbaf7c6 414dfdc dbaf7c6 cf27cda dbaf7c6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
# -*- coding: utf-8 -*-
from fastapi import APIRouter
from pythainlp.tokenize import word_tokenize as py_word_tokenize, subword_tokenize as py_subword_tokenize
from enum import Enum
from typing import List, Optional
from pydantic import BaseModel
router = APIRouter()
class SentTokenizeEngine(str, Enum):
whitespace = "whitespace"
whitespace_newline = "whitespace+newline"
class WordTokenizeEngine(str, Enum):
newmm = "newmm"
longest = "longest"
class SubwordTokenizeEngine(str, Enum):
tcc = "tcc"
etcc = "etcc"
class WordTokenizeResponse(BaseModel):
words: List[str] = []
class SubwordTokenizeResponse(BaseModel):
subwords: List[str] = []
@router.post('/word_tokenize', response_model=WordTokenizeResponse)
def word_tokenize(text: str, engine: WordTokenizeEngine = "newmm"):
"""
Word tokenize
"""
return {"words": py_word_tokenize(text=text, engine=engine)}
@router.post('/subword_tokenize', response_model=SubwordTokenizeResponse)
def subword_tokenize(text: str, engine: SubwordTokenizeEngine = "tcc"):
return {"subwords": py_subword_tokenize(text=text, engine=engine)}
|