utils4text.py
def token_lemmatize(token, lemmatizer):
    """
    Lemmatize a token to convert a token back to its root form.
    When dealing with punctuation marks or emojis, simply return them as usual.

    Arg:
        token: A word in the string type.
        lemmatizer: The object from wordnet lemmatizer.
    Return:
        token in its root form.
    """
    if token == 'NUM':
        # keep NUM token as usual
        return token
    elif token in string.punctuation:
        # keep punctuation marks as usual
        return token
    elif token in UNICODE_EMOJI:
        # keep emojis
        return token
    elif token.isalpha(): 
        # consider English words
        token, tag = pos_tag([token])[0][0], pos_tag([token])[0][1]
        return lemmatizer.lemmatize(token, get_wordnet_pos(tag))
    # else:
    #     # transform those nonwords as the token NOWORD
    #     token = 'NONWORD'
    #     return token

    Lemmatize a token to convert a token back to its root form.
    When dealing with punctuation marks or emojis, simply return them as usual.

    Arg:
        token: A word in the string type.
        lemmatizer: The object from wordnet lemmatizer.
    Return:
        token in its root form.
    
Lemmatize a token to convert a token back to its root form.
When dealing with punctuation marks or emojis, simply return them as usual.

A word in the string type.
lemmatizer: The object from wordnet lemmatizer.
Return:
token in its root form.
Lemmatize a token to convert a token back to its root form . When dealing with punctuation marks or emojis simply return them as usual . A word in the string type . lemmatizer : The object from wordnet lemmatizer . Return : token in its root form .
def token_lemmatize(token, lemmatizer):
    if token == 'NUM':
        return token
    elif token in string.punctuation:
        return token
    elif token in UNICODE_EMOJI:
        return token
    elif token.isalpha(): 
        token, tag = pos_tag([token])[0][0], pos_tag([token])[0][1]
        return lemmatizer.lemmatize(token, get_wordnet_pos(tag))
def token_lemmatize ( token , lemmatizer ) : if token == 'NUM' : return token elif token in string . punctuation : return token elif token in UNICODE_EMOJI : return token elif token . isalpha ( ) : token , tag = pos_tag ( [ token ] ) [ 0 ] [ 0 ] , pos_tag ( [ token ] ) [ 0 ] [ 1 ] return lemmatizer . lemmatize ( token , get_wordnet_pos ( tag ) )
Lemmatize a token to convert a token back to its root form.
Lemmatize a token to convert a token back to its root form .
"""
    Lemmatize a token to convert a token back to its root form.
    When dealing with punctuation marks or emojis, simply return them as usual.

    Arg:
        token: A word in the string type.
        lemmatizer: The object from wordnet lemmatizer.
    Return:
        token in its root form.
    """ # keep NUM token as usual # keep punctuation marks as usual # keep emojis # consider English words # else: #     # transform those nonwords as the token NOWORD #     token = 'NONWORD' #     return token
token
lemmatizer