demessifyme/doc2vec.py
def tokenize(text):
    """
    Split text into tokens, removing punctuation and replacing numbers with $number tokens.

    :param text: text to tokenize
    :return: list of tokens
    """
    for punc in punctuation:
        text = text.replace(punc, " ")
    # for number in "0123456789":
    #     text = text.replace(number, " number ")
    # tokens = word_tokenize(text)
    tokens = [("$number" if s.isnumeric() else s) for s in text.lower().split()]
    return tokens

    Split text into tokens, removing punctuation and replacing numbers with $number tokens.

    :param text: text to tokenize
    :return: list of tokens
    
Split text into tokens, removing punctuation and replacing numbers with $number tokens.
Split text into tokens removing punctuation and replacing numbers with $number tokens .
def tokenize(text):
    for punc in punctuation:
        text = text.replace(punc, " ")
    tokens = [("$number" if s.isnumeric() else s) for s in text.lower().split()]
    return tokens
def tokenize ( text ) : for punc in punctuation : text = text . replace ( punc , " " ) tokens = [ ( "$number" if s . isnumeric ( ) else s ) for s in text . lower ( ) . split ( ) ] return tokens
Split text into tokens, removing punctuation and replacing numbers with $number tokens.
Split text into tokens removing punctuation and replacing numbers with $number tokens .
"""
    Split text into tokens, removing punctuation and replacing numbers with $number tokens.

    :param text: text to tokenize
    :return: list of tokens
    """ # for number in "0123456789": #     text = text.replace(number, " number ") # tokens = word_tokenize(text)
list of tokens
list of tokens
text
text to tokenize
text to tokenize