lda/leipzig/2_lda_module/lda_helpers.py
def tokenize(text, rm_ascii=False):
    """Tokenize and rm stopwords. The Gensim `simple_preprocess` will work fine
    here becuase the Greek text has already been aggressively cleaned up.
    https://radimrehurek.com/gensim/utils.html#gensim.utils.simple_preprocess
    """
    if rm_ascii:
        text = [char for char in text if char not in ascii_str]
        text = ''.join(text)
    tokens = [token for token in simple_preprocess(text, deacc=PREPROCESS_DEACCENT, min_len=TOK_MIN, max_len=TOK_MAX)]
    return [token for token in tokens if token not in STOPS_LIST]
Tokenize and rm stopwords. The Gensim `simple_preprocess` will work fine
    here becuase the Greek text has already been aggressively cleaned up.
    https://radimrehurek.com/gensim/utils.html#gensim.utils.simple_preprocess
    
Tokenize and rm stopwords. The Gensim `simple_preprocess` will work fine
here becuase the Greek text has already been aggressively cleaned up.
Tokenize and rm stopwords . The Gensim ` simple_preprocess ` will work fine here becuase the Greek text has already been aggressively cleaned up .
def tokenize(text, rm_ascii=False):
    if rm_ascii:
        text = [char for char in text if char not in ascii_str]
        text = ''.join(text)
    tokens = [token for token in simple_preprocess(text, deacc=PREPROCESS_DEACCENT, min_len=TOK_MIN, max_len=TOK_MAX)]
    return [token for token in tokens if token not in STOPS_LIST]
def tokenize ( text , rm_ascii = False ) : if rm_ascii : text = [ char for char in text if char not in ascii_str ] text = '' . join ( text ) tokens = [ token for token in simple_preprocess ( text , deacc = PREPROCESS_DEACCENT , min_len = TOK_MIN , max_len = TOK_MAX ) ] return [ token for token in tokens if token not in STOPS_LIST ]
Tokenize and rm stopwords.
Tokenize and rm stopwords .
"""Tokenize and rm stopwords. The Gensim `simple_preprocess` will work fine
    here becuase the Greek text has already been aggressively cleaned up.
    https://radimrehurek.com/gensim/utils.html#gensim.utils.simple_preprocess
    """
text
rm_ascii