models/train_classifier.py
def tokenize(text):
    """
    Takes text and return a tokenized, cleaned and lemmatized list
    :param
    text: string. input text
    :return:
    clean_tokens: list. List of relevant words lemmatized
    """
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())

    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    lemmatizer = WordNetLemmatizer()

    tokens = [lemmatizer.lemmatize(word) for word in word_tokenize(text) if word not in stopwords.words('english')]

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens

    Takes text and return a tokenized, cleaned and lemmatized list
    :param
    text: string. input text
    :return:
    clean_tokens: list. List of relevant words lemmatized
    
Takes text and return a tokenized, cleaned and lemmatized list
Takes text and return a tokenized cleaned and lemmatized list
def tokenize(text):
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())
    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")
    lemmatizer = WordNetLemmatizer()
    tokens = [lemmatizer.lemmatize(word) for word in word_tokenize(text) if word not in stopwords.words('english')]
    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)
    return clean_tokens
def tokenize ( text ) : text = re . sub ( r"[^a-zA-Z0-9]" , " " , text . lower ( ) ) url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' detected_urls = re . findall ( url_regex , text ) for url in detected_urls : text = text . replace ( url , "urlplaceholder" ) lemmatizer = WordNetLemmatizer ( ) tokens = [ lemmatizer . lemmatize ( word ) for word in word_tokenize ( text ) if word not in stopwords . words ( 'english' ) ] clean_tokens = [ ] for tok in tokens : clean_tok = lemmatizer . lemmatize ( tok ) . lower ( ) . strip ( ) clean_tokens . append ( clean_tok ) return clean_tokens
Takes text and return a tokenized, cleaned and lemmatized list
Takes text and return a tokenized cleaned and lemmatized list
"""
    Takes text and return a tokenized, cleaned and lemmatized list
    :param
    text: string. input text
    :return:
    clean_tokens: list. List of relevant words lemmatized
    """
list. List of relevant words lemmatized
list . List of relevant words lemmatized
text
string. input text
string . input text