webapp/scrape.py
def pos_text(text):
    """ This function uses Spacy, open source NLP
        toolkit to find the most frequent words
        and parts of speech and return only nouns and 
        verbs for word cloud
    """
    nlp = spacy.load('en')
    doc = nlp(text)
    # all tokens that arent stop words or punctuations
    words = [token.text.encode('ascii', 'ignore') for token in doc if token.is_stop != True and token.is_punct != True]

    # noun tokens that arent stop words or punctuations
    final_tokens = [token.text.encode('ascii', 'ignore') for token in doc if token.is_stop != True and \
                    token.is_punct != True and (token.pos_ == "NOUN" or token.pos_ == "VERB")]

    # frequency dictionary for all tokens
    word_freq = Counter(words)

    #top 100 words to display in wordcloud which are noun or verb
    #frequency will be used to show big/small words in wordcloud
    final_tokens_freq = Counter(final_tokens)
    result = final_tokens_freq.most_common(config.config["MAX_FREQUENCY"])
    #print result
    return result
 This function uses Spacy, open source NLP
        toolkit to find the most frequent words
        and parts of speech and return only nouns and 
        verbs for word cloud
    
This function uses Spacy, open source NLP
toolkit to find the most frequent words
and parts of speech and return only nouns and
verbs for word cloud
This function uses Spacy open source NLP toolkit to find the most frequent words and parts of speech and return only nouns and verbs for word cloud
def pos_text(text):
    nlp = spacy.load('en')
    doc = nlp(text)
    words = [token.text.encode('ascii', 'ignore') for token in doc if token.is_stop != True and token.is_punct != True]
    final_tokens = [token.text.encode('ascii', 'ignore') for token in doc if token.is_stop != True and \
                    token.is_punct != True and (token.pos_ == "NOUN" or token.pos_ == "VERB")]
    word_freq = Counter(words)
    final_tokens_freq = Counter(final_tokens)
    result = final_tokens_freq.most_common(config.config["MAX_FREQUENCY"])
    return result
def pos_text ( text ) : nlp = spacy . load ( 'en' ) doc = nlp ( text ) words = [ token . text . encode ( 'ascii' , 'ignore' ) for token in doc if token . is_stop != True and token . is_punct != True ] final_tokens = [ token . text . encode ( 'ascii' , 'ignore' ) for token in doc if token . is_stop != True and token . is_punct != True and ( token . pos_ == "NOUN" or token . pos_ == "VERB" ) ] word_freq = Counter ( words ) final_tokens_freq = Counter ( final_tokens ) result = final_tokens_freq . most_common ( config . config [ "MAX_FREQUENCY" ] ) return result
This function uses Spacy, open source NLP
toolkit to find the most frequent words
and parts of speech and return only nouns and
verbs for word cloud
This function uses Spacy open source NLP toolkit to find the most frequent words and parts of speech and return only nouns and verbs for word cloud
""" This function uses Spacy, open source NLP
        toolkit to find the most frequent words
        and parts of speech and return only nouns and 
        verbs for word cloud
    """ # all tokens that arent stop words or punctuations # noun tokens that arent stop words or punctuations # frequency dictionary for all tokens #top 100 words to display in wordcloud which are noun or verb #frequency will be used to show big/small words in wordcloud #print result
text