modules/NLP.py
def digest(self, text):
        normalized = self.normalizer.normalize(text)
        parsed = TextBlob(normalized, np_extractor=self.np_ex)
        counted_noun_phrases = parsed.noun_phrases
        abbreviations = self.abbreviations_finder.digest(parsed)
        '''make sure defined acronym are used as noun phrases '''
        for abbr in abbreviations:
            if abbr['long'].lower() not in counted_noun_phrases:
                counted_noun_phrases.append(abbr['long'].lower())
        '''improved singularisation still needs refinement'''
        # singular_counted_noun_phrases = []
        # for np in counted_noun_phrases:
        #     if not (np.endswith('sis') or np.endswith('ess')):
        #         singular_counted_noun_phrases.append(singularize(np))
        #     else:
        #         singular_counted_noun_phrases.append(np)
        # singular_counted_noun_phrases = Counter(singular_counted_noun_phrases)
        counted_noun_phrases = Counter(counted_noun_phrases)
        base_noun_phrases = counted_noun_phrases.keys()
        '''remove plurals with appended s'''
        for np in counted_noun_phrases.keys():
            if np + 's' in counted_noun_phrases:
                counted_noun_phrases[np] += counted_noun_phrases[np + 's']
                del counted_noun_phrases[np + 's']
        '''increase count of shorter form with longer form'''
        for abbr in abbreviations:
            short = abbr['short'].lower()
            if short in counted_noun_phrases:
                counted_noun_phrases[abbr['long'].lower()] += counted_noun_phrases[short]
                del counted_noun_phrases[short]

        # count substrings occurrences as well
        for k in counted_noun_phrases:
            for s in counted_noun_phrases:
                if k != s and k in s:
                    counted_noun_phrases[k] += 1
        return dict(chunks=base_noun_phrases,
                    recurring_chunks=[i for i, k in counted_noun_phrases.items() if k > 1],
                    top_chunks=[i[0] for i in counted_noun_phrases.most_common(5) if i[1] > 1],
                    abbreviations=abbreviations)
make sure defined acronym are used as noun phrases 
make sure defined acronym are used as noun phrases
make sure defined acronym are used as noun phrases
def digest(self, text):
        normalized = self.normalizer.normalize(text)
        parsed = TextBlob(normalized, np_extractor=self.np_ex)
        counted_noun_phrases = parsed.noun_phrases
        abbreviations = self.abbreviations_finder.digest(parsed)
        for abbr in abbreviations:
            if abbr['long'].lower() not in counted_noun_phrases:
                counted_noun_phrases.append(abbr['long'].lower())
        counted_noun_phrases = Counter(counted_noun_phrases)
        base_noun_phrases = counted_noun_phrases.keys()
        for np in counted_noun_phrases.keys():
            if np + 's' in counted_noun_phrases:
                counted_noun_phrases[np] += counted_noun_phrases[np + 's']
                del counted_noun_phrases[np + 's']
        for abbr in abbreviations:
            short = abbr['short'].lower()
            if short in counted_noun_phrases:
                counted_noun_phrases[abbr['long'].lower()] += counted_noun_phrases[short]
                del counted_noun_phrases[short]
        for k in counted_noun_phrases:
            for s in counted_noun_phrases:
                if k != s and k in s:
                    counted_noun_phrases[k] += 1
        return dict(chunks=base_noun_phrases,
                    recurring_chunks=[i for i, k in counted_noun_phrases.items() if k > 1],
                    top_chunks=[i[0] for i in counted_noun_phrases.most_common(5) if i[1] > 1],
                    abbreviations=abbreviations)
def digest ( self , text ) : normalized = self . normalizer . normalize ( text ) parsed = TextBlob ( normalized , np_extractor = self . np_ex ) counted_noun_phrases = parsed . noun_phrases abbreviations = self . abbreviations_finder . digest ( parsed ) for abbr in abbreviations : if abbr [ 'long' ] . lower ( ) not in counted_noun_phrases : counted_noun_phrases . append ( abbr [ 'long' ] . lower ( ) ) '''improved singularisation still needs refinement''' counted_noun_phrases = Counter ( counted_noun_phrases ) base_noun_phrases = counted_noun_phrases . keys ( ) '''remove plurals with appended s''' for np in counted_noun_phrases . keys ( ) : if np + 's' in counted_noun_phrases : counted_noun_phrases [ np ] += counted_noun_phrases [ np + 's' ] del counted_noun_phrases [ np + 's' ] '''increase count of shorter form with longer form''' for abbr in abbreviations : short = abbr [ 'short' ] . lower ( ) if short in counted_noun_phrases : counted_noun_phrases [ abbr [ 'long' ] . lower ( ) ] += counted_noun_phrases [ short ] del counted_noun_phrases [ short ] for k in counted_noun_phrases : for s in counted_noun_phrases : if k != s and k in s : counted_noun_phrases [ k ] += 1 return dict ( chunks = base_noun_phrases , recurring_chunks = [ i for i , k in counted_noun_phrases . items ( ) if k > 1 ] , top_chunks = [ i [ 0 ] for i in counted_noun_phrases . most_common ( 5 ) if i [ 1 ] > 1 ] , abbreviations = abbreviations )
make sure defined acronym are used as noun phrases
make sure defined acronym are used as noun phrases
'''make sure defined acronym are used as noun phrases ''' '''improved singularisation still needs refinement''' # singular_counted_noun_phrases = [] # for np in counted_noun_phrases: #     if not (np.endswith('sis') or np.endswith('ess')): #         singular_counted_noun_phrases.append(singularize(np)) #     else: #         singular_counted_noun_phrases.append(np) # singular_counted_noun_phrases = Counter(singular_counted_noun_phrases) '''remove plurals with appended s''' '''increase count of shorter form with longer form''' # count substrings occurrences as well
self
text