search_apis/search_apis.py
def _extract_keywords(self, sites, k=10):
        """
        Extract top k most frequent keywords. Skip ones that were selected.
        """
        stop = stopwords.words('english')
        counter = Counter()
        for site in sites:
            for p in site:
                text = p.get_text('meta')
                text = URLUtility.clean_text(text) 
                words = nltk.word_tokenize(text)
                words = [word for word in words if word not in stop and len(word)>2]
                bigram_words = [words[i] + ' ' + words[i+1] for i in xrange(len(words)-1)]
                counter += Counter(words + bigram_words)
        
        # Get the topk words
        """
        counter = [(counter[w], w) for w in counter if counter[w]>1] # convert to array
        heapq.heapify(counter)
        topk = heapq.nlargest(k, counter)
        return [w[1] for w in topk]
        """
        top_words = counter.most_common(k + len(self.keywords)) 
        result = [] # list of keywords to return
        i = 0
        while len(result)<k and i<len(top_words):
            if top_words[i][0] not in self.keywords:
                result.append(top_words[i][0])
                self.keywords.add(top_words[i][0])
            i += 1
        print "    List of selected keywords: ", result
        return result

        Extract top k most frequent keywords. Skip ones that were selected.
        
Extract top k most frequent keywords. Skip ones that were selected.
Extract top k most frequent keywords . Skip ones that were selected .
def _extract_keywords(self, sites, k=10):
        stop = stopwords.words('english')
        counter = Counter()
        for site in sites:
            for p in site:
                text = p.get_text('meta')
                text = URLUtility.clean_text(text) 
                words = nltk.word_tokenize(text)
                words = [word for word in words if word not in stop and len(word)>2]
                bigram_words = [words[i] + ' ' + words[i+1] for i in xrange(len(words)-1)]
                counter += Counter(words + bigram_words)
        top_words = counter.most_common(k + len(self.keywords)) 
        result = [] 
        i = 0
        while len(result)<k and i<len(top_words):
            if top_words[i][0] not in self.keywords:
                result.append(top_words[i][0])
                self.keywords.add(top_words[i][0])
            i += 1
        print "    List of selected keywords: ", result
        return result
def _extract_keywords ( self , sites , k = 10 ) : stop = stopwords . words ( 'english' ) counter = Counter ( ) for site in sites : for p in site : text = p . get_text ( 'meta' ) text = URLUtility . clean_text ( text ) words = nltk . word_tokenize ( text ) words = [ word for word in words if word not in stop and len ( word ) > 2 ] bigram_words = [ words [ i ] + ' ' + words [ i + 1 ] for i in xrange ( len ( words ) - 1 ) ] counter += Counter ( words + bigram_words ) """
        counter = [(counter[w], w) for w in counter if counter[w]>1] # convert to array
        heapq.heapify(counter)
        topk = heapq.nlargest(k, counter)
        return [w[1] for w in topk]
        """ top_words = counter . most_common ( k + len ( self . keywords ) ) result = [ ] i = 0 while len ( result ) < k and i < len ( top_words ) : if top_words [ i ] [ 0 ] not in self . keywords : result . append ( top_words [ i ] [ 0 ] ) self . keywords . add ( top_words [ i ] [ 0 ] ) i += 1 print "    List of selected keywords: " , result return result
Extract top k most frequent keywords.
Extract top k most frequent keywords .
"""
        Extract top k most frequent keywords. Skip ones that were selected.
        """ # Get the topk words """
        counter = [(counter[w], w) for w in counter if counter[w]>1] # convert to array
        heapq.heapify(counter)
        topk = heapq.nlargest(k, counter)
        return [w[1] for w in topk]
        """ # list of keywords to return
self
sites
k