discovery/collect_candidates.py
def extract_keywords(sites, k=10):
    """
    Extract top k most frequent keywords
    """
    stop = stopwords.words('english')
    counter = Counter()
    for site in sites:
        for p in site:
            text = p.get_text('meta')
            text = URLUtility.clean_text(text) 
            words = word_tokenize(text)
            words = [word for word in words if word not in stop and len(word)>2]
            counter += Counter(words)
    
    # Get the topk words
    counter = [(counter[w], w) for w in counter if counter[w]>1] # convert to array
    heapq.heapify(counter)
    topk = heapq.nlargest(k, counter)
    print "Top extracted keywords: ", topk
    return [w[1] for w in topk]

    Extract top k most frequent keywords
    
Extract top k most frequent keywords
Extract top k most frequent keywords
def extract_keywords(sites, k=10):
    stop = stopwords.words('english')
    counter = Counter()
    for site in sites:
        for p in site:
            text = p.get_text('meta')
            text = URLUtility.clean_text(text) 
            words = word_tokenize(text)
            words = [word for word in words if word not in stop and len(word)>2]
            counter += Counter(words)
    counter = [(counter[w], w) for w in counter if counter[w]>1] 
    heapq.heapify(counter)
    topk = heapq.nlargest(k, counter)
    print "Top extracted keywords: ", topk
    return [w[1] for w in topk]
def extract_keywords ( sites , k = 10 ) : stop = stopwords . words ( 'english' ) counter = Counter ( ) for site in sites : for p in site : text = p . get_text ( 'meta' ) text = URLUtility . clean_text ( text ) words = word_tokenize ( text ) words = [ word for word in words if word not in stop and len ( word ) > 2 ] counter += Counter ( words ) counter = [ ( counter [ w ] , w ) for w in counter if counter [ w ] > 1 ] heapq . heapify ( counter ) topk = heapq . nlargest ( k , counter ) print "Top extracted keywords: " , topk return [ w [ 1 ] for w in topk ]
Extract top k most frequent keywords
Extract top k most frequent keywords
"""
    Extract top k most frequent keywords
    """ # Get the topk words # convert to array
sites
k