discovery/discovery.py
def discover_with_related_search(self):
        '''url discovery using google related search'''

        next_urls = [(-1.0, url) for url in self.seed_urls]  # negate the scores to turn minheap into maxheap
        heapq.heapify(next_urls) # make next_urls be priority queue
        new_discovered_urls = [] # urls with relevant scores
        ranked_urls = [] # discovered urls with ranking scores

        while next_urls:
            seed = heapq.heappop(next_urls)[1]
            results =  self.searcher.search_related(seed) 
            for url in results:
                if url not in self.discovered_urls:
                    new_discovered_urls.append(url)
                    self.discovered_urls.add(url)

            print "Seed: ", seed, "Retrieved ", len(results), " related urls"

            # Rank the discovered urls
            new_seed_urls = []
            if new_discovered_urls:
                new_ranked_urls = self.ranker.rank(new_discovered_urls)
                self._save_ranked_urls(new_ranked_urls)
                ranked_urls.extend(new_ranked_urls)
                for url, score in new_ranked_urls:
                    if score>self.seed_threshold:
                        new_seed_urls.append(url)
                    if score>self.search_threshold:
                        heapq.heappush(next_urls,(-score, url))
                new_discovered_urls = []
                for url, score in new_ranked_urls: print url, score
            self.ranker.update_seeds(new_seed_urls)

            if len(self.discovered_urls)>300: break
url discovery using google related search
url discovery using google related search
url discovery using google related search
def discover_with_related_search(self):
        next_urls = [(-1.0, url) for url in self.seed_urls]  
        heapq.heapify(next_urls) 
        new_discovered_urls = [] 
        ranked_urls = [] 
        while next_urls:
            seed = heapq.heappop(next_urls)[1]
            results =  self.searcher.search_related(seed) 
            for url in results:
                if url not in self.discovered_urls:
                    new_discovered_urls.append(url)
                    self.discovered_urls.add(url)
            print "Seed: ", seed, "Retrieved ", len(results), " related urls"
            new_seed_urls = []
            if new_discovered_urls:
                new_ranked_urls = self.ranker.rank(new_discovered_urls)
                self._save_ranked_urls(new_ranked_urls)
                ranked_urls.extend(new_ranked_urls)
                for url, score in new_ranked_urls:
                    if score>self.seed_threshold:
                        new_seed_urls.append(url)
                    if score>self.search_threshold:
                        heapq.heappush(next_urls,(-score, url))
                new_discovered_urls = []
                for url, score in new_ranked_urls: print url, score
            self.ranker.update_seeds(new_seed_urls)
            if len(self.discovered_urls)>300: break
def discover_with_related_search ( self ) : next_urls = [ ( - 1.0 , url ) for url in self . seed_urls ] heapq . heapify ( next_urls ) new_discovered_urls = [ ] ranked_urls = [ ] while next_urls : seed = heapq . heappop ( next_urls ) [ 1 ] results = self . searcher . search_related ( seed ) for url in results : if url not in self . discovered_urls : new_discovered_urls . append ( url ) self . discovered_urls . add ( url ) print "Seed: " , seed , "Retrieved " , len ( results ) , " related urls" new_seed_urls = [ ] if new_discovered_urls : new_ranked_urls = self . ranker . rank ( new_discovered_urls ) self . _save_ranked_urls ( new_ranked_urls ) ranked_urls . extend ( new_ranked_urls ) for url , score in new_ranked_urls : if score > self . seed_threshold : new_seed_urls . append ( url ) if score > self . search_threshold : heapq . heappush ( next_urls , ( - score , url ) ) new_discovered_urls = [ ] for url , score in new_ranked_urls : print url , score self . ranker . update_seeds ( new_seed_urls ) if len ( self . discovered_urls ) > 300 : break
url discovery using google related search
url discovery using google related search
'''url discovery using google related search''' # negate the scores to turn minheap into maxheap # make next_urls be priority queue # urls with relevant scores # discovered urls with ranking scores # Rank the discovered urls
self