discovery/site_discovery.py
def run_mix_search(self, ranking, selection=None, online=True, max_results=50, seed_keyword="gun", search="kw", iters=5, representation='body', negative_file=None):
        """
        seed_sites: urls that are used for search
        selected_urls: urls that were used for search

        Only top-ranked urls will become seed urls

        Important Args:
            ranking: a ranking method
            max_results: Maximum number of results to return in related and keyword search
        """
        max_pages = 1 # Always use single page to represent a website
        train_sites = self.fetcher.fetch_sites(self.train_urls, max_pages, selection, online)

        if negative_file: # (random) reliably negative examples
            neg_urls = URLUtility.load_urls(negative_file)
            neg_urls = neg_urls[:200]
        else:
            neg_urls = []
        print "neg_urls: ", len(neg_urls)
        neg_sites = self.fetcher.fetch_sites(neg_urls, 1, None, online)
        ranker = Ranker(train_sites, representation, ranking, neg_sites)

        # Data
        scores = [] # Avoid exception when iters=0
        #seed_sites = self.train_urls # topk urls from each search batch
        seed_sites = train_sites # topk urls from each search batch
        selected_urls = {} # avoid searching with these urls again
        selected_urls['kw'] = set()
        selected_urls['bl'] = set()
        selected_urls['rl'] = set()
        selected_urls['fw'] = set()
        results = [] # Search results for ranking 
        urls = set() # Avoid fetch and rank these urls again
        sites = set() # used to compute reward 
        

        # Hyperparameters
        #max_numb_pages = 12000 # stop condition
        max_numb_pages = 51000 # stop condition
        #iters = 500
        iters = 2000
        k = 20 #  number of pages from the newly discovered pages to be added to the seed list
        max_kw = 20 # maximum number of keywords to select from the seed pages
        self.searcher.set_max_keywords(max_kw)

        # Initialize Search Operator Selection Strategy
        count = {} # Count number of results yeilded by each search operator
        count['bl'] = count['kw'] = count['rl'] = count['fw'] = 0 
        count['bl'] = 20000 # never choose this
        #ucb = UCB1(['rl', 'bl', 'kw'])
        ucb = UCB1(['rl', 'bl', 'kw', 'fw'])
        
        site_mode = False # used in get_top_ranked_urls function 

        for i in xrange(iters):
            t = time.time()

            print "Searching... ", len(seed_sites), "  seed urls"
            searchop = self.select_searchop(count, search, ucb)

            if searchop=='rl' or searchop=='bl':
                site_mode = True
            else:
                site_mode = False

            print "\n Iteration ", i, searchop 
            new_urls = self.searcher.search(seed_sites, \
                                            searchop, seed_keyword=seed_keyword, \
                                            max_results=max_results)
            new_urls = [url for url in new_urls if url not in urls]

            if len(new_urls)==0:
                print "Searcher found 0 url"
                seed_sites = self.get_top_ranked_urls(scores, k, selected_urls[searchop], site_mode) # Backlink search and related search only use host name to form the query. searchop!='kw' <-> searchop=='bl' or searchop=='rl'
                if len(seed_sites) == 0:
                    print "Stop. Running out of seeds"
                    break
                else:
                    continue 

            urls.update(new_urls)

            print "Time to search ", i, ": ", time.time()-t
            t = time.time()

            new_sites = self.fetcher.fetch_sites(new_urls, max_pages, selection, online)

            print "Time to fetch ", i, ": ", time.time()-t
            t = time.time()

            temp = len(results)
            results.extend(new_sites)
            print "Size of candidates (after): ", len(results)
            print "Number of new candidates (after): ", len(results) - temp
            scores = ranker.rank(results)
            if len(scores)>=max_numb_pages: 
                print "Stop. Retrieved ", max_numb_pages, " pages"
                break
            #seed_sites = self.get_top_ranked_urls(scores, k, selected_urls[searchop])
            seed_sites = self.get_top_ranked_urls(scores, k, selected_urls[searchop], site_mode) # Backlink search and related search only use host name to form the query. searchop!='kw' <-> searchop=='bl' or searchop=='rl'
            if len(seed_sites) == 0:
                print "Stop. Running out of seeds"
                break
            self.save_urls(new_sites, i)
            
            # Update information from the search results to the operation selector
            count[searchop] += len(new_urls)
            if (search == 'bandit') and new_sites:
                reward = self.get_reward(scores, new_sites, sites)
                print "UCB Rewards", searchop, reward
                ucb.update(searchop, reward, len(new_sites))
                sites.update([s.get_host() for s in new_sites])
            print "Time to rank ", i, ": ", time.time()-t

        self.save_scores(scores)

        seed_sites: urls that are used for search
        selected_urls: urls that were used for search

        Only top-ranked urls will become seed urls

        Important Args:
            ranking: a ranking method
            max_results: Maximum number of results to return in related and keyword search
        
urls that are used for search
selected_urls: urls that were used for search

Only top-ranked urls will become seed urls

Important Args:
ranking: a ranking method
max_results: Maximum number of results to return in related and keyword search
urls that are used for search selected_urls : urls that were used for search Only top - ranked urls will become seed urls Important Args : ranking : a ranking method max_results : Maximum number of results to return in related and keyword search
def run_mix_search(self, ranking, selection=None, online=True, max_results=50, seed_keyword="gun", search="kw", iters=5, representation='body', negative_file=None):
        max_pages = 1 
        train_sites = self.fetcher.fetch_sites(self.train_urls, max_pages, selection, online)
        if negative_file: 
            neg_urls = URLUtility.load_urls(negative_file)
            neg_urls = neg_urls[:200]
        else:
            neg_urls = []
        print "neg_urls: ", len(neg_urls)
        neg_sites = self.fetcher.fetch_sites(neg_urls, 1, None, online)
        ranker = Ranker(train_sites, representation, ranking, neg_sites)
        scores = [] 
        seed_sites = train_sites 
        selected_urls = {} 
        selected_urls['kw'] = set()
        selected_urls['bl'] = set()
        selected_urls['rl'] = set()
        selected_urls['fw'] = set()
        results = [] 
        urls = set() 
        sites = set() 
        max_numb_pages = 51000 
        iters = 2000
        k = 20 
        max_kw = 20 
        self.searcher.set_max_keywords(max_kw)
        count = {} 
        count['bl'] = count['kw'] = count['rl'] = count['fw'] = 0 
        count['bl'] = 20000 
        ucb = UCB1(['rl', 'bl', 'kw', 'fw'])
        site_mode = False 
        for i in xrange(iters):
            t = time.time()
            print "Searching... ", len(seed_sites), "  seed urls"
            searchop = self.select_searchop(count, search, ucb)
            if searchop=='rl' or searchop=='bl':
                site_mode = True
            else:
                site_mode = False
            print "\n Iteration ", i, searchop 
            new_urls = self.searcher.search(seed_sites, \
                                            searchop, seed_keyword=seed_keyword, \
                                            max_results=max_results)
            new_urls = [url for url in new_urls if url not in urls]
            if len(new_urls)==0:
                print "Searcher found 0 url"
                seed_sites = self.get_top_ranked_urls(scores, k, selected_urls[searchop], site_mode) 
                if len(seed_sites) == 0:
                    print "Stop. Running out of seeds"
                    break
                else:
                    continue 
            urls.update(new_urls)
            print "Time to search ", i, ": ", time.time()-t
            t = time.time()
            new_sites = self.fetcher.fetch_sites(new_urls, max_pages, selection, online)
            print "Time to fetch ", i, ": ", time.time()-t
            t = time.time()
            temp = len(results)
            results.extend(new_sites)
            print "Size of candidates (after): ", len(results)
            print "Number of new candidates (after): ", len(results) - temp
            scores = ranker.rank(results)
            if len(scores)>=max_numb_pages: 
                print "Stop. Retrieved ", max_numb_pages, " pages"
                break
            seed_sites = self.get_top_ranked_urls(scores, k, selected_urls[searchop], site_mode) 
            if len(seed_sites) == 0:
                print "Stop. Running out of seeds"
                break
            self.save_urls(new_sites, i)
            count[searchop] += len(new_urls)
            if (search == 'bandit') and new_sites:
                reward = self.get_reward(scores, new_sites, sites)
                print "UCB Rewards", searchop, reward
                ucb.update(searchop, reward, len(new_sites))
                sites.update([s.get_host() for s in new_sites])
            print "Time to rank ", i, ": ", time.time()-t
        self.save_scores(scores)
def run_mix_search ( self , ranking , selection = None , online = True , max_results = 50 , seed_keyword = "gun" , search = "kw" , iters = 5 , representation = 'body' , negative_file = None ) : max_pages = 1 train_sites = self . fetcher . fetch_sites ( self . train_urls , max_pages , selection , online ) if negative_file : neg_urls = URLUtility . load_urls ( negative_file ) neg_urls = neg_urls [ : 200 ] else : neg_urls = [ ] print "neg_urls: " , len ( neg_urls ) neg_sites = self . fetcher . fetch_sites ( neg_urls , 1 , None , online ) ranker = Ranker ( train_sites , representation , ranking , neg_sites ) scores = [ ] seed_sites = train_sites selected_urls = { } selected_urls [ 'kw' ] = set ( ) selected_urls [ 'bl' ] = set ( ) selected_urls [ 'rl' ] = set ( ) selected_urls [ 'fw' ] = set ( ) results = [ ] urls = set ( ) sites = set ( ) max_numb_pages = 51000 iters = 2000 k = 20 max_kw = 20 self . searcher . set_max_keywords ( max_kw ) count = { } count [ 'bl' ] = count [ 'kw' ] = count [ 'rl' ] = count [ 'fw' ] = 0 count [ 'bl' ] = 20000 ucb = UCB1 ( [ 'rl' , 'bl' , 'kw' , 'fw' ] ) site_mode = False for i in xrange ( iters ) : t = time . time ( ) print "Searching... " , len ( seed_sites ) , "  seed urls" searchop = self . select_searchop ( count , search , ucb ) if searchop == 'rl' or searchop == 'bl' : site_mode = True else : site_mode = False print "\n Iteration " , i , searchop new_urls = self . searcher . search ( seed_sites , searchop , seed_keyword = seed_keyword , max_results = max_results ) new_urls = [ url for url in new_urls if url not in urls ] if len ( new_urls ) == 0 : print "Searcher found 0 url" seed_sites = self . get_top_ranked_urls ( scores , k , selected_urls [ searchop ] , site_mode ) if len ( seed_sites ) == 0 : print "Stop. Running out of seeds" break else : continue urls . update ( new_urls ) print "Time to search " , i , ": " , time . time ( ) - t t = time . time ( ) new_sites = self . fetcher . fetch_sites ( new_urls , max_pages , selection , online ) print "Time to fetch " , i , ": " , time . time ( ) - t t = time . time ( ) temp = len ( results ) results . extend ( new_sites ) print "Size of candidates (after): " , len ( results ) print "Number of new candidates (after): " , len ( results ) - temp scores = ranker . rank ( results ) if len ( scores ) >= max_numb_pages : print "Stop. Retrieved " , max_numb_pages , " pages" break seed_sites = self . get_top_ranked_urls ( scores , k , selected_urls [ searchop ] , site_mode ) if len ( seed_sites ) == 0 : print "Stop. Running out of seeds" break self . save_urls ( new_sites , i ) count [ searchop ] += len ( new_urls ) if ( search == 'bandit' ) and new_sites : reward = self . get_reward ( scores , new_sites , sites ) print "UCB Rewards" , searchop , reward ucb . update ( searchop , reward , len ( new_sites ) ) sites . update ( [ s . get_host ( ) for s in new_sites ] ) print "Time to rank " , i , ": " , time . time ( ) - t self . save_scores ( scores )
seed_sites: urls that are used for search
selected_urls: urls that were used for search
seed_sites : urls that are used for search selected_urls : urls that were used for search
"""
        seed_sites: urls that are used for search
        selected_urls: urls that were used for search

        Only top-ranked urls will become seed urls

        Important Args:
            ranking: a ranking method
            max_results: Maximum number of results to return in related and keyword search
        """ # Always use single page to represent a website # (random) reliably negative examples # Data # Avoid exception when iters=0 #seed_sites = self.train_urls # topk urls from each search batch # topk urls from each search batch # avoid searching with these urls again # Search results for ranking  # Avoid fetch and rank these urls again # used to compute reward  # Hyperparameters #max_numb_pages = 12000 # stop condition # stop condition #iters = 500 #  number of pages from the newly discovered pages to be added to the seed list # maximum number of keywords to select from the seed pages # Initialize Search Operator Selection Strategy # Count number of results yeilded by each search operator # never choose this #ucb = UCB1(['rl', 'bl', 'kw']) # used in get_top_ranked_urls function  # Backlink search and related search only use host name to form the query. searchop!='kw' <-> searchop=='bl' or searchop=='rl' #seed_sites = self.get_top_ranked_urls(scores, k, selected_urls[searchop]) # Backlink search and related search only use host name to form the query. searchop!='kw' <-> searchop=='bl' or searchop=='rl' # Update information from the search results to the operation selector
self
ranking
selection
online
max_results
seed_keyword
search
iters
representation
negative_file