search_apis/search_apis.py
def select_subset(self, urls):
        """
        Each page might contain thousand of external urls which pollute the results, so we only keep a fixed number of links from each page
        How this works:
            - Pick one url in each site  
            - If not yet reaching max, select random urls
        Returns:
            - list of urls
        """
        if len(urls)<=self.max_urls:
            return urls

        results = []
        """
        cur = urls
        while len(results)<self.max_urls:
            sites = set()
            next = []
            for url in cur: 
                site = URLUtility.get_host(url)
                if site not in sites:
                    sites.add(site)
                    results.append(url)
                else:
                    next.append(url)
                if len(results) == self.max_urls:
                    break
            cur = next
        """
        sites = set()
        for url in urls: 
            site = URLUtility.get_host(url)
            if site not in sites:
                sites.add(site)
                results.append(url)
            if len(results) == self.max_urls:
                break

        return results

        Each page might contain thousand of external urls which pollute the results, so we only keep a fixed number of links from each page
        How this works:
            - Pick one url in each site  
            - If not yet reaching max, select random urls
        Returns:
            - list of urls
        
Each page might contain thousand of external urls which pollute the results, so we only keep a fixed number of links from each page
How this works:
Pick one url in each site
If not yet reaching max, select random urls
Each page might contain thousand of external urls which pollute the results so we only keep a fixed number of links from each page How this works : Pick one url in each site If not yet reaching max select random urls
def select_subset(self, urls):
        if len(urls)<=self.max_urls:
            return urls
        results = []
        sites = set()
        for url in urls: 
            site = URLUtility.get_host(url)
            if site not in sites:
                sites.add(site)
                results.append(url)
            if len(results) == self.max_urls:
                break
        return results
def select_subset ( self , urls ) : if len ( urls ) <= self . max_urls : return urls results = [ ] """
        cur = urls
        while len(results)<self.max_urls:
            sites = set()
            next = []
            for url in cur: 
                site = URLUtility.get_host(url)
                if site not in sites:
                    sites.add(site)
                    results.append(url)
                else:
                    next.append(url)
                if len(results) == self.max_urls:
                    break
            cur = next
        """ sites = set ( ) for url in urls : site = URLUtility . get_host ( url ) if site not in sites : sites . add ( site ) results . append ( url ) if len ( results ) == self . max_urls : break return results
Each page might contain thousand of external urls which pollute the results, so we only keep a fixed number of links from each page
How this works:
 Pick one url in each site
 If not yet reaching max, select random urls
Each page might contain thousand of external urls which pollute the results so we only keep a fixed number of links from each page How this works : Pick one url in each site If not yet reaching max select random urls
"""
        Each page might contain thousand of external urls which pollute the results, so we only keep a fixed number of links from each page
        How this works:
            - Pick one url in each site  
            - If not yet reaching max, select random urls
        Returns:
            - list of urls
        """ """
        cur = urls
        while len(results)<self.max_urls:
            sites = set()
            next = []
            for url in cur: 
                site = URLUtility.get_host(url)
                if site not in sites:
                    sites.add(site)
                    results.append(url)
                else:
                    next.append(url)
                if len(results) == self.max_urls:
                    break
            cur = next
        """
list of urls
list of urls
self
urls