utils/multiproc_fetcher.py
def fetch_sites(self, urls, max_pages=1, selection="random", online=True, allow_fetch_later=False):
        """
        Fetch the sites.
        Step 1: Fetch the home pages
        Step 2: Select representative pages from the fetched sites
        Step 3: Fetch the selected representative pages
        Step 4: Merge the fetched results 

        Parameters:
        -----------
        urls: list of url. Each url represents a website
        max_pages: maximum number of pages to be selected in each website
        selection: method to select pages in a website
        online (boolean): online or offline mode. In offline mode, read data from the cache.

        Returns:
        --------
        list<website>: list of fetched websites
        """
        """
        # Hack: used pre-fetched data
        if online==False and selection=="search":
            # Read from the cache and return results
            print "hacking ..."
            print max_pages
            websites = {}
            for url in urls:
                tld = URLUtility.get_tld(url)
                websites[tld] = Website(url)
            for url in self.cache.keys():
                tld = URLUtility.get_tld(url)
                if tld in websites:
                    #if len(websites[tld].pages)<max_pages:
                    page = Page(url)
                    page.load_from_json(self.cache.get(url))
                    websites[tld].add_page(page)

            websites = [w for w in websites.values() if not w.is_empty()]
            print "Number of websites that found in cache: ", len(websites)
            for w in websites:
                print w.get_host(), len(w.pages)

            return websites
        """

        #THE BELOW CODE WORKS FOR MULTIPLE-PAGES CASE BUT NOT OPTIMIZED FOR ONE-PAGE CASE
        # Step 1: Fetch the home pages
        pages = self.fetch_pages(urls, online, allow_fetch_later) 
        max_pages -= 1 # exclude the home page
        websites = {}
        for p in pages:
            tld = p.get_tld()
            w = Website(p.get_url())  
            w.add_page(p)
            websites[tld] = w
       
        # Step 2: Select representative pages from the fetched sites
        if max_pages:
            print "    Selecting insite pages for representation. Selection method: ", selection
            insite_urls = []
            if selection==None:
                # Only fetch the homepage
                return websites.values()
            if selection=="random":
                for page in pages:
                    insite_urls.extend(self._select_random(page, max_pages))
            elif selection=="search":
                for page in pages:
                    insite_urls.extend(self._select_by_search(page, max_pages))
            else:
                print "Wrong selection method"

            print "    Selected ", len(insite_urls), " urls from ", len(pages), " sites"

            # Step 3: Fetch the selected representative pages
            pages = self.fetch_pages(insite_urls, online, allow_fetch_later)

            # Step 4: Merge the fetched results 
            for p in pages:
                tld = p.get_tld()
                if tld not in websites:
                    print "    Error: host does not exist", tld, p.get_url()
                else:
                    websites[tld].add_page(p)

        total_pages = sum([len(websites[tld].pages) for tld in websites])
        print total_pages, len(websites)
        if websites:
            print "    Average number of pages per site: ", total_pages/float(len(websites))

        return websites.values()
        """
        # Keep one url for each website
        uniq_hosts = set()
        temp_urls = []
        for url in urls:
            host = URLUtility.get_host(url)
            if host not in uniq_hosts:
                uniq_hosts.add(host)
                temp_urls.append(url)
                print "Fetching ", url
        urls = temp_urls

        # Fetch
        pages = self.fetch_pages(urls, online, allow_fetch_later) 

        # Construct website list from page list (one page per site)
        websites = []
        for p in pages:
            w = Website(p.get_url())
            w.add_page(p)
            websites.append(w)
        return websites
        """

        Fetch the sites.
        Step 1: Fetch the home pages
        Step 2: Select representative pages from the fetched sites
        Step 3: Fetch the selected representative pages
        Step 4: Merge the fetched results 

        Parameters:
        -----------
        urls: list of url. Each url represents a website
        max_pages: maximum number of pages to be selected in each website
        selection: method to select pages in a website
        online (boolean): online or offline mode. In offline mode, read data from the cache.

        Returns:
        --------
        list<website>: list of fetched websites
        
Fetch the sites.
Step 1: Fetch the home pages
Step 2: Select representative pages from the fetched sites
Step 3: Fetch the selected representative pages
Step 4: Merge the fetched results



list of url. Each url represents a website
max_pages: maximum number of pages to be selected in each website
selection: method to select pages in a website
online (boolean): online or offline mode. In offline mode, read data from the cache.



list: list of fetched websites
Fetch the sites . Step 1 : Fetch the home pages Step 2 : Select representative pages from the fetched sites Step 3 : Fetch the selected representative pages Step 4 : Merge the fetched results list of url . Each url represents a website max_pages : maximum number of pages to be selected in each website selection : method to select pages in a website online ( boolean ) : online or offline mode . In offline mode read data from the cache . list : list of fetched websites
def fetch_sites(self, urls, max_pages=1, selection="random", online=True, allow_fetch_later=False):
        pages = self.fetch_pages(urls, online, allow_fetch_later) 
        max_pages -= 1 
        websites = {}
        for p in pages:
            tld = p.get_tld()
            w = Website(p.get_url())  
            w.add_page(p)
            websites[tld] = w
        if max_pages:
            print "    Selecting insite pages for representation. Selection method: ", selection
            insite_urls = []
            if selection==None:
                return websites.values()
            if selection=="random":
                for page in pages:
                    insite_urls.extend(self._select_random(page, max_pages))
            elif selection=="search":
                for page in pages:
                    insite_urls.extend(self._select_by_search(page, max_pages))
            else:
                print "Wrong selection method"
            print "    Selected ", len(insite_urls), " urls from ", len(pages), " sites"
            pages = self.fetch_pages(insite_urls, online, allow_fetch_later)
            for p in pages:
                tld = p.get_tld()
                if tld not in websites:
                    print "    Error: host does not exist", tld, p.get_url()
                else:
                    websites[tld].add_page(p)
        total_pages = sum([len(websites[tld].pages) for tld in websites])
        print total_pages, len(websites)
        if websites:
            print "    Average number of pages per site: ", total_pages/float(len(websites))
        return websites.values()
def fetch_sites ( self , urls , max_pages = 1 , selection = "random" , online = True , allow_fetch_later = False ) : """
        # Hack: used pre-fetched data
        if online==False and selection=="search":
            # Read from the cache and return results
            print "hacking ..."
            print max_pages
            websites = {}
            for url in urls:
                tld = URLUtility.get_tld(url)
                websites[tld] = Website(url)
            for url in self.cache.keys():
                tld = URLUtility.get_tld(url)
                if tld in websites:
                    #if len(websites[tld].pages)<max_pages:
                    page = Page(url)
                    page.load_from_json(self.cache.get(url))
                    websites[tld].add_page(page)

            websites = [w for w in websites.values() if not w.is_empty()]
            print "Number of websites that found in cache: ", len(websites)
            for w in websites:
                print w.get_host(), len(w.pages)

            return websites
        """ pages = self . fetch_pages ( urls , online , allow_fetch_later ) max_pages -= 1 websites = { } for p in pages : tld = p . get_tld ( ) w = Website ( p . get_url ( ) ) w . add_page ( p ) websites [ tld ] = w if max_pages : print "    Selecting insite pages for representation. Selection method: " , selection insite_urls = [ ] if selection == None : return websites . values ( ) if selection == "random" : for page in pages : insite_urls . extend ( self . _select_random ( page , max_pages ) ) elif selection == "search" : for page in pages : insite_urls . extend ( self . _select_by_search ( page , max_pages ) ) else : print "Wrong selection method" print "    Selected " , len ( insite_urls ) , " urls from " , len ( pages ) , " sites" pages = self . fetch_pages ( insite_urls , online , allow_fetch_later ) for p in pages : tld = p . get_tld ( ) if tld not in websites : print "    Error: host does not exist" , tld , p . get_url ( ) else : websites [ tld ] . add_page ( p ) total_pages = sum ( [ len ( websites [ tld ] . pages ) for tld in websites ] ) print total_pages , len ( websites ) if websites : print "    Average number of pages per site: " , total_pages / float ( len ( websites ) ) return websites . values ( ) """
        # Keep one url for each website
        uniq_hosts = set()
        temp_urls = []
        for url in urls:
            host = URLUtility.get_host(url)
            if host not in uniq_hosts:
                uniq_hosts.add(host)
                temp_urls.append(url)
                print "Fetching ", url
        urls = temp_urls

        # Fetch
        pages = self.fetch_pages(urls, online, allow_fetch_later) 

        # Construct website list from page list (one page per site)
        websites = []
        for p in pages:
            w = Website(p.get_url())
            w.add_page(p)
            websites.append(w)
        return websites
        """
Fetch the sites.
Fetch the sites .
"""
        Fetch the sites.
        Step 1: Fetch the home pages
        Step 2: Select representative pages from the fetched sites
        Step 3: Fetch the selected representative pages
        Step 4: Merge the fetched results 

        Parameters:
        -----------
        urls: list of url. Each url represents a website
        max_pages: maximum number of pages to be selected in each website
        selection: method to select pages in a website
        online (boolean): online or offline mode. In offline mode, read data from the cache.

        Returns:
        --------
        list<website>: list of fetched websites
        """ """
        # Hack: used pre-fetched data
        if online==False and selection=="search":
            # Read from the cache and return results
            print "hacking ..."
            print max_pages
            websites = {}
            for url in urls:
                tld = URLUtility.get_tld(url)
                websites[tld] = Website(url)
            for url in self.cache.keys():
                tld = URLUtility.get_tld(url)
                if tld in websites:
                    #if len(websites[tld].pages)<max_pages:
                    page = Page(url)
                    page.load_from_json(self.cache.get(url))
                    websites[tld].add_page(page)

            websites = [w for w in websites.values() if not w.is_empty()]
            print "Number of websites that found in cache: ", len(websites)
            for w in websites:
                print w.get_host(), len(w.pages)

            return websites
        """ #THE BELOW CODE WORKS FOR MULTIPLE-PAGES CASE BUT NOT OPTIMIZED FOR ONE-PAGE CASE # Step 1: Fetch the home pages # exclude the home page # Step 2: Select representative pages from the fetched sites # Only fetch the homepage # Step 3: Fetch the selected representative pages # Step 4: Merge the fetched results  """
        # Keep one url for each website
        uniq_hosts = set()
        temp_urls = []
        for url in urls:
            host = URLUtility.get_host(url)
            if host not in uniq_hosts:
                uniq_hosts.add(host)
                temp_urls.append(url)
                print "Fetching ", url
        urls = temp_urls

        # Fetch
        pages = self.fetch_pages(urls, online, allow_fetch_later) 

        # Construct website list from page list (one page per site)
        websites = []
        for p in pages:
            w = Website(p.get_url())
            w.add_page(p)
            websites.append(w)
        return websites
        """
self
urls
max_pages
selection
online
allow_fetch_later