utils/multiproc_fetcher.py
def fetch_pages(self, urls, online=True, allow_fetch_later=False):
        """
        Fetch the urls that are not in cache

        Parameters:
        -----------
        urls: list of url. Each url represents a website

        Returns:
        --------
        list<website>: list of fetched websites
        """

        # Remove urls that were crawled previously
        print "Number of urls considering to fetch: ", len(urls)
        if allow_fetch_later==False:
            temp_urls = []
            for url in urls:
                if url not in self.uniq_urls:
                    self.uniq_urls.add(url)
                    temp_urls.append(url)
            urls = temp_urls

        print "Number of urls will be fetched: ", len(urls)
        
        pages, urls = self._load_from_cache(urls)
        print "    ", len(pages), " urls loaded from the cache, ", len(urls), " urls left"
        if online:
            jobs = []
            results = Queue()
            for i in range(self.proc_numb):
                p = Process(target = self.fetch_pages_helper, args = (urls, i, self.proc_numb, self.caches[i], results))
                jobs.append(p)

            for p in jobs:
                p.start()

            fetched_pages = [p for _ in jobs for p in results.get()] # Result must be collected before join()

            for p in jobs:
                p.join()

            if urls:
                print "    Fetched ", len(fetched_pages), " urls"
            pages.extend(fetched_pages)

        return pages

        Fetch the urls that are not in cache

        Parameters:
        -----------
        urls: list of url. Each url represents a website

        Returns:
        --------
        list<website>: list of fetched websites
        
Fetch the urls that are not in cache
Parameters.

list of url. Each url represents a website



list: list of fetched websites
Fetch the urls that are not in cache Parameters . list of url . Each url represents a website list : list of fetched websites
def fetch_pages(self, urls, online=True, allow_fetch_later=False):
        print "Number of urls considering to fetch: ", len(urls)
        if allow_fetch_later==False:
            temp_urls = []
            for url in urls:
                if url not in self.uniq_urls:
                    self.uniq_urls.add(url)
                    temp_urls.append(url)
            urls = temp_urls
        print "Number of urls will be fetched: ", len(urls)
        pages, urls = self._load_from_cache(urls)
        print "    ", len(pages), " urls loaded from the cache, ", len(urls), " urls left"
        if online:
            jobs = []
            results = Queue()
            for i in range(self.proc_numb):
                p = Process(target = self.fetch_pages_helper, args = (urls, i, self.proc_numb, self.caches[i], results))
                jobs.append(p)
            for p in jobs:
                p.start()
            fetched_pages = [p for _ in jobs for p in results.get()] 
            for p in jobs:
                p.join()
            if urls:
                print "    Fetched ", len(fetched_pages), " urls"
            pages.extend(fetched_pages)
        return pages
def fetch_pages ( self , urls , online = True , allow_fetch_later = False ) : print "Number of urls considering to fetch: " , len ( urls ) if allow_fetch_later == False : temp_urls = [ ] for url in urls : if url not in self . uniq_urls : self . uniq_urls . add ( url ) temp_urls . append ( url ) urls = temp_urls print "Number of urls will be fetched: " , len ( urls ) pages , urls = self . _load_from_cache ( urls ) print "    " , len ( pages ) , " urls loaded from the cache, " , len ( urls ) , " urls left" if online : jobs = [ ] results = Queue ( ) for i in range ( self . proc_numb ) : p = Process ( target = self . fetch_pages_helper , args = ( urls , i , self . proc_numb , self . caches [ i ] , results ) ) jobs . append ( p ) for p in jobs : p . start ( ) fetched_pages = [ p for _ in jobs for p in results . get ( ) ] for p in jobs : p . join ( ) if urls : print "    Fetched " , len ( fetched_pages ) , " urls" pages . extend ( fetched_pages ) return pages
Fetch the urls that are not in cache
Parameters:
Fetch the urls that are not in cache Parameters :
"""
        Fetch the urls that are not in cache

        Parameters:
        -----------
        urls: list of url. Each url represents a website

        Returns:
        --------
        list<website>: list of fetched websites
        """ # Remove urls that were crawled previously # Result must be collected before join()
self
urls
online
allow_fetch_later