utils/fetcher.py
def fetch(self, urls, out_file, extraction=True):
        """
        Parameters:
        -----------
        urls: list of url. Each url represents a website

        Returns:
        --------
        list<website>: list of fetched websites
        """
        if os.path.exists(out_file):
            fetched_urls = []
            with open(out_file) as lines:
                for line in lines:
                    try:
                        jsobj = json.loads(line)
                        fetched_urls.append(jsobj['url'])
                    except:
                        traceback.print_exc()
            urls = [url for url in urls if url not in fetched_urls]

        print "Number of urls to fetch: ", len(urls)
        out = open(out_file, 'a+')

        for i, url in enumerate(urls):
            if (i+1)%20==0:
                print "Fetched ", i, " urls"
            try:
                res = requests.get(url, headers=self.header, verify=False, timeout=10)
                if res.status_code == 200:
                    page = Page(url)
                    if len(res.text)<self.max_html_size: 
                        page.add_html(res.text)
                        if extraction:
                            jspage = page.get_json_obj()
                        else:
                            jspage = {'url': url, 'html':res.text}
                        out.write(json.dumps(jspage) + '\n')
                else:
                    print res.status_code, url
            except:
                print "Failed to fetch ", url
                traceback.print_exc()
            
        out.close()

        Parameters:
        -----------
        urls: list of url. Each url represents a website

        Returns:
        --------
        list<website>: list of fetched websites
        
list of url. Each url represents a website



list: list of fetched websites
list of url . Each url represents a website list : list of fetched websites
def fetch(self, urls, out_file, extraction=True):
        if os.path.exists(out_file):
            fetched_urls = []
            with open(out_file) as lines:
                for line in lines:
                    try:
                        jsobj = json.loads(line)
                        fetched_urls.append(jsobj['url'])
                    except:
                        traceback.print_exc()
            urls = [url for url in urls if url not in fetched_urls]
        print "Number of urls to fetch: ", len(urls)
        out = open(out_file, 'a+')
        for i, url in enumerate(urls):
            if (i+1)%20==0:
                print "Fetched ", i, " urls"
            try:
                res = requests.get(url, headers=self.header, verify=False, timeout=10)
                if res.status_code == 200:
                    page = Page(url)
                    if len(res.text)<self.max_html_size: 
                        page.add_html(res.text)
                        if extraction:
                            jspage = page.get_json_obj()
                        else:
                            jspage = {'url': url, 'html':res.text}
                        out.write(json.dumps(jspage) + '\n')
                else:
                    print res.status_code, url
            except:
                print "Failed to fetch ", url
                traceback.print_exc()
        out.close()
def fetch ( self , urls , out_file , extraction = True ) : if os . path . exists ( out_file ) : fetched_urls = [ ] with open ( out_file ) as lines : for line in lines : try : jsobj = json . loads ( line ) fetched_urls . append ( jsobj [ 'url' ] ) except : traceback . print_exc ( ) urls = [ url for url in urls if url not in fetched_urls ] print "Number of urls to fetch: " , len ( urls ) out = open ( out_file , 'a+' ) for i , url in enumerate ( urls ) : if ( i + 1 ) % 20 == 0 : print "Fetched " , i , " urls" try : res = requests . get ( url , headers = self . header , verify = False , timeout = 10 ) if res . status_code == 200 : page = Page ( url ) if len ( res . text ) < self . max_html_size : page . add_html ( res . text ) if extraction : jspage = page . get_json_obj ( ) else : jspage = { 'url' : url , 'html' : res . text } out . write ( json . dumps ( jspage ) + '\n' ) else : print res . status_code , url except : print "Failed to fetch " , url traceback . print_exc ( ) out . close ( )
Parameters:
urls: list of url.
Parameters : urls : list of url .
"""
        Parameters:
        -----------
        urls: list of url. Each url represents a website

        Returns:
        --------
        list<website>: list of fetched websites
        """
self
urls
out_file
extraction