back-end/threats_monitoring/downloader_phishing.py
def download(self, url, user_agent, num_tries=2, charset='utf-8'):
        """ This function downloads a website's source code.
            @parameters
                url:        (str)           website's url
                user_agent: (str)           specifies the user_agent string
                num_tries:  (int)           if a download fails due to a problem with the request (4xx) or the server
                                            (5xx) the function calls it self recursively #num_tries times
                charset:    (str)           helps specify the desired codec of the HTTP responses
            @returns
                html_code:  (str or None)   html code of web site or None if no code is returned
        """

        print("Downloading %s ... " % url)
        # construct a Request object
        request = urllib.request.Request(url)
        # set user-agent for this request
        request.add_header('User-Agent', user_agent)
        try:
            if self.proxy:
                proxy_support = urllib.request.ProxyHandler({'http': self.proxy})
                opener = urllib.request.build_opener(proxy_support)
                urllib.request.install_opener(opener)
            # make a request and get an HTTPResponse object back
            # response is a context manager (.info(), .getcode(), .geturl())
            response = urllib.request.urlopen(request)
            # reading response as string (bytes originally)
            # 'ignore' arg is crucial to avoid errors when decoding bytes with codec different than charset ('utf-8')
            html_code = response.read().decode(charset, 'ignore')
            response_code = response.getcode()
        except (URLError, HTTPError, ContentTooShortError) as e:
            print("Downloading Error:", e.reason)
            html_code = None
            if hasattr(e, 'code'):
                response_code = e.code
            else:
                response_code = None
            if num_tries > 0:
                if hasattr(e, 'code') and 500 <= e.code < 600:
                    # recursively retry 5xx HTTP errors (server errors)
                    return self.download(url, user_agent, num_tries-1, charset)

        # Our beloved html_code is UTF-8 STRING or NONE
        # TODO(4) delete statement
        # print("HTML: {0}".format(type(html_code)))

        return {'html': html_code, 'code': response_code}
 This function downloads a website's source code.
            @parameters
                url:        (str)           website's url
                user_agent: (str)           specifies the user_agent string
                num_tries:  (int)           if a download fails due to a problem with the request (4xx) or the server
                                            (5xx) the function calls it self recursively #num_tries times
                charset:    (str)           helps specify the desired codec of the HTTP responses
            @returns
                html_code:  (str or None)   html code of web site or None if no code is returned
        
This function downloads a website's source code.
This function downloads a website ' s source code .
def download(self, url, user_agent, num_tries=2, charset='utf-8'):
        print("Downloading %s ... " % url)
        request = urllib.request.Request(url)
        request.add_header('User-Agent', user_agent)
        try:
            if self.proxy:
                proxy_support = urllib.request.ProxyHandler({'http': self.proxy})
                opener = urllib.request.build_opener(proxy_support)
                urllib.request.install_opener(opener)
            response = urllib.request.urlopen(request)
            html_code = response.read().decode(charset, 'ignore')
            response_code = response.getcode()
        except (URLError, HTTPError, ContentTooShortError) as e:
            print("Downloading Error:", e.reason)
            html_code = None
            if hasattr(e, 'code'):
                response_code = e.code
            else:
                response_code = None
            if num_tries > 0:
                if hasattr(e, 'code') and 500 <= e.code < 600:
                    return self.download(url, user_agent, num_tries-1, charset)
        return {'html': html_code, 'code': response_code}
def download ( self , url , user_agent , num_tries = 2 , charset = 'utf-8' ) : print ( "Downloading %s ... " % url ) request = urllib . request . Request ( url ) request . add_header ( 'User-Agent' , user_agent ) try : if self . proxy : proxy_support = urllib . request . ProxyHandler ( { 'http' : self . proxy } ) opener = urllib . request . build_opener ( proxy_support ) urllib . request . install_opener ( opener ) response = urllib . request . urlopen ( request ) html_code = response . read ( ) . decode ( charset , 'ignore' ) response_code = response . getcode ( ) except ( URLError , HTTPError , ContentTooShortError ) as e : print ( "Downloading Error:" , e . reason ) html_code = None if hasattr ( e , 'code' ) : response_code = e . code else : response_code = None if num_tries > 0 : if hasattr ( e , 'code' ) and 500 <= e . code < 600 : return self . download ( url , user_agent , num_tries - 1 , charset ) return { 'html' : html_code , 'code' : response_code }
This function downloads a website's source code.
This function downloads a website ' s source code .
""" This function downloads a website's source code.
            @parameters
                url:        (str)           website's url
                user_agent: (str)           specifies the user_agent string
                num_tries:  (int)           if a download fails due to a problem with the request (4xx) or the server
                                            (5xx) the function calls it self recursively #num_tries times
                charset:    (str)           helps specify the desired codec of the HTTP responses
            @returns
                html_code:  (str or None)   html code of web site or None if no code is returned
        """ # construct a Request object
 # set user-agent for this request
 # make a request and get an HTTPResponse object back
 # response is a context manager (.info(), .getcode(), .geturl())
 # reading response as string (bytes originally)
 # 'ignore' arg is crucial to avoid errors when decoding bytes with codec different than charset ('utf-8')
 # recursively retry 5xx HTTP errors (server errors)
 # Our beloved html_code is UTF-8 STRING or NONE
 # TODO(4) delete statement
 # print("HTML: {0}".format(type(html_code)))

self
url
user_agent
num_tries
charset
parameters
(str)           website's url
user_agent: (str)           specifies the user_agent string
num_tries:  (int)           if a download fails due to a problem with the request (4xx) or the server
(5xx) the function calls it self recursively #num_tries times
charset:    (str)           helps specify the desired codec of the HTTP responses
( str ) website ' s url user_agent : ( str ) specifies the user_agent string num_tries : ( int ) if a download fails due to a problem with the request ( 4xx ) or the server ( 5xx ) the function calls it self recursively #num_tries times charset : ( str ) helps specify the desired codec of the HTTP responses
returns
(str or None)   html code of web site or None if no code is returned
( str or None ) html code of web site or None if no code is returned