back-end/threats_monitoring/malwaredomains.py
def crawl_malware_domains(url):
    """ This function crawls the malware domain indicator and returns all the dataset links to be downloaded and scraped
        later.
    @param
        url     (string)        url of the indicator web page
    @return
    """

    print('Crawling site: ', url)
    downloader = Downloader()
    print(url)
    html = downloader(url)

    soup = BeautifulSoup(html, 'html5lib')
    possible_links = soup.find_all('a')

    htmlLinks, htmlRemovedLinks = list([]), list([])

    for link in possible_links :
        if link.has_attr('href') and link.attrs['href'][0].isdigit():
            # construct full path using function parameter url = 'https://mirror.uce.edu.ec/malwaredomains/'
            full_link = '{}{}'.format(url, link.attrs['href'])
            htmlLinks.append(full_link)
        elif link.has_attr('href') and link.attrs['href'].startswith('removed-domains-'):
            # in this loop we gather all the removed ip lists
            full_link = '{}{}'.format(url, link.attrs['href'])
            htmlRemovedLinks.append(full_link)

    return {"blocked": htmlLinks, "removed":  htmlRemovedLinks}
 This function crawls the malware domain indicator and returns all the dataset links to be downloaded and scraped
        later.
    @param
        url     (string)        url of the indicator web page
    @return
    
This function crawls the malware domain indicator and returns all the dataset links to be downloaded and scraped
later.
@param
url     (string)        url of the indicator web page
@return
This function crawls the malware domain indicator and returns all the dataset links to be downloaded and scraped later . @param url ( string ) url of the indicator web page @return
def crawl_malware_domains(url):
    print('Crawling site: ', url)
    downloader = Downloader()
    print(url)
    html = downloader(url)
    soup = BeautifulSoup(html, 'html5lib')
    possible_links = soup.find_all('a')
    htmlLinks, htmlRemovedLinks = list([]), list([])
    for link in possible_links :
        if link.has_attr('href') and link.attrs['href'][0].isdigit():
            full_link = '{}{}'.format(url, link.attrs['href'])
            htmlLinks.append(full_link)
        elif link.has_attr('href') and link.attrs['href'].startswith('removed-domains-'):
            full_link = '{}{}'.format(url, link.attrs['href'])
            htmlRemovedLinks.append(full_link)
    return {"blocked": htmlLinks, "removed":  htmlRemovedLinks}
def crawl_malware_domains ( url ) : print ( 'Crawling site: ' , url ) downloader = Downloader ( ) print ( url ) html = downloader ( url ) soup = BeautifulSoup ( html , 'html5lib' ) possible_links = soup . find_all ( 'a' ) htmlLinks , htmlRemovedLinks = list ( [ ] ) , list ( [ ] ) for link in possible_links : if link . has_attr ( 'href' ) and link . attrs [ 'href' ] [ 0 ] . isdigit ( ) : full_link = '{}{}' . format ( url , link . attrs [ 'href' ] ) htmlLinks . append ( full_link ) elif link . has_attr ( 'href' ) and link . attrs [ 'href' ] . startswith ( 'removed-domains-' ) : full_link = '{}{}' . format ( url , link . attrs [ 'href' ] ) htmlRemovedLinks . append ( full_link ) return { "blocked" : htmlLinks , "removed" : htmlRemovedLinks }
This function crawls the malware domain indicator and returns all the dataset links to be downloaded and scraped
later.
This function crawls the malware domain indicator and returns all the dataset links to be downloaded and scraped later .
""" This function crawls the malware domain indicator and returns all the dataset links to be downloaded and scraped
        later.
    @param
        url     (string)        url of the indicator web page
    @return
    """ # construct full path using function parameter url = 'https://mirror.uce.edu.ec/malwaredomains/'
 # in this loop we gather all the removed ip lists

url