back-end/threats_monitoring/ipmasterlist_summer.py
def aggregate_content(html_content):
    """ This function models html content into a dictionary form
        :param
            html_content    (list)  each item of this list is a string representation of a line from c2-ipmasterlist.txt
        :returns
            dict_list       (list)  this list contains the dictionary representation of each html_content line
    """

    data_list = []

    for line in html_content:

        if line == "":
            # "" is present at the end of ipmasterlsit.txt
            continue

        items = [word for word in line.split(',')]  # split line and turn it into a list of word

        ''' IP objects '''
        # ip
        ip = items[0]

        # ip user
        ip_user = ""
        for word in items[1].split():
            if word not in ["IP", "used", "by"]:
                ip_user += word + " "

        ''' Time objects '''
        # utc datetime of event
        datetime_utc = datetime.strptime(items[2], '%Y-%m-%d %H:%M')
        # string representation
        datetime_utc_string = str(datetime_utc)
        # utc timestamp of event
        timestamp_utc = float(datetime_utc.timestamp())

        datetime_utc_cti_string = str(datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
        datetime_utc_cti = datetime.strptime(datetime_utc_cti_string, '%Y-%m-%d %H:%M:%S')
        timestamp_utc_cti = float(datetime_utc_cti.timestamp())

        ''' Description Object '''
        description = items[3]

        row_list = ["Botnets", "IP", ip, ip_user, timestamp_utc, datetime_utc, datetime_utc_string,
                    timestamp_utc_cti, datetime_utc_cti, datetime_utc_cti_string, description]
        data_list.append(row_list)

    print("Scraping Finished")

    return data_list
 This function models html content into a dictionary form
        :param
            html_content    (list)  each item of this list is a string representation of a line from c2-ipmasterlist.txt
        :returns
            dict_list       (list)  this list contains the dictionary representation of each html_content line
    
This function models html content into a dictionary form
:param
html_content    (list)  each item of this list is a string representation of a line from c2-ipmasterlist.txt
:returns
dict_list       (list)  this list contains the dictionary representation of each html_content line
This function models html content into a dictionary form : param html_content ( list ) each item of this list is a string representation of a line from c2 - ipmasterlist . txt : returns dict_list ( list ) this list contains the dictionary representation of each html_content line
def aggregate_content(html_content):
    data_list = []
    for line in html_content:
        if line == "":
            continue
        items = [word for word in line.split(',')]  
        ip = items[0]
        ip_user = ""
        for word in items[1].split():
            if word not in ["IP", "used", "by"]:
                ip_user += word + " "
        datetime_utc = datetime.strptime(items[2], '%Y-%m-%d %H:%M')
        datetime_utc_string = str(datetime_utc)
        timestamp_utc = float(datetime_utc.timestamp())
        datetime_utc_cti_string = str(datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
        datetime_utc_cti = datetime.strptime(datetime_utc_cti_string, '%Y-%m-%d %H:%M:%S')
        timestamp_utc_cti = float(datetime_utc_cti.timestamp())
        description = items[3]
        row_list = ["Botnets", "IP", ip, ip_user, timestamp_utc, datetime_utc, datetime_utc_string,
                    timestamp_utc_cti, datetime_utc_cti, datetime_utc_cti_string, description]
        data_list.append(row_list)
    print("Scraping Finished")
    return data_list
def aggregate_content ( html_content ) : data_list = [ ] for line in html_content : if line == "" : continue items = [ word for word in line . split ( ',' ) ] ''' IP objects ''' ip = items [ 0 ] ip_user = "" for word in items [ 1 ] . split ( ) : if word not in [ "IP" , "used" , "by" ] : ip_user += word + " " ''' Time objects ''' datetime_utc = datetime . strptime ( items [ 2 ] , '%Y-%m-%d %H:%M' ) datetime_utc_string = str ( datetime_utc ) timestamp_utc = float ( datetime_utc . timestamp ( ) ) datetime_utc_cti_string = str ( datetime . utcnow ( ) . strftime ( '%Y-%m-%d %H:%M:%S' ) ) datetime_utc_cti = datetime . strptime ( datetime_utc_cti_string , '%Y-%m-%d %H:%M:%S' ) timestamp_utc_cti = float ( datetime_utc_cti . timestamp ( ) ) ''' Description Object ''' description = items [ 3 ] row_list = [ "Botnets" , "IP" , ip , ip_user , timestamp_utc , datetime_utc , datetime_utc_string , timestamp_utc_cti , datetime_utc_cti , datetime_utc_cti_string , description ] data_list . append ( row_list ) print ( "Scraping Finished" ) return data_list
This function models html content into a dictionary form
:param
html_content    (list)  each item of this list is a string representation of a line from c2-ipmasterlist.txt
:returns
dict_list       (list)  this list contains the dictionary representation of each html_content line
This function models html content into a dictionary form : param html_content ( list ) each item of this list is a string representation of a line from c2 - ipmasterlist . txt : returns dict_list ( list ) this list contains the dictionary representation of each html_content line
""" This function models html content into a dictionary form
        :param
            html_content    (list)  each item of this list is a string representation of a line from c2-ipmasterlist.txt
        :returns
            dict_list       (list)  this list contains the dictionary representation of each html_content line
    """ # "" is present at the end of ipmasterlsit.txt
 # split line and turn it into a list of word
 ''' IP objects ''' # ip
 # ip user
 ''' Time objects ''' # utc datetime of event
 # string representation
 # utc timestamp of event
 ''' Description Object '''
html_content