back-end/utils/descriptive_analysis.py
def time_series_analysis_per_month(results_cursor, mongo_date_type='mongoDate', entity_type=None):
    """ Given a cursor that contains a query result set, this function performs time series analysis and returns
        the result data frame for analysed number of attacks per month.
        @parameters
            results_cursor      (cursor)    pymongo's result cursor. It is returned by a query
            mongo_date_type     (str)       this parameter sets the datetime object based on which will take place the
                                            time series analysis. Could be 'mongoDate' or 'mongoDate-CTI'.
            entity_type         (str)       this is a special case ONLY for RANSOMWARE attacks. It can be 'IP' or 'URL'
                                            or 'Domain' to narrow down and specify which records will be analyzed
        @returns
            s        (pandas data frame)    this data frame could be empty or contains a number of attacks for each day
    """

    print("\nBegin Descriptive Analysis Phase... ", end='')

    # process data if the collection is not empty
    if results_cursor.count() == 0:
        try:
            raise Warning("No documents retrieved")
        except Exception as e:
            print("\ndescriptive_analysis module > time_series_analysis: ", e)
        # return an empty DataFrame
        s = pd.DataFrame()
        return s

    if entity_type in ['IP', 'URL', 'Domain']:
        docs_of_interest = list([])
        for doc in results_cursor.rewind():
            if doc["Entity-Type"] == entity_type:
                docs_of_interest.append(doc)
    else:
        # in any other case entity-type is an empty string by default
        # so the whole collection is retrieved
        docs_of_interest = [doc for doc in results_cursor.rewind()]

    # aggregate dates
    dates_list = list([])
    for doc in docs_of_interest:
        dates_list.append(doc[mongo_date_type])

    ''' Time Series '''
    # a list of "1" to count the docs
    ones = [1] * len(dates_list)
    # the index of the series
    idx = pd.DatetimeIndex(dates_list)
    # the actual series (at series of 1s for the moment)
    time_series = pd.Series(ones, index=idx)
    # re-sampling / bucketing
    per_month = time_series.resample('1M').sum().fillna(0)
    # results data frame
    s = pd.DataFrame(per_month)

    print('\tCompleted')
    print('\nDescriptive Analysis Results')
    print("\nTime Series Head")
    print(time_series.head())
    print("\nPer Month Time Series")
    print(per_month.head())
    print("\nPer Month DataFrame")
    print(s.head(), "\n")

    return s

    # -----------------------------------------------------------------------------------------------------------------#

 Given a cursor that contains a query result set, this function performs time series analysis and returns
        the result data frame for analysed number of attacks per month.
        @parameters
            results_cursor      (cursor)    pymongo's result cursor. It is returned by a query
            mongo_date_type     (str)       this parameter sets the datetime object based on which will take place the
                                            time series analysis. Could be 'mongoDate' or 'mongoDate-CTI'.
            entity_type         (str)       this is a special case ONLY for RANSOMWARE attacks. It can be 'IP' or 'URL'
                                            or 'Domain' to narrow down and specify which records will be analyzed
        @returns
            s        (pandas data frame)    this data frame could be empty or contains a number of attacks for each day
    
Given a cursor that contains a query result set, this function performs time series analysis and returns
the result data frame for analysed number of attacks per month.
@parameters
results_cursor      (cursor)    pymongo's result cursor. It is returned by a query
mongo_date_type     (str)       this parameter sets the datetime object based on which will take place the
time series analysis. Could be 'mongoDate' or 'mongoDate-CTI'.
entity_type         (str)       this is a special case ONLY for RANSOMWARE attacks.
Given a cursor that contains a query result set this function performs time series analysis and returns the result data frame for analysed number of attacks per month . @parameters results_cursor ( cursor ) pymongo ' s result cursor . It is returned by a query mongo_date_type ( str ) this parameter sets the datetime object based on which will take place the time series analysis . Could be ' mongoDate ' or ' mongoDate - CTI ' . entity_type ( str ) this is a special case ONLY for RANSOMWARE attacks .
def time_series_analysis_per_month(results_cursor, mongo_date_type='mongoDate', entity_type=None):
    print("\nBegin Descriptive Analysis Phase... ", end='')
    if results_cursor.count() == 0:
        try:
            raise Warning("No documents retrieved")
        except Exception as e:
            print("\ndescriptive_analysis module > time_series_analysis: ", e)
        s = pd.DataFrame()
        return s
    if entity_type in ['IP', 'URL', 'Domain']:
        docs_of_interest = list([])
        for doc in results_cursor.rewind():
            if doc["Entity-Type"] == entity_type:
                docs_of_interest.append(doc)
    else:
        docs_of_interest = [doc for doc in results_cursor.rewind()]
    dates_list = list([])
    for doc in docs_of_interest:
        dates_list.append(doc[mongo_date_type])
    ones = [1] * len(dates_list)
    idx = pd.DatetimeIndex(dates_list)
    time_series = pd.Series(ones, index=idx)
    per_month = time_series.resample('1M').sum().fillna(0)
    s = pd.DataFrame(per_month)
    print('\tCompleted')
    print('\nDescriptive Analysis Results')
    print("\nTime Series Head")
    print(time_series.head())
    print("\nPer Month Time Series")
    print(per_month.head())
    print("\nPer Month DataFrame")
    print(s.head(), "\n")
    return s
def time_series_analysis_per_month ( results_cursor , mongo_date_type = 'mongoDate' , entity_type = None ) : print ( "\nBegin Descriptive Analysis Phase... " , end = '' ) if results_cursor . count ( ) == 0 : try : raise Warning ( "No documents retrieved" ) except Exception as e : print ( "\ndescriptive_analysis module > time_series_analysis: " , e ) s = pd . DataFrame ( ) return s if entity_type in [ 'IP' , 'URL' , 'Domain' ] : docs_of_interest = list ( [ ] ) for doc in results_cursor . rewind ( ) : if doc [ "Entity-Type" ] == entity_type : docs_of_interest . append ( doc ) else : docs_of_interest = [ doc for doc in results_cursor . rewind ( ) ] dates_list = list ( [ ] ) for doc in docs_of_interest : dates_list . append ( doc [ mongo_date_type ] ) ''' Time Series ''' ones = [ 1 ] * len ( dates_list ) idx = pd . DatetimeIndex ( dates_list ) time_series = pd . Series ( ones , index = idx ) per_month = time_series . resample ( '1M' ) . sum ( ) . fillna ( 0 ) s = pd . DataFrame ( per_month ) print ( '\tCompleted' ) print ( '\nDescriptive Analysis Results' ) print ( "\nTime Series Head" ) print ( time_series . head ( ) ) print ( "\nPer Month Time Series" ) print ( per_month . head ( ) ) print ( "\nPer Month DataFrame" ) print ( s . head ( ) , "\n" ) return s
Given a cursor that contains a query result set, this function performs time series analysis and returns
the result data frame for analysed number of attacks per month.
Given a cursor that contains a query result set this function performs time series analysis and returns the result data frame for analysed number of attacks per month .
""" Given a cursor that contains a query result set, this function performs time series analysis and returns
        the result data frame for analysed number of attacks per month.
        @parameters
            results_cursor      (cursor)    pymongo's result cursor. It is returned by a query
            mongo_date_type     (str)       this parameter sets the datetime object based on which will take place the
                                            time series analysis. Could be 'mongoDate' or 'mongoDate-CTI'.
            entity_type         (str)       this is a special case ONLY for RANSOMWARE attacks. It can be 'IP' or 'URL'
                                            or 'Domain' to narrow down and specify which records will be analyzed
        @returns
            s        (pandas data frame)    this data frame could be empty or contains a number of attacks for each day
    """ # process data if the collection is not empty
 # return an empty DataFrame
 # in any other case entity-type is an empty string by default
 # so the whole collection is retrieved
 # aggregate dates
 ''' Time Series ''' # a list of "1" to count the docs
 # the index of the series
 # the actual series (at series of 1s for the moment)
 # re-sampling / bucketing
 # results data frame
 # -----------------------------------------------------------------------------------------------------------------#

results_cursor
mongo_date_type
entity_type