preprocessing.py
def parse_logs_into_source_based_sequence_events(self, raw_log_file):
        """
        Prase AWS logs into sequence-based events such that
        each line contains unique events called by the same source (happened in a row)

        output: a list of parsed-log files per event source
        """
        input_dir = '../logs/AWS/'  # The input directory of log file

        aws_agent = AWSLogAnalyzer(raw_log_file, input_dir, read_all_file=True)
        unique_sources = aws_agent.get_feature_from_dataset("eventsource")

        for service in unique_sources:
            output_file = input_dir + service
            event_list = aws_agent.get_sequence_of_events(source_service=service)

            events_per_line = []
            for index, log in event_list.iterrows():
                event = str(events_encoding[log["eventname"]])
                #event_code = int(hashlib.sha256(event.encode('utf-8')).hexdigest(), 16) % 10 ** 8
                if event not in events_per_line:
                    events_per_line.append(event)
                else:
                    with open(output_file, 'a') as output:
                        #pickle.dump(events_per_line, output)
                        output.write(' '.join(events_per_line)+"\n")
                        output.close()

                    events_per_line.clear()
                    events_per_line.append(event)

            # in case that there is only one event from a service
            if events_per_line:
                with open(output_file, 'a') as output:
                        #pickle.dump(events_per_line, output)
                        output.write(' '.join(events_per_line)+"\n")
                        output.close()

        return unique_sources

        Prase AWS logs into sequence-based events such that
        each line contains unique events called by the same source (happened in a row)

        output: a list of parsed-log files per event source
        
Prase AWS logs into sequence-based events such that
each line contains unique events called by the same source (happened in a row)

a list of parsed-log files per event source
Prase AWS logs into sequence - based events such that each line contains unique events called by the same source ( happened in a row ) a list of parsed - log files per event source
def parse_logs_into_source_based_sequence_events(self, raw_log_file):
        input_dir = '../logs/AWS/'  
        aws_agent = AWSLogAnalyzer(raw_log_file, input_dir, read_all_file=True)
        unique_sources = aws_agent.get_feature_from_dataset("eventsource")
        for service in unique_sources:
            output_file = input_dir + service
            event_list = aws_agent.get_sequence_of_events(source_service=service)
            events_per_line = []
            for index, log in event_list.iterrows():
                event = str(events_encoding[log["eventname"]])
                if event not in events_per_line:
                    events_per_line.append(event)
                else:
                    with open(output_file, 'a') as output:
                        output.write(' '.join(events_per_line)+"\n")
                        output.close()
                    events_per_line.clear()
                    events_per_line.append(event)
            if events_per_line:
                with open(output_file, 'a') as output:
                        output.write(' '.join(events_per_line)+"\n")
                        output.close()
        return unique_sources
def parse_logs_into_source_based_sequence_events ( self , raw_log_file ) : input_dir = '../logs/AWS/' aws_agent = AWSLogAnalyzer ( raw_log_file , input_dir , read_all_file = True ) unique_sources = aws_agent . get_feature_from_dataset ( "eventsource" ) for service in unique_sources : output_file = input_dir + service event_list = aws_agent . get_sequence_of_events ( source_service = service ) events_per_line = [ ] for index , log in event_list . iterrows ( ) : event = str ( events_encoding [ log [ "eventname" ] ] ) if event not in events_per_line : events_per_line . append ( event ) else : with open ( output_file , 'a' ) as output : output . write ( ' ' . join ( events_per_line ) + "\n" ) output . close ( ) events_per_line . clear ( ) events_per_line . append ( event ) if events_per_line : with open ( output_file , 'a' ) as output : output . write ( ' ' . join ( events_per_line ) + "\n" ) output . close ( ) return unique_sources
Prase AWS logs into sequence-based events such that
each line contains unique events called by the same source (happened in a row)
Prase AWS logs into sequence - based events such that each line contains unique events called by the same source ( happened in a row )
"""
        Prase AWS logs into sequence-based events such that
        each line contains unique events called by the same source (happened in a row)

        output: a list of parsed-log files per event source
        """ # The input directory of log file #event_code = int(hashlib.sha256(event.encode('utf-8')).hexdigest(), 16) % 10 ** 8 #pickle.dump(events_per_line, output) # in case that there is only one event from a service #pickle.dump(events_per_line, output)
self
raw_log_file