aggregation_modules/utils.py
def read_data_file(
        self, filename_or_buffer, file_format, chosen_field_names, custom_field_names
    ) -> pd.DataFrame:
        """
        reads the content from the provided filename or iobuffer
        and returns pandas dataframe
        """
        output_dataframe = None

        logger.debug("all columns in the input file... \n%s",
                     chosen_field_names)
        logger.debug("columns needed for aggregation... \n%s",
                     custom_field_names)

        if file_format == "STRUCTURED":
            output_dataframe = pd.read_csv(
                filename_or_buffer,
                index_col=False,
                header=None,
                compression="gzip",
                names=chosen_field_names,
                usecols=custom_field_names,
                delimiter=" ",
            )
        else:
            output_dataframe = pd.read_json(
                filename_or_buffer, lines="True", compression="gzip", encoding="utf-8"
            )
            json_cnames = list(output_dataframe.columns.values)
            json_new_cname = [x.lower() for x in json_cnames]
            new_name_dic = {}
            count = 0
            for cname in json_cnames:
                new_name_dic[cname] = json_new_cname[count]
                count = count + 1
                output_dataframe = output_dataframe.rename(
                    columns=new_name_dic, inplace=False
                )

        # check if read properly
        logger.debug("top 5 rows... \n%s", output_dataframe.head(5))
        logger.debug("total rows: %s", output_dataframe.size)
        logger.debug("columns... \n%s", output_dataframe.columns)

        if logging.getLogger().isEnabledFor(logging.DEBUG):
            buffer = io.StringIO()
            output_dataframe.info(verbose=True, buf=buffer)
            logger.debug("info...\n%s", buffer.getvalue())

        return output_dataframe

        reads the content from the provided filename or iobuffer
        and returns pandas dataframe
        
reads the content from the provided filename or iobuffer
and returns pandas dataframe
reads the content from the provided filename or iobuffer and returns pandas dataframe
def read_data_file(
        self, filename_or_buffer, file_format, chosen_field_names, custom_field_names
    ) -> pd.DataFrame:
        output_dataframe = None
        logger.debug("all columns in the input file... \n%s",
                     chosen_field_names)
        logger.debug("columns needed for aggregation... \n%s",
                     custom_field_names)
        if file_format == "STRUCTURED":
            output_dataframe = pd.read_csv(
                filename_or_buffer,
                index_col=False,
                header=None,
                compression="gzip",
                names=chosen_field_names,
                usecols=custom_field_names,
                delimiter=" ",
            )
        else:
            output_dataframe = pd.read_json(
                filename_or_buffer, lines="True", compression="gzip", encoding="utf-8"
            )
            json_cnames = list(output_dataframe.columns.values)
            json_new_cname = [x.lower() for x in json_cnames]
            new_name_dic = {}
            count = 0
            for cname in json_cnames:
                new_name_dic[cname] = json_new_cname[count]
                count = count + 1
                output_dataframe = output_dataframe.rename(
                    columns=new_name_dic, inplace=False
                )
        logger.debug("top 5 rows... \n%s", output_dataframe.head(5))
        logger.debug("total rows: %s", output_dataframe.size)
        logger.debug("columns... \n%s", output_dataframe.columns)
        if logging.getLogger().isEnabledFor(logging.DEBUG):
            buffer = io.StringIO()
            output_dataframe.info(verbose=True, buf=buffer)
            logger.debug("info...\n%s", buffer.getvalue())
        return output_dataframe
def read_data_file ( self , filename_or_buffer , file_format , chosen_field_names , custom_field_names ) -> pd . DataFrame : output_dataframe = None logger . debug ( "all columns in the input file... \n%s" , chosen_field_names ) logger . debug ( "columns needed for aggregation... \n%s" , custom_field_names ) if file_format == "STRUCTURED" : output_dataframe = pd . read_csv ( filename_or_buffer , index_col = False , header = None , compression = "gzip" , names = chosen_field_names , usecols = custom_field_names , delimiter = " " , ) else : output_dataframe = pd . read_json ( filename_or_buffer , lines = "True" , compression = "gzip" , encoding = "utf-8" ) json_cnames = list ( output_dataframe . columns . values ) json_new_cname = [ x . lower ( ) for x in json_cnames ] new_name_dic = { } count = 0 for cname in json_cnames : new_name_dic [ cname ] = json_new_cname [ count ] count = count + 1 output_dataframe = output_dataframe . rename ( columns = new_name_dic , inplace = False ) logger . debug ( "top 5 rows... \n%s" , output_dataframe . head ( 5 ) ) logger . debug ( "total rows: %s" , output_dataframe . size ) logger . debug ( "columns... \n%s" , output_dataframe . columns ) if logging . getLogger ( ) . isEnabledFor ( logging . DEBUG ) : buffer = io . StringIO ( ) output_dataframe . info ( verbose = True , buf = buffer ) logger . debug ( "info...\n%s" , buffer . getvalue ( ) ) return output_dataframe
reads the content from the provided filename or iobuffer
and returns pandas dataframe
reads the content from the provided filename or iobuffer and returns pandas dataframe
"""
        reads the content from the provided filename or iobuffer
        and returns pandas dataframe
        """ # check if read properly
self
filename_or_buffer
file_format
chosen_field_names
custom_field_names