data/process_data.py
def clean_data(df):
    ''' This function will clean and prepare the merged data to make it more efficient to work with. 
    
    The steps this function will take to clean and prepare the data are:
        - Split the categories into separate category columns
        - Rename every column to its corresponding category
        - Convert category values to a boolean format (0 and 1)
        - Replace the original categories column in the merged dataframe with the new category columns
        - Drop any dulplicates in the newly merged dataset

    If the input is invalid or the data does not exist, this function will raise an error. 
    
    INPUT:
    df --> a Pandas DataFrame with the merged data
    
    OUTPUT:
    df --> a new Pandas Dataframe with each category as a column and its entries as 0/1 indicators.
    This is to flag if a message is classified under each category column.
    '''
    #Split the categories into 36 individual category columns and create a dataframe
    cat_cols = df["categories"].str.split(";", expand=True)
        
    #Rename every column to its corresponding category
    
    ##First, calling the first row of cat_cols to extract a new list of new column names
    ##Using a lambda function that takes everything 
    ##up to the second to last character of each string with slicing
    row = cat_cols.iloc[0]
    string_slicer = lambda x: x[:-2]
    cat_colnames = [string_slicer(i) for i in list(row)]
    cat_cols.columns = cat_colnames
    
    #Convert category values to a boolean format (0 and 1)
    
    #Iterating through the category columns in df to keep only the last character of each string (the 1 or 0) 
    ##Then convert the string into a numeric value 
    ##Using the slicing method once again
    int_slicer = lambda x: int(x[-1])
    for column in cat_cols:
        cat_cols[column] = [int_slicer(i) for i in list(cat_cols[column])]
        
    #Replace the original categories column in the merged dataframe with the new category columns
    df = df.drop(['categories'], axis=1)   
    df = pd.merge(df, cat_cols, left_index=True, right_index=True)
    df['related'] = df['related'].astype('str').str.replace('2', '1')
    df['related'] = df['related'].astype('int')
    
    #Drop any dulplicates in the newly merged dataset
    df = df.drop_duplicates()
    
    return df
 This function will clean and prepare the merged data to make it more efficient to work with. 
    
    The steps this function will take to clean and prepare the data are:
        - Split the categories into separate category columns
        - Rename every column to its corresponding category
        - Convert category values to a boolean format (0 and 1)
        - Replace the original categories column in the merged dataframe with the new category columns
        - Drop any dulplicates in the newly merged dataset

    If the input is invalid or the data does not exist, this function will raise an error. 
    
    INPUT:
    df --> a Pandas DataFrame with the merged data
    
    OUTPUT:
    df --> a new Pandas Dataframe with each category as a column and its entries as 0/1 indicators.
    This is to flag if a message is classified under each category column.
    
This function will clean and prepare the merged data to make it more efficient to work with.
The steps this function will take to clean and prepare the data are:
Split the categories into separate category columns
Rename every column to its corresponding category
Convert category values to a boolean format (0 and 1)
Replace the original categories column in the merged dataframe with the new category columns
Drop any dulplicates in the newly merged dataset

If the input is invalid or the data does not exist, this function will raise an error.

-> a Pandas DataFrame with the merged data

-> a new Pandas Dataframe with each category as a column and its entries as 0/1 indicators.
This is to flag if a message is classified under each category column.
This function will clean and prepare the merged data to make it more efficient to work with . The steps this function will take to clean and prepare the data are : Split the categories into separate category columns Rename every column to its corresponding category Convert category values to a boolean format ( 0 and 1 ) Replace the original categories column in the merged dataframe with the new category columns Drop any dulplicates in the newly merged dataset If the input is invalid or the data does not exist this function will raise an error . - > a Pandas DataFrame with the merged data - > a new Pandas Dataframe with each category as a column and its entries as 0 / 1 indicators . This is to flag if a message is classified under each category column .
def clean_data(df):
    cat_cols = df["categories"].str.split(";", expand=True)
    row = cat_cols.iloc[0]
    string_slicer = lambda x: x[:-2]
    cat_colnames = [string_slicer(i) for i in list(row)]
    cat_cols.columns = cat_colnames
    int_slicer = lambda x: int(x[-1])
    for column in cat_cols:
        cat_cols[column] = [int_slicer(i) for i in list(cat_cols[column])]
    df = df.drop(['categories'], axis=1)   
    df = pd.merge(df, cat_cols, left_index=True, right_index=True)
    df['related'] = df['related'].astype('str').str.replace('2', '1')
    df['related'] = df['related'].astype('int')
    df = df.drop_duplicates()
    return df
def clean_data ( df ) : cat_cols = df [ "categories" ] . str . split ( ";" , expand = True ) row = cat_cols . iloc [ 0 ] string_slicer = lambda x : x [ : - 2 ] cat_colnames = [ string_slicer ( i ) for i in list ( row ) ] cat_cols . columns = cat_colnames int_slicer = lambda x : int ( x [ - 1 ] ) for column in cat_cols : cat_cols [ column ] = [ int_slicer ( i ) for i in list ( cat_cols [ column ] ) ] df = df . drop ( [ 'categories' ] , axis = 1 ) df = pd . merge ( df , cat_cols , left_index = True , right_index = True ) df [ 'related' ] = df [ 'related' ] . astype ( 'str' ) . str . replace ( '2' , '1' ) df [ 'related' ] = df [ 'related' ] . astype ( 'int' ) df = df . drop_duplicates ( ) return df
This function will clean and prepare the merged data to make it more efficient to work with.
This function will clean and prepare the merged data to make it more efficient to work with .
''' This function will clean and prepare the merged data to make it more efficient to work with. 
    
    The steps this function will take to clean and prepare the data are:
        - Split the categories into separate category columns
        - Rename every column to its corresponding category
        - Convert category values to a boolean format (0 and 1)
        - Replace the original categories column in the merged dataframe with the new category columns
        - Drop any dulplicates in the newly merged dataset

    If the input is invalid or the data does not exist, this function will raise an error. 
    
    INPUT:
    df --> a Pandas DataFrame with the merged data
    
    OUTPUT:
    df --> a new Pandas Dataframe with each category as a column and its entries as 0/1 indicators.
    This is to flag if a message is classified under each category column.
    ''' #Split the categories into 36 individual category columns and create a dataframe #Rename every column to its corresponding category ##First, calling the first row of cat_cols to extract a new list of new column names ##Using a lambda function that takes everything  ##up to the second to last character of each string with slicing #Convert category values to a boolean format (0 and 1) #Iterating through the category columns in df to keep only the last character of each string (the 1 or 0)  ##Then convert the string into a numeric value  ##Using the slicing method once again #Replace the original categories column in the merged dataframe with the new category columns #Drop any dulplicates in the newly merged dataset
df