src/ei_mcoe/ei_mcoe.py
def compare_heatrate(raw_eia_df, merge_df, sd_mean_cols=False):
    """Compare heatrates within plants to find outliers.

    Outputs a pandas DataFrame containing information about whether unit level
    heat rates differ significantly from plant level heatrates. To determine
    significant deviation, we calculate the mean and standard deviation of
    heatrate values for plants of a certain fuel type during a certain year.
    Units that are more than one standard deviation from the mean value are
    considered significantly different and are marked as True.

    Args:
        raw_eia_df (pandas.DataFrame): A DataFrame containing EIA data from the
                pudl_out.mcoe() function that has been run through the
                prep_raw_eia() function.
        merge_df (pandas.DataFrame): The DataFrame you'd like to merge the
            significant heat rate column with.

    Returns:
        pandas.DataFrame: A Dataframe with a boolean column to show whether
            the heat rate of a given unit is significantly different from
            the others within its aggregation group.
    """
    logger.info(' - Comparing heat rates internally')
    # Get plant and unit level wahr then merge for comparison.
    unit_level_df = build_part1_output(raw_eia_df, 'unit-fuel')
    # Find and report difference between min and max heat rate values
    unit_groups = (
        unit_level_df.groupby(input_dict['plant-fuel_index_cols']))
    max_min_df = (
        unit_groups['heat_rate_mmbtu_mwh']
        .agg(max_heat_rate=('max'),
             min_heat_rate=('min'))
        .assign(
            max_min_hr_diff=lambda x: x.max_heat_rate - x.min_heat_rate)
        # .round(2)
        .reset_index())
    max_min_df = (max_min_df[input_dict['plant-fuel_index_cols']
                             + ['max_min_hr_diff']])
    # Remove crazy outliers for mean / std calculation
    unit_level_df_low = (
        unit_level_df.loc[
            unit_level_df['heat_rate_mmbtu_mwh'] < 20])
    # Find average heat rate for fuel type and year and standard dev.
    mean_yr_hr_df = (
        unit_level_df_low
        .groupby(['report_year', 'fuel_type_code_pudl'])
        .agg(
            mean_hr_mwh=('heat_rate_mmbtu_mwh', 'mean'),
            std_hr=('heat_rate_mmbtu_mwh', 'std'))
        .reset_index())
    # Merge year and fuel type hr average & std with unit_level df
    unit_level_df_mean = (
        pd.merge(unit_level_df, mean_yr_hr_df,
                 on=['report_year', 'fuel_type_code_pudl'], how='left')
        .assign(
            hr_dist_from_mean=lambda x: (
                abs(x.heat_rate_mmbtu_mwh - x.mean_hr_mwh)),
            hr_z_score=lambda x: (x.hr_dist_from_mean / x.std_hr),
            sig_hr=lambda x: x.hr_z_score > 1))
    # Account for mean and standard deviation of heat rates for each fuel type.
    # mean_dict = {}
    # std_dict = {}
    # for fuel in fuel_types:
    #     mean = (
    #         unit_level_df_low.query(f"fuel_type_code_pudl=='{fuel}'")
    #         ['heat_rate_mmbtu_mwh'].mean()).round(2)
    #     std = (
    #         unit_level_df_low.query(f"fuel_type_code_pudl=='{fuel}'")
    #         ['heat_rate_mmbtu_mwh'].std()).round(2)
    #     mean_dict[fuel] = mean
    #     std_dict[fuel] = std
    # print('excluding heat rates over 34:')
    # print(f'average heat rates for fuel types: {mean_dict}')
    # print(f'standard deviation for fuel types: {std_dict}')
    # # Create columns for distance from mean and bool for whether over one std
    # unit_level_df['hr_dist_from_mean'] = (
    #     unit_level_df.apply(
    #         lambda x: abs(x.heat_rate_mmbtu_mwh -
    #                       mean_dict[x.fuel_type_code_pudl]), axis=1))
    # unit_level_df['sig_hr'] = (
    #     unit_level_df.apply(
    #         lambda x: (x.hr_dist_from_mean > std_dict[x.fuel_type_code_pudl])
    #         axis=1))
    # Return columns with mean and SD calculations OR merge to regular output
    if sd_mean_cols is True:
        final_hr_df = unit_level_df_mean
    else:
        # Aggregate to plant level
        plant_level_df = (
            unit_level_df_mean
            .groupby(input_dict['plant-fuel_index_cols'])['sig_hr']
            .any()
            .reset_index())
        # Merge with max min difference calculations
        with_min_max_df = (
            pd.merge(plant_level_df, max_min_df,
                     on=input_dict['plant-fuel_index_cols'], how='left'))
        # Merge with input df (merge_df)
        logger.info('preparing merge: checking df length compatability')
        if len(merge_df) != len(with_min_max_df):
            print(
                'df passed not the same length as plant-level aggreation df; '
                f'{len(merge_df)} v {len(with_min_max_df)} check for dupes'
            )
        final_hr_df = (
            pd.merge(merge_df, with_min_max_df,
                     on=input_dict['plant-fuel_index_cols'], how='left'))
        # If only one eia unit, change heat rate diff to say "only one unit"
        final_hr_df.loc[
            (final_hr_df['eia_unit_count'] == 1)
            & (final_hr_df['max_min_hr_diff'] == 0),
            'max_min_hr_diff'] = 'only one unit'
    return final_hr_df
Compare heatrates within plants to find outliers.

    Outputs a pandas DataFrame containing information about whether unit level
    heat rates differ significantly from plant level heatrates. To determine
    significant deviation, we calculate the mean and standard deviation of
    heatrate values for plants of a certain fuel type during a certain year.
    Units that are more than one standard deviation from the mean value are
    considered significantly different and are marked as True.

    Args:
        raw_eia_df (pandas.DataFrame): A DataFrame containing EIA data from the
                pudl_out.mcoe() function that has been run through the
                prep_raw_eia() function.
        merge_df (pandas.DataFrame): The DataFrame you'd like to merge the
            significant heat rate column with.

    Returns:
        pandas.DataFrame: A Dataframe with a boolean column to show whether
            the heat rate of a given unit is significantly different from
            the others within its aggregation group.
    
Compare heatrates within plants to find outliers.
Outputs a pandas DataFrame containing information about whether unit level
heat rates differ significantly from plant level heatrates. To determine
significant deviation, we calculate the mean and standard deviation of
heatrate values for plants of a certain fuel type during a certain year.
Units that are more than one standard deviation from the mean value are
considered significantly different and are marked as True.
Compare heatrates within plants to find outliers . Outputs a pandas DataFrame containing information about whether unit level heat rates differ significantly from plant level heatrates . To determine significant deviation we calculate the mean and standard deviation of heatrate values for plants of a certain fuel type during a certain year . Units that are more than one standard deviation from the mean value are considered significantly different and are marked as True .
def compare_heatrate(raw_eia_df, merge_df, sd_mean_cols=False):
    logger.info(' - Comparing heat rates internally')
    unit_level_df = build_part1_output(raw_eia_df, 'unit-fuel')
    unit_groups = (
        unit_level_df.groupby(input_dict['plant-fuel_index_cols']))
    max_min_df = (
        unit_groups['heat_rate_mmbtu_mwh']
        .agg(max_heat_rate=('max'),
             min_heat_rate=('min'))
        .assign(
            max_min_hr_diff=lambda x: x.max_heat_rate - x.min_heat_rate)
        .reset_index())
    max_min_df = (max_min_df[input_dict['plant-fuel_index_cols']
                             + ['max_min_hr_diff']])
    unit_level_df_low = (
        unit_level_df.loc[
            unit_level_df['heat_rate_mmbtu_mwh'] < 20])
    mean_yr_hr_df = (
        unit_level_df_low
        .groupby(['report_year', 'fuel_type_code_pudl'])
        .agg(
            mean_hr_mwh=('heat_rate_mmbtu_mwh', 'mean'),
            std_hr=('heat_rate_mmbtu_mwh', 'std'))
        .reset_index())
    unit_level_df_mean = (
        pd.merge(unit_level_df, mean_yr_hr_df,
                 on=['report_year', 'fuel_type_code_pudl'], how='left')
        .assign(
            hr_dist_from_mean=lambda x: (
                abs(x.heat_rate_mmbtu_mwh - x.mean_hr_mwh)),
            hr_z_score=lambda x: (x.hr_dist_from_mean / x.std_hr),
            sig_hr=lambda x: x.hr_z_score > 1))
    if sd_mean_cols is True:
        final_hr_df = unit_level_df_mean
    else:
        plant_level_df = (
            unit_level_df_mean
            .groupby(input_dict['plant-fuel_index_cols'])['sig_hr']
            .any()
            .reset_index())
        with_min_max_df = (
            pd.merge(plant_level_df, max_min_df,
                     on=input_dict['plant-fuel_index_cols'], how='left'))
        logger.info('preparing merge: checking df length compatability')
        if len(merge_df) != len(with_min_max_df):
            print(
                'df passed not the same length as plant-level aggreation df; '
                f'{len(merge_df)} v {len(with_min_max_df)} check for dupes'
            )
        final_hr_df = (
            pd.merge(merge_df, with_min_max_df,
                     on=input_dict['plant-fuel_index_cols'], how='left'))
        final_hr_df.loc[
            (final_hr_df['eia_unit_count'] == 1)
            & (final_hr_df['max_min_hr_diff'] == 0),
            'max_min_hr_diff'] = 'only one unit'
    return final_hr_df
def compare_heatrate ( raw_eia_df , merge_df , sd_mean_cols = False ) : logger . info ( ' - Comparing heat rates internally' ) unit_level_df = build_part1_output ( raw_eia_df , 'unit-fuel' ) unit_groups = ( unit_level_df . groupby ( input_dict [ 'plant-fuel_index_cols' ] ) ) max_min_df = ( unit_groups [ 'heat_rate_mmbtu_mwh' ] . agg ( max_heat_rate = ( 'max' ) , min_heat_rate = ( 'min' ) ) . assign ( max_min_hr_diff = lambda x : x . max_heat_rate - x . min_heat_rate ) . reset_index ( ) ) max_min_df = ( max_min_df [ input_dict [ 'plant-fuel_index_cols' ] + [ 'max_min_hr_diff' ] ] ) unit_level_df_low = ( unit_level_df . loc [ unit_level_df [ 'heat_rate_mmbtu_mwh' ] < 20 ] ) mean_yr_hr_df = ( unit_level_df_low . groupby ( [ 'report_year' , 'fuel_type_code_pudl' ] ) . agg ( mean_hr_mwh = ( 'heat_rate_mmbtu_mwh' , 'mean' ) , std_hr = ( 'heat_rate_mmbtu_mwh' , 'std' ) ) . reset_index ( ) ) unit_level_df_mean = ( pd . merge ( unit_level_df , mean_yr_hr_df , on = [ 'report_year' , 'fuel_type_code_pudl' ] , how = 'left' ) . assign ( hr_dist_from_mean = lambda x : ( abs ( x . heat_rate_mmbtu_mwh - x . mean_hr_mwh ) ) , hr_z_score = lambda x : ( x . hr_dist_from_mean / x . std_hr ) , sig_hr = lambda x : x . hr_z_score > 1 ) ) if sd_mean_cols is True : final_hr_df = unit_level_df_mean else : plant_level_df = ( unit_level_df_mean . groupby ( input_dict [ 'plant-fuel_index_cols' ] ) [ 'sig_hr' ] . any ( ) . reset_index ( ) ) with_min_max_df = ( pd . merge ( plant_level_df , max_min_df , on = input_dict [ 'plant-fuel_index_cols' ] , how = 'left' ) ) logger . info ( 'preparing merge: checking df length compatability' ) if len ( merge_df ) != len ( with_min_max_df ) : print ( 'df passed not the same length as plant-level aggreation df; ' f'{len(merge_df)} v {len(with_min_max_df)} check for dupes' ) final_hr_df = ( pd . merge ( merge_df , with_min_max_df , on = input_dict [ 'plant-fuel_index_cols' ] , how = 'left' ) ) final_hr_df . loc [ ( final_hr_df [ 'eia_unit_count' ] == 1 ) & ( final_hr_df [ 'max_min_hr_diff' ] == 0 ) , 'max_min_hr_diff' ] = 'only one unit' return final_hr_df
Compare heatrates within plants to find outliers.
Compare heatrates within plants to find outliers .
"""Compare heatrates within plants to find outliers.

    Outputs a pandas DataFrame containing information about whether unit level
    heat rates differ significantly from plant level heatrates. To determine
    significant deviation, we calculate the mean and standard deviation of
    heatrate values for plants of a certain fuel type during a certain year.
    Units that are more than one standard deviation from the mean value are
    considered significantly different and are marked as True.

    Args:
        raw_eia_df (pandas.DataFrame): A DataFrame containing EIA data from the
                pudl_out.mcoe() function that has been run through the
                prep_raw_eia() function.
        merge_df (pandas.DataFrame): The DataFrame you'd like to merge the
            significant heat rate column with.

    Returns:
        pandas.DataFrame: A Dataframe with a boolean column to show whether
            the heat rate of a given unit is significantly different from
            the others within its aggregation group.
    """ # Get plant and unit level wahr then merge for comparison. # Find and report difference between min and max heat rate values # .round(2) # Remove crazy outliers for mean / std calculation # Find average heat rate for fuel type and year and standard dev. # Merge year and fuel type hr average & std with unit_level df # Account for mean and standard deviation of heat rates for each fuel type. # mean_dict = {} # std_dict = {} # for fuel in fuel_types: #     mean = ( #         unit_level_df_low.query(f"fuel_type_code_pudl=='{fuel}'") #         ['heat_rate_mmbtu_mwh'].mean()).round(2) #     std = ( #         unit_level_df_low.query(f"fuel_type_code_pudl=='{fuel}'") #         ['heat_rate_mmbtu_mwh'].std()).round(2) #     mean_dict[fuel] = mean #     std_dict[fuel] = std # print('excluding heat rates over 34:') # print(f'average heat rates for fuel types: {mean_dict}') # print(f'standard deviation for fuel types: {std_dict}') # # Create columns for distance from mean and bool for whether over one std # unit_level_df['hr_dist_from_mean'] = ( #     unit_level_df.apply( #         lambda x: abs(x.heat_rate_mmbtu_mwh - #                       mean_dict[x.fuel_type_code_pudl]), axis=1)) # unit_level_df['sig_hr'] = ( #     unit_level_df.apply( #         lambda x: (x.hr_dist_from_mean > std_dict[x.fuel_type_code_pudl]) #         axis=1)) # Return columns with mean and SD calculations OR merge to regular output # Aggregate to plant level # Merge with max min difference calculations # Merge with input df (merge_df) # If only one eia unit, change heat rate diff to say "only one unit"
A Dataframe with a boolean column to show whether
the heat rate of a given unit is significantly different from
the others within its aggregation group.
A Dataframe with a boolean column to show whether the heat rate of a given unit is significantly different from the others within its aggregation group .
pandas.DataFrame
raw_eia_df
A DataFrame containing EIA data from the
pudl_out.mcoe() function that has been run through the
prep_raw_eia() function.
A DataFrame containing EIA data from the pudl_out . mcoe () function that has been run through the prep_raw_eia () function .
merge_df
The DataFrame you'd like to merge the
significant heat rate column with.
The DataFrame you ' d like to merge the significant heat rate column with .
sd_mean_cols