src/ei_mcoe/ei_mcoe.py
def main(pudl_out, cems_df, level, separate_nems_cols,
         add_sources=False, aesthetic=False, fill_in_fuel_cost=False):
    """Create and compile tables from all three parts; final output.

    Args:
        pudl_out (pudl.output.pudltabl.PudlTabl): An object used to create
            the tables used for EIA and FERC analysis.
        cems_df (pandas.DataFrame): The output of running get_cems(). Added as
            a parameter so only have to run get_cems() once because it takes a
            while.
        level (str): A string indicating the level of desired aggregation.
            Either 'plant-fuel' or 'unit-fuel' are acceptable.
        add_sources (boolean): if True, add the sources as a MultiIndex
        aesthetic (boolean): if True, convert the columns to be more human
            readable
    Returns:
        pandas.DataFrame: A DataFrame with outputs from part 1, 2, & 3 for the
            speciied level of aggregation.
    """
    # os.chdir(os.getcwd() + '/ei-mcoe-map')
    # level = 'plant-fuel'

    # Compile raw files to pass to methods.
    raw_eia_df = prep_raw_eia(pudl_out)
    bga_df = date_to_year(pudl_out.bga_eia860())
    # Calculate first and third parts.
    p1 = clean_part1_output(build_part1_output(raw_eia_df, level))
    p3 = clean_part3_output(build_part3_output(raw_eia_df, p1, bga_df, cems_df,
                                               level))
    # Calculate second part based on specified aggregation level.
    if level == 'plant-fuel':
        p2 = clean_part2_output(build_part2_output(raw_eia_df, pudl_out),
                                separate_nems_cols)
    elif level == 'unit-fuel':
        p2 = pd.DataFrame(columns=input_dict[level + '_index_cols'])  # Empty
    else:
        print("Not a valid level. Input either 'plant-fuel' or 'unit-fuel'.")
    # Merge data from three parts together.
    p1p3 = (
        pd.merge(p1, p3, on=input_dict[level + '_index_cols'], how='outer'))
    first_cols = [
        'plant_id_pudl', 'plant_id_eia', 'plant_name_eia',
        'fuel_type_code_pudl', 'mcoe', 'fuel_cost_mwh_eia923',
        'fix_var_om_mwh', 'variable_om_mwh', 'fixed_om_mwh',
        'fix_var_om_mwh_19_nems', 'variable_om_mwh_19_nems',
        'fixed_om_mwh_19_nems', 'capacity_mw', 'net_generation_mwh',
        'premature_deaths', 'total_monetized_deaths'
    ]
    full_df = (
        # merge but drop a few columns from parts 1 and 3, because they also
        # show up in part 2
        pd.merge(p1p3.drop(
            columns=['total_fuel_cost', 'net_generation_mwh',
                     'capacity_mw', 'total_mmbtu']),
                 p2, on=input_dict[level + '_index_cols'], how='outer')
    )
    # Account for eia/pudl id descrepancies by grouping by pudl id.
    clean_df = (
        full_df.groupby(input_dict[level + '_index_cols'], as_index=False)
        .first())
    # Add plant_id_eia column to plant-level data; combine like technology col.
    if level == 'plant-fuel':
        eia_id_df = (
            pd.merge(
                full_df, raw_eia_df[
                    ['plant_id_pudl', 'plant_id_eia',
                     'sector_id_eia', 'sector_name_eia']],
                on=['plant_id_pudl'], how='outer')
            .groupby(input_dict[level + '_index_cols'], as_index=False)
            .agg({
                'plant_id_eia':
                    lambda x: '; '.join(list(map(str, list(x.unique())))),
                'sector_id_eia':
                    lambda x: '; '.join(list(map(str, list(x.unique())))),
                'sector_name_eia':
                    lambda x: '; '.join(list(map(str, list(x.unique())))),
            }
            )
        )

        clean_df = (
            pd.merge(clean_df, eia_id_df, on=input_dict[level + '_index_cols'],
                     how='outer'))
        clean_df = (
            pudl.helpers.clean_eia_counties(
                clean_df, fixes=pudl.transform.eia861.EIA_FIPS_COUNTY_FIXES)
            .pipe(pudl.helpers.add_fips_ids, state_col="state",
                  county_col="county", vintage=2018)
            .drop(columns=['nom_val', 'base_val', 'pct_chg', 'index', 'count'])
            .sort_values(['capacity_mw'], ascending=False)
            .pipe(pudl.helpers.organize_cols, first_cols)
            .astype({
                'fix_var_is_NEMS': pd.BooleanDtype(),
                'fuel_cost_from_eiaapi': pd.BooleanDtype(), })
        )
        if fill_in_fuel_cost:
            clean_df = fill_in_fuel_cost_avg(clean_df)
        clean_df = clean_df.assign(
            mcoe=lambda x: x.fuel_cost_mwh_eia923 + x.fix_var_om_mwh)
    clean_df = (  # LAST BITS
        clean_df
        .assign(
            report_date=lambda x: pd.to_datetime(x.report_year, format='%Y'))
        .pipe(fill_om_with_plant_avg, pudl_out)
        .pipe(make_confidence_tier)
        # .pipe(fill_in_state_and_county)
    )
    logger.info('Finished compiling all parts!')
    return clean_df
Create and compile tables from all three parts; final output.

    Args:
        pudl_out (pudl.output.pudltabl.PudlTabl): An object used to create
            the tables used for EIA and FERC analysis.
        cems_df (pandas.DataFrame): The output of running get_cems(). Added as
            a parameter so only have to run get_cems() once because it takes a
            while.
        level (str): A string indicating the level of desired aggregation.
            Either 'plant-fuel' or 'unit-fuel' are acceptable.
        add_sources (boolean): if True, add the sources as a MultiIndex
        aesthetic (boolean): if True, convert the columns to be more human
            readable
    Returns:
        pandas.DataFrame: A DataFrame with outputs from part 1, 2, & 3 for the
            speciied level of aggregation.
    
Create and compile tables from all three parts; final output.
Create and compile tables from all three parts ; final output .
def main(pudl_out, cems_df, level, separate_nems_cols,
         add_sources=False, aesthetic=False, fill_in_fuel_cost=False):
    raw_eia_df = prep_raw_eia(pudl_out)
    bga_df = date_to_year(pudl_out.bga_eia860())
    p1 = clean_part1_output(build_part1_output(raw_eia_df, level))
    p3 = clean_part3_output(build_part3_output(raw_eia_df, p1, bga_df, cems_df,
                                               level))
    if level == 'plant-fuel':
        p2 = clean_part2_output(build_part2_output(raw_eia_df, pudl_out),
                                separate_nems_cols)
    elif level == 'unit-fuel':
        p2 = pd.DataFrame(columns=input_dict[level + '_index_cols'])  
    else:
        print("Not a valid level. Input either 'plant-fuel' or 'unit-fuel'.")
    p1p3 = (
        pd.merge(p1, p3, on=input_dict[level + '_index_cols'], how='outer'))
    first_cols = [
        'plant_id_pudl', 'plant_id_eia', 'plant_name_eia',
        'fuel_type_code_pudl', 'mcoe', 'fuel_cost_mwh_eia923',
        'fix_var_om_mwh', 'variable_om_mwh', 'fixed_om_mwh',
        'fix_var_om_mwh_19_nems', 'variable_om_mwh_19_nems',
        'fixed_om_mwh_19_nems', 'capacity_mw', 'net_generation_mwh',
        'premature_deaths', 'total_monetized_deaths'
    ]
    full_df = (
        pd.merge(p1p3.drop(
            columns=['total_fuel_cost', 'net_generation_mwh',
                     'capacity_mw', 'total_mmbtu']),
                 p2, on=input_dict[level + '_index_cols'], how='outer')
    )
    clean_df = (
        full_df.groupby(input_dict[level + '_index_cols'], as_index=False)
        .first())
    if level == 'plant-fuel':
        eia_id_df = (
            pd.merge(
                full_df, raw_eia_df[
                    ['plant_id_pudl', 'plant_id_eia',
                     'sector_id_eia', 'sector_name_eia']],
                on=['plant_id_pudl'], how='outer')
            .groupby(input_dict[level + '_index_cols'], as_index=False)
            .agg({
                'plant_id_eia':
                    lambda x: '; '.join(list(map(str, list(x.unique())))),
                'sector_id_eia':
                    lambda x: '; '.join(list(map(str, list(x.unique())))),
                'sector_name_eia':
                    lambda x: '; '.join(list(map(str, list(x.unique())))),
            }
            )
        )
        clean_df = (
            pd.merge(clean_df, eia_id_df, on=input_dict[level + '_index_cols'],
                     how='outer'))
        clean_df = (
            pudl.helpers.clean_eia_counties(
                clean_df, fixes=pudl.transform.eia861.EIA_FIPS_COUNTY_FIXES)
            .pipe(pudl.helpers.add_fips_ids, state_col="state",
                  county_col="county", vintage=2018)
            .drop(columns=['nom_val', 'base_val', 'pct_chg', 'index', 'count'])
            .sort_values(['capacity_mw'], ascending=False)
            .pipe(pudl.helpers.organize_cols, first_cols)
            .astype({
                'fix_var_is_NEMS': pd.BooleanDtype(),
                'fuel_cost_from_eiaapi': pd.BooleanDtype(), })
        )
        if fill_in_fuel_cost:
            clean_df = fill_in_fuel_cost_avg(clean_df)
        clean_df = clean_df.assign(
            mcoe=lambda x: x.fuel_cost_mwh_eia923 + x.fix_var_om_mwh)
    clean_df = (  
        clean_df
        .assign(
            report_date=lambda x: pd.to_datetime(x.report_year, format='%Y'))
        .pipe(fill_om_with_plant_avg, pudl_out)
        .pipe(make_confidence_tier)
    )
    logger.info('Finished compiling all parts!')
    return clean_df
def main ( pudl_out , cems_df , level , separate_nems_cols , add_sources = False , aesthetic = False , fill_in_fuel_cost = False ) : raw_eia_df = prep_raw_eia ( pudl_out ) bga_df = date_to_year ( pudl_out . bga_eia860 ( ) ) p1 = clean_part1_output ( build_part1_output ( raw_eia_df , level ) ) p3 = clean_part3_output ( build_part3_output ( raw_eia_df , p1 , bga_df , cems_df , level ) ) if level == 'plant-fuel' : p2 = clean_part2_output ( build_part2_output ( raw_eia_df , pudl_out ) , separate_nems_cols ) elif level == 'unit-fuel' : p2 = pd . DataFrame ( columns = input_dict [ level + '_index_cols' ] ) else : print ( "Not a valid level. Input either 'plant-fuel' or 'unit-fuel'." ) p1p3 = ( pd . merge ( p1 , p3 , on = input_dict [ level + '_index_cols' ] , how = 'outer' ) ) first_cols = [ 'plant_id_pudl' , 'plant_id_eia' , 'plant_name_eia' , 'fuel_type_code_pudl' , 'mcoe' , 'fuel_cost_mwh_eia923' , 'fix_var_om_mwh' , 'variable_om_mwh' , 'fixed_om_mwh' , 'fix_var_om_mwh_19_nems' , 'variable_om_mwh_19_nems' , 'fixed_om_mwh_19_nems' , 'capacity_mw' , 'net_generation_mwh' , 'premature_deaths' , 'total_monetized_deaths' ] full_df = ( pd . merge ( p1p3 . drop ( columns = [ 'total_fuel_cost' , 'net_generation_mwh' , 'capacity_mw' , 'total_mmbtu' ] ) , p2 , on = input_dict [ level + '_index_cols' ] , how = 'outer' ) ) clean_df = ( full_df . groupby ( input_dict [ level + '_index_cols' ] , as_index = False ) . first ( ) ) if level == 'plant-fuel' : eia_id_df = ( pd . merge ( full_df , raw_eia_df [ [ 'plant_id_pudl' , 'plant_id_eia' , 'sector_id_eia' , 'sector_name_eia' ] ] , on = [ 'plant_id_pudl' ] , how = 'outer' ) . groupby ( input_dict [ level + '_index_cols' ] , as_index = False ) . agg ( { 'plant_id_eia' : lambda x : '; ' . join ( list ( map ( str , list ( x . unique ( ) ) ) ) ) , 'sector_id_eia' : lambda x : '; ' . join ( list ( map ( str , list ( x . unique ( ) ) ) ) ) , 'sector_name_eia' : lambda x : '; ' . join ( list ( map ( str , list ( x . unique ( ) ) ) ) ) , } ) ) clean_df = ( pd . merge ( clean_df , eia_id_df , on = input_dict [ level + '_index_cols' ] , how = 'outer' ) ) clean_df = ( pudl . helpers . clean_eia_counties ( clean_df , fixes = pudl . transform . eia861 . EIA_FIPS_COUNTY_FIXES ) . pipe ( pudl . helpers . add_fips_ids , state_col = "state" , county_col = "county" , vintage = 2018 ) . drop ( columns = [ 'nom_val' , 'base_val' , 'pct_chg' , 'index' , 'count' ] ) . sort_values ( [ 'capacity_mw' ] , ascending = False ) . pipe ( pudl . helpers . organize_cols , first_cols ) . astype ( { 'fix_var_is_NEMS' : pd . BooleanDtype ( ) , 'fuel_cost_from_eiaapi' : pd . BooleanDtype ( ) , } ) ) if fill_in_fuel_cost : clean_df = fill_in_fuel_cost_avg ( clean_df ) clean_df = clean_df . assign ( mcoe = lambda x : x . fuel_cost_mwh_eia923 + x . fix_var_om_mwh ) clean_df = ( clean_df . assign ( report_date = lambda x : pd . to_datetime ( x . report_year , format = '%Y' ) ) . pipe ( fill_om_with_plant_avg , pudl_out ) . pipe ( make_confidence_tier ) ) logger . info ( 'Finished compiling all parts!' ) return clean_df
Create and compile tables from all three parts; final output.
Create and compile tables from all three parts ; final output .
"""Create and compile tables from all three parts; final output.

    Args:
        pudl_out (pudl.output.pudltabl.PudlTabl): An object used to create
            the tables used for EIA and FERC analysis.
        cems_df (pandas.DataFrame): The output of running get_cems(). Added as
            a parameter so only have to run get_cems() once because it takes a
            while.
        level (str): A string indicating the level of desired aggregation.
            Either 'plant-fuel' or 'unit-fuel' are acceptable.
        add_sources (boolean): if True, add the sources as a MultiIndex
        aesthetic (boolean): if True, convert the columns to be more human
            readable
    Returns:
        pandas.DataFrame: A DataFrame with outputs from part 1, 2, & 3 for the
            speciied level of aggregation.
    """ # os.chdir(os.getcwd() + '/ei-mcoe-map') # level = 'plant-fuel' # Compile raw files to pass to methods. # Calculate first and third parts. # Calculate second part based on specified aggregation level. # Empty # Merge data from three parts together. # merge but drop a few columns from parts 1 and 3, because they also # show up in part 2 # Account for eia/pudl id descrepancies by grouping by pudl id. # Add plant_id_eia column to plant-level data; combine like technology col. # LAST BITS # .pipe(fill_in_state_and_county)
A DataFrame with outputs from part 1, 2, & 3 for the
speciied level of aggregation.
A DataFrame with outputs from part 1 2 & 3 for the speciied level of aggregation .
pandas.DataFrame
pudl_out
An object used to create
the tables used for EIA and FERC analysis.
An object used to create the tables used for EIA and FERC analysis .
cems_df
The output of running get_cems(). Added as
a parameter so only have to run get_cems() once because it takes a
while.
The output of running get_cems () . Added as a parameter so only have to run get_cems () once because it takes a while .
level
A string indicating the level of desired aggregation.
Either 'plant-fuel' or 'unit-fuel' are acceptable.
A string indicating the level of desired aggregation . Either ' plant - fuel ' or ' unit - fuel ' are acceptable .
separate_nems_cols
add_sources
if True, add the sources as a MultiIndex
if True add the sources as a MultiIndex
aesthetic
if True, convert the columns to be more human
readable
if True convert the columns to be more human readable
fill_in_fuel_cost