wiod_folder = '/tmp/mrios/WIOD2013'

"""
Explanation: Advanced functionality - pandas groupby with pymrio satellite accounts
This notebook examplifies how to directly apply Pandas core functions (in this case groupby and aggregation) to the pymrio system.
WIOD material extension aggregation - stressor w/o compartment info
Here we use the WIOD MRIO system (see the notebook "Automatic downloading of MRIO databases" for how to automatically retrieve this database) and will aggregate the WIOD material stressor for used and unused materials.  We assume, that the WIOD system is available at
End of explanation
"""


import pymrio

"""
Explanation: To get started we import pymrio
End of explanation
"""


wiod09 = pymrio.parse_wiod(path=wiod_folder, year=2009)

"""
Explanation: For the example here, we use the data from 2009:
End of explanation
"""


wiod09.mat.F

"""
Explanation: WIOD includes multiple material accounts, specified for the "Used" and "Unused" category, as well as information on the total. We will use the latter to confirm our calculations:
End of explanation
"""


groups = wiod09.mat.get_index(as_dict=True, grouping_pattern = {'.*_Used': 'Material Used', 
                                                                '.*_Unused': 'Material Unused'})
groups

"""
Explanation: To aggregate these with the Pandas groupby function, we need to specify the groups which should be grouped by Pandas.
Pymrio contains a helper function which builds such a matching dictionary.
The matching can also include regular expressions to simplify the build:
End of explanation
"""


wiod09.mat_agg = wiod09.mat.copy(new_name='Aggregated matrial accounts')

"""
Explanation: Note, that the grouping contains the rows which do not match any of the specified groups. 
This allows to easily aggregates only parts of a specific stressor set. To actually omit these groups
include them in the matching pattern and provide None as value.
To have the aggregated data alongside the original data, we first copy the detailed satellite account:
End of explanation
"""


for df_name, df in zip(wiod09.mat_agg.get_DataFrame(data=False, with_unit=True, with_population=False),
                       wiod09.mat_agg.get_DataFrame(data=True, with_unit=True, with_population=False)):
    if df_name == 'unit':
        wiod09.mat_agg.__dict__[df_name] = df.groupby(groups).apply(lambda x: ' & '.join(x.unit.unique()))
    else:
        wiod09.mat_agg.__dict__[df_name] = df.groupby(groups).sum()

wiod09.mat_agg.F

wiod09.mat_agg.unit

"""
Explanation: Then, we use the pymrio get_DataFrame iterator together with the pandas groupby and sum functions to aggregate the stressors.
For the dataframe containing the unit information, we pass a custom function which concatenate non-unique unit strings.
End of explanation
"""


tt = pymrio.load_test()
tt.emissions.get_index(as_dict=True)

"""
Explanation: Use with stressors including compartment information:
The same regular expression grouping can be used to aggregate stressor data which is given per compartment.
To do so, the matching dict needs to consist of tuples corresponding to a valid index value in the DataFrames. 
Each position in the tuple is interprested as a regular expression. 
Using the get_index method gives a good indication how a valid grouping dict should look like:
End of explanation
"""


agg_groups = {('emis.*', '.*'): 'all emissions'}

group_dict = tt.emissions.get_index(as_dict=True,
                                    grouping_pattern=agg_groups)
group_dict

"""
Explanation: With that information, we can now build our own grouping dict, e.g.:
End of explanation
"""


for df_name, df in zip(tt.emissions.get_DataFrame(data=False, with_unit=True, with_population=False),
                       tt.emissions.get_DataFrame(data=True, with_unit=True, with_population=False)):
    if df_name == 'unit':
        tt.emissions.__dict__[df_name] = df.groupby(group_dict).apply(lambda x: ' & '.join(x.unit.unique()))
    else:
        tt.emissions.__dict__[df_name] = df.groupby(group_dict).sum()

"""
Explanation: Which can then be used to aggregate the satellite account:
End of explanation
"""


import pandas as pd
tt.emissions.set_index(pd.Index(tt.emissions.get_index(), name='stressor'))

tt.emissions.F

"""
Explanation: In this case we loose the information on the compartment. To reset the index do:
End of explanation
"""


