# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [Root]
#     language: python
#     name: Python [Root]
# ---

import pandas as pd
from pprint import pprint
import json

# +
# ICD_list table must be re-built from, presumably, ICD_for_Enc due to some entries being
# pre-18th birthday.  ICD_list entries are not timestamped!
table_names = ['all_encounter_data', 'demographics', 'encounters', 'family_hist_for_Enc',
               'family_hist_list', 'ICD_for_Enc', 'ICD_list', 'macula_findings_for_Enc',
               'SL_Lens_for_Enc', 'SNOMED_problem_list', 'systemic_disease_for_Enc', 'systemic_disease_list']

person_data = ['demographics','family_hist_list', 'systemic_disease_list', 'SNOMED_problem_list']

encounter_data = ['all_encounter_data', 'encounters', 'family_hist_for_Enc', 'ICD_for_Enc', 'macula_findings_for_Enc',
                   'SL_Lens_for_Enc', 'systemic_disease_for_Enc']

# -

path = 'C:\\Users\\asimon6\\Downloads\\ICO_data\\'

# read tables into dataframes
dfs = [pd.read_pickle(path + name + '.pickle') if name != 'ICD_list' else None
        for name in table_names ]

# rename columns in all dataframes to avoid unicode decode error
for df in dfs:
    if df is not None:
        df.columns = [col.decode("utf-8-sig") for col in df.columns]

for df in dfs:
    if df is not None:
        print df.columns.values

# +
# aggregate all person data by Person_ID
persons = {} 
id_key ='Person_Nbr'
for person_df_name in person_data:
#     print person_df_name,
    df_index = table_names.index(person_df_name)
    person_data_df = dfs[df_index]

    if person_data_df is not None:
#         print len(person_data_df)
        for i, dfrow in person_data_df.iterrows():
            rowdict = dict(dfrow)
            id_value = rowdict[id_key]
            
            for k, v in rowdict.iteritems():
                if isinstance(v, pd.tslib.Timestamp):
                    rowdict[k] = v.toordinal()
            
            if id_value not in persons:
                persons[id_value] = {}
            
            persons[id_value].setdefault(person_df_name, []).append(rowdict)


with open('allPersons.json', 'w') as fh:
    json.dump(persons, fh)            
            

# -

for person_id in persons:
    pprint(persons[person_id])
    break

# +
# find the range of count of rows nested under person id for all keys(table names)
all_count_by_table_name_person = {}
for person_id in persons:
    person = persons[person_id]
    for table_name in person:
        all_count_by_table_name_person.setdefault(table_name, set()).add(len(person[table_name]))

min_max_count_by_table_name_person = {}
for table_name in all_count_by_table_name_person:
    min_max_count_by_table_name_person[table_name] = {}
    min_max_count_by_table_name_person[table_name]['minimum'] = min(all_count_by_table_name_person[table_name])
    min_max_count_by_table_name_person[table_name]['maximum'] = max(all_count_by_table_name_person[table_name])

pprint(min_max_count_by_table_name_person)

# +
# aggregate all encounter data by Enc_ID
encounters = {} 
id_key = 'Enc_Nbr'
for encounter_df_name in encounter_data:
#     print encounter_df_name,
    df_index = table_names.index(encounter_df_name)
    encounter_df = dfs[df_index]

    if encounter_df is not None:
#         print len(encounter_df)
#         print encounter_df.columns.values
        for i, dfrow in encounter_df.iterrows():
            rowdict = dict(dfrow)
            id_value = rowdict[id_key]
            
            for k, v in rowdict.iteritems():
                if isinstance(v, pd.tslib.Timestamp):
                    rowdict[k] = v.toordinal()
            
            if id_value not in encounters:
                encounters[id_value] = {}
            
            encounters[id_value].setdefault(encounter_df_name, []).append(rowdict)

with open('allEncounters.json', 'w') as fh:
    json.dump(encounters, fh)            
# -

for encounter_id in encounters:
    if 'ICD_for_Enc' in encounters[encounter_id]:
        if len(encounters[encounter_id]['ICD_for_Enc']) >1:
            pprint(encounters[encounter_id])
            break

# +
# find the range of count of rows nested under encounter id for all keys(table names)
all_count_by_table_name_encounter = {}
for encounter_id in encounters:
    encounter = encounters[encounter_id]
    for table_name in encounter:
        all_count_by_table_name_encounter.setdefault(table_name, set()).add(len(encounter[table_name]))

min_max_count_by_table_name_encounter = {}
for table_name in all_count_by_table_name_encounter:
    min_max_count_by_table_name_encounter[table_name] = {}
    min_max_count_by_table_name_encounter[table_name]['minimum'] = min(all_count_by_table_name_encounter[table_name])
    min_max_count_by_table_name_encounter[table_name]['maximum'] = max(all_count_by_table_name_encounter[table_name])

pprint(min_max_count_by_table_name_encounter)

StarcoderdataJupyterScriptsDedupFiltered