/ ---
/ jupyter:
/   jupytext:
/     text_representation:
/       extension: .q
/       format_name: light
/       format_version: '1.5'
/       jupytext_version: 1.14.4
/ ---

/ + cell_id="4019f2a0-3bbc-40d0-8867-6f1a2711c57e" deepnote_cell_height=426 deepnote_cell_type="code" owner_user_id="90b26b03-99c4-40c0-a870-8175b99a6ae3" tags=[]
cleaned_dataframes = {}

#Iterate through all the data files
for idx in range(len(file_paths)):

    #Clean the data for the particular occupation
    cleaned_df = data_cleaner(files[idx])

    #Get the occupation name
    df_name = file_paths[idx][12:-4]
    print(df_name)

    #Display cleaned data
    display(cleaned_df.head())

    #Store in dictionary for retrieval
    cleaned_dataframes[df_name] = cleaned_df

    #Export to CSV
    cleaned_df.to_csv('../data/interim/'+df_name+'-cleaned.csv',index=False)


/ + [markdown] cell_id="03c3cc3df9a24fa686b9e5deee8fd26d" deepnote_cell_height=62 deepnote_cell_type="markdown" tags=[]
/ ### 6. Merge cleaned data

/ + [markdown] cell_id="92985583867f44fa9a72861178a364be" deepnote_cell_height=54 deepnote_cell_type="markdown" tags=[]
/ #### a. Get the paths to all the cleaned CSV files

/ + cell_id="8ccf3d5e92d34912b6bdb0d0549a2196" deepnote_cell_height=84 deepnote_cell_type="code" tags=[]
cleaned_files_paths = [file for file in glob('../data/interim/*-cleaned.csv')]
cleaned_files_paths

/ + [markdown] cell_id="56df2bbad6354017a70803644c23443b" deepnote_cell_height=54 deepnote_cell_type="markdown" tags=[]
/ #### b. Read the cleaned files into CSVs and store in list

/ + cell_id="82440657414e4fe1b6ed7ccb5445e033" deepnote_cell_height=84 deepnote_cell_type="code" tags=[]
#Read the cleaned files into a CSV
cleaned_dfs = [pd.read_csv(path,low_memory=False) for path in cleaned_files_paths ]

/ + [markdown] cell_id="575af8bfadbe410cad701643356f647c" deepnote_cell_height=54 deepnote_cell_type="markdown" tags=[]
/ #### c. Merge into one CSV

/ + cell_id="e9616d336a674355946beebfacb8f864" deepnote_cell_height=66 deepnote_cell_type="code" tags=[]
merged_df = pd.concat(cleaned_dfs,ignore_index=True)
print(merged_df.shape)
merged_df.head()

/ + [markdown] created_in_deepnote_cell=true deepnote_cell_type="markdown" tags=[]
/ <a style='text-decoration:none;line-height:16px;display:flex;color:#5B5B62;padding:10px;justify-content:end;' href='https://deepnote.com?utm_source=created-in-deepnote-cell&projectId=acc27b92-84be-4130-8026-204943f38189' target="_blank">
/ <img alt='Created in deepnote.<EMAIL>' style='display:inline;max-height:16px;margin:0px;margin-right:7.5px;' src='data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iODBweCIgaGVpZ<KEY>4IiB2aWV3Qm94PSIwIDAgODAgODAiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDU0LjEgKDc2NDkwKSAtIGh0dHBzOi8vc2tldGNoYXBwLmNvbSAtLT4KICAgIDx0aXRsZT5Hcm91cCAzPC90aXRsZT4KICAgIDxkZXNjPkNyZWF0ZWQgd2l0aCBTa2V0Y2guPC9kZXNjPgogICAgPGcgaWQ9IkxhbmRpbmciIHN0cm9rZT0ibm9uZSIgc3Ryb2tlLXdpZHRoPSIxIiBmaWxsPSJub25lIiBmaWxsLX<KEY>PS<KEY>ZCIgdHJhbnN<KEY>LTE<KEY>LC<KEY>KSI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0zIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgxMjM1LjAwMDAwMCwgNzkuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8cG9seWdvbiBpZD<KEY> > </img>
/ Created in <span style='font-weight:600;margin-left:4px;'>Deepnote</span></a>

StarcoderdataJupyterScriptsDedupFiltered