# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:py37]
#     language: python
#     name: conda-env-py37-py
# ---

# # Arctic Heat IWG data Cleanup
#
# The IWG flight data from ArcticHeat flights can be parsed and cleaned to ease analysis and minimize transmission costs
#
# The big two tasks are:
# + clean up the repeated headerlines
# + choose just the columns you wish to keep

# ### Specify files to work with

# +
file_path = '/Users/bell/ecoraid/2019/Additional_FieldData/ArcticHeat/IWG_data/'
file_name = '20190720_231146_IWG.dat'

df = file_path+file_name
# -

# Include standard package imports 

import pandas as pd
import os


# +
#read in file and drop all duplicate header lines leaving only the first occurance
ds = pd.read_csv(df,header=None)
ds.drop_duplicates(inplace=True) 

column_names = (ds.loc[ds[0] == 'IWG1_NAMES']).values[0]
# -

# find the first occurence of the header line and use it to name columns (and drop it)
ds.drop((ds.loc[ds[0] == 'IWG1_NAMES']).index,inplace=True)
ds.columns = column_names

# ### A quick summary of all contents of the file

ds.describe()

ds.columns

# ### Fields to keep and smaller file output
#
# # copy wanted header fields as shown above to the keep_columns variable below

keep_columns = ['TIME','LAT', 'LON', 'ALTGPS','SST', 'PYRAUCLEAR', 'RH']

ds[keep_columns].info()

ds[keep_columns].to_csv(file_path+file_name.replace('.dat','.clean.csv'))

# +
o_size=os.stat(df).st_size
n_size=os.stat(df.replace('.dat','.clean.csv')).st_size

#new fstring format instead of or format()
print(f"The original file was {o_size / 1024 / 1000 :02.2f}MB")
print(f"The new file is {n_size / 1024 / 1000 :02.2f}MB")
# -



StarcoderdataJupyterScriptsDedupFiltered