# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Import Modules
#
# This notebook is an extension to `snowpack.ipynb` which describes how the raw data was downloaded, formatted, and cleaned from the Department of Agriculture's API. Please see the main notebook for EDA, modeling, feature selection, etc.

import os, urllib, re
import pandas as pd
from tqdm import tqdm

# There are also some custom functions contained in `./functions/snowpack.py` that we will want to import

import functions.snowpack as sp

# # Download Raw Data

# First we need to get the meta data of site names so we can query the Department of Agriculture's API. We'll also define the list of variables needed for the `eCodes` parameter. The abbreviations used by the API are:
#
# * T = temperature (average, maximum, minimum, observered), $^\circ$F
# * PREC = yearly accumulated precipitation at start of day, inches
# * SNWD = snow depth at start of day, inches
# * WTEQ = Depth of water that would theoretically result if the entire snowpack were melted instantaneously, inches
# * STO:x = soil temperature observed at depth x inches, $^\circ$F
# * SMS:x = volumetric soil moisture at depth x inches, percent

# +
os.chdir('../data')
ntwk = pd.read_csv('snowpack-meta.csv')

eCodes = ['TAVG', 'TMAX', 'TMIN', 'TOBS', 'PREC', 'SNWD', 'WTEQ','STO','SMS']
eCodes = [s + '::value' for s in eCodes]
eCodes = [s.replace('STO', 'STO:-2:value,STO:-8:value,STO:-20:value') for s in eCodes]
eCodes = [s.replace('SMS', 'SMS:-2:value,SMS:-8:value,SMS:-20:value') for s in eCodes]
eCodes = ','.join(eCodes)
# -

# Now we've gathered all the information needed to read in data from website API for each row in snow-meta.csv. Due to the large amount of data, this will take a **long** time (~30 minutes) to run, so only do so if you actually want to get the raw data onto your computer.

for row in tqdm(ntwk.itertuples()): 
    temp = sp.ReadSnowData(row.state, row.site_id, eCodes)
    if temp is not None:
        if row.Index != 1:
            master = master.append(temp)
        else:
            master = temp
    else:
        continue

# # Data Cleaning and Export
#
# * Remove `NA`s
# * Rename columns to shorter abbreviations
# * Add meta data (latitude, longitude, elevation, etc.)
# * Parse dates into additional formats

clean = master.dropna()
clean.rename(columns = {'temp avg (degf)':'tAvg',
                        'temp max (degf)':'tMax',
                        'temp min (degf)':'tMin',
                        'temp  (degf)':'t',
                        'precip accum (in)':'precipAccum',
                        'snow  (in)':'snow',
                        'snow water equiv (in)':'waterEquiv',
                        'soil temp  2in (degf)':'tSoil2',
                        'soil temp  8in (degf)':'tSoil8'}, inplace=True)
meta = ntwk[['state', 'site_name', 'latitude', 'longitude', 'elev_ft', 'county', 'huc', 'site_id']]
meta.rename(columns = {'site_name':'name', 
                       'latitude':'lat', 
                       'longitude':'long', 
                       'elev_ft':'elev', 
                       'site_id':'id'}, inplace=True)
clean = pd.merge(clean, meta)
clean = clean[clean['snow'].notnull()]
clean.insert(loc=0, column='md',    value=clean.date.str.extract(r'((?<=-)\d{2}-\d{2})', expand=False))
clean.insert(loc=0, column='day',   value=clean.date.str.extract(r'((?<=-)\d{2}$)',      expand=False))
clean.insert(loc=0, column='month', value=clean.date.str.extract(r'((?<=-)\d{2}(?=-))',  expand=False))
clean.insert(loc=0, column='year',  value=clean.date.str.extract(r'(\d{4}(?=-))',        expand=False))

# Export a copy of both the original and cleaned data

master.to('snow-raw.csv', index=False)
clean.to_csv('snow-clean.csv', index=False)

StarcoderdataJupyterScriptsDedupFiltered