03b_model/src/run_model.py
def select_inputs_targets(inputs, target, train_start_date, test_end_date, out_dir):
    '''select the variables you are interested in examining, 
    srcs must be a list using the exact variable names,
    it will return a list of dataframes, each dataframe corresponding to a site
    with the requested variables as columns'''
    inputs_df = pd.DataFrame()
    
    #print('Looking for sources: ', inputs)
    
    files_to_read = []
    for top,dirs,files in os.walk('02_munge/out/'):
        if top == '02_munge/out/D' or top == '02_munge/out/daily_summaries':
            files_to_read.extend([os.path.join(top,files) for files in files])
    
    for inp in inputs:
        var = '_'.join(inp.split('_')[:-1])
        site = inp.split('_')[-1]
        
        #if site == 'delsjmet':
        #    continue
        
        inp_file = [s for s in files_to_read if site in s]
        data = pd.read_csv(inp_file[0], parse_dates = True, index_col = 'datetime')
        try:
            inputs_df[inp] = data[var]
        except:
            continue
    
    #inputs_df = inputs_df.join(nerrs_input)
    inputs_df = inputs_df[train_start_date:test_end_date]
    
    
    #read in the salt front record
    target_df = pd.read_csv(os.path.join('03a_it_analysis', 'in', 'saltfront.csv'), parse_dates = True, index_col = 'datetime')
    target_df = target_df['saltfront_daily'].to_frame()
    target_df.index = pd.to_datetime(target_df.index.date)
    target_df = target_df[train_start_date:test_end_date]
    target_df.index = target_df.index.rename('datetime')
    
    #set everything below 54 to nan
    target_df_c = target_df.copy()
    mask = target_df_c['saltfront_daily'] < 54
    target_df_c.loc[mask,'saltfront_daily'] = np.nan
    
    inputs_xarray = inputs_df.to_xarray()
    target_xarray = target_df_c.to_xarray()
    
    if os.path.exists(os.path.join(out_dir,'inputs.zarr')):
        shutil.rmtree(os.path.join(out_dir,'inputs.zarr'))
    if os.path.exists(os.path.join(out_dir, 'target.zarr')):
        shutil.rmtree(os.path.join(out_dir, 'target.zarr'))
    
    inputs_xarray.to_zarr(os.path.join(out_dir,'inputs.zarr'))
    target_xarray.to_zarr(os.path.join(out_dir,'target.zarr'))
    
    return inputs_xarray, target_xarray
select the variables you are interested in examining, 
    srcs must be a list using the exact variable names,
    it will return a list of dataframes, each dataframe corresponding to a site
    with the requested variables as columns
select the variables you are interested in examining,
srcs must be a list using the exact variable names,
it will return a list of dataframes, each dataframe corresponding to a site
with the requested variables as columns
select the variables you are interested in examining srcs must be a list using the exact variable names it will return a list of dataframes each dataframe corresponding to a site with the requested variables as columns
def select_inputs_targets(inputs, target, train_start_date, test_end_date, out_dir):
    inputs_df = pd.DataFrame()
    files_to_read = []
    for top,dirs,files in os.walk('02_munge/out/'):
        if top == '02_munge/out/D' or top == '02_munge/out/daily_summaries':
            files_to_read.extend([os.path.join(top,files) for files in files])
    for inp in inputs:
        var = '_'.join(inp.split('_')[:-1])
        site = inp.split('_')[-1]
        inp_file = [s for s in files_to_read if site in s]
        data = pd.read_csv(inp_file[0], parse_dates = True, index_col = 'datetime')
        try:
            inputs_df[inp] = data[var]
        except:
            continue
    inputs_df = inputs_df[train_start_date:test_end_date]
    target_df = pd.read_csv(os.path.join('03a_it_analysis', 'in', 'saltfront.csv'), parse_dates = True, index_col = 'datetime')
    target_df = target_df['saltfront_daily'].to_frame()
    target_df.index = pd.to_datetime(target_df.index.date)
    target_df = target_df[train_start_date:test_end_date]
    target_df.index = target_df.index.rename('datetime')
    target_df_c = target_df.copy()
    mask = target_df_c['saltfront_daily'] < 54
    target_df_c.loc[mask,'saltfront_daily'] = np.nan
    inputs_xarray = inputs_df.to_xarray()
    target_xarray = target_df_c.to_xarray()
    if os.path.exists(os.path.join(out_dir,'inputs.zarr')):
        shutil.rmtree(os.path.join(out_dir,'inputs.zarr'))
    if os.path.exists(os.path.join(out_dir, 'target.zarr')):
        shutil.rmtree(os.path.join(out_dir, 'target.zarr'))
    inputs_xarray.to_zarr(os.path.join(out_dir,'inputs.zarr'))
    target_xarray.to_zarr(os.path.join(out_dir,'target.zarr'))
    return inputs_xarray, target_xarray
def select_inputs_targets ( inputs , target , train_start_date , test_end_date , out_dir ) : inputs_df = pd . DataFrame ( ) files_to_read = [ ] for top , dirs , files in os . walk ( '02_munge/out/' ) : if top == '02_munge/out/D' or top == '02_munge/out/daily_summaries' : files_to_read . extend ( [ os . path . join ( top , files ) for files in files ] ) for inp in inputs : var = '_' . join ( inp . split ( '_' ) [ : - 1 ] ) site = inp . split ( '_' ) [ - 1 ] inp_file = [ s for s in files_to_read if site in s ] data = pd . read_csv ( inp_file [ 0 ] , parse_dates = True , index_col = 'datetime' ) try : inputs_df [ inp ] = data [ var ] except : continue inputs_df = inputs_df [ train_start_date : test_end_date ] target_df = pd . read_csv ( os . path . join ( '03a_it_analysis' , 'in' , 'saltfront.csv' ) , parse_dates = True , index_col = 'datetime' ) target_df = target_df [ 'saltfront_daily' ] . to_frame ( ) target_df . index = pd . to_datetime ( target_df . index . date ) target_df = target_df [ train_start_date : test_end_date ] target_df . index = target_df . index . rename ( 'datetime' ) target_df_c = target_df . copy ( ) mask = target_df_c [ 'saltfront_daily' ] < 54 target_df_c . loc [ mask , 'saltfront_daily' ] = np . nan inputs_xarray = inputs_df . to_xarray ( ) target_xarray = target_df_c . to_xarray ( ) if os . path . exists ( os . path . join ( out_dir , 'inputs.zarr' ) ) : shutil . rmtree ( os . path . join ( out_dir , 'inputs.zarr' ) ) if os . path . exists ( os . path . join ( out_dir , 'target.zarr' ) ) : shutil . rmtree ( os . path . join ( out_dir , 'target.zarr' ) ) inputs_xarray . to_zarr ( os . path . join ( out_dir , 'inputs.zarr' ) ) target_xarray . to_zarr ( os . path . join ( out_dir , 'target.zarr' ) ) return inputs_xarray , target_xarray
select the variables you are interested in examining,
srcs must be a list using the exact variable names,
it will return a list of dataframes, each dataframe corresponding to a site
with the requested variables as columns
select the variables you are interested in examining srcs must be a list using the exact variable names it will return a list of dataframes each dataframe corresponding to a site with the requested variables as columns
'''select the variables you are interested in examining, 
    srcs must be a list using the exact variable names,
    it will return a list of dataframes, each dataframe corresponding to a site
    with the requested variables as columns''' #print('Looking for sources: ', inputs) #if site == 'delsjmet': #    continue #inputs_df = inputs_df.join(nerrs_input) #read in the salt front record #set everything below 54 to nan
inputs
target
train_start_date
test_end_date
out_dir